Store student submissions in files instead of .pickle

4 files changed, 62 insertions, 85 deletions

diff --git a/abml/evaluate.py b/abml/evaluate.py
index 2e318fb..cb959ed 100644
--- a/abml/evaluate.py
+++ b/abml/evaluate.py
@@ -1,4 +1,4 @@
-import pickle
+import os.path
 import argparse
 import Orange
 from Orange.evaluation import TestOnTestData, CA, AUC, LogLoss
@@ -8,30 +8,28 @@ import orangecontrib.evcrules.logistic as logistic
 import orangecontrib.abml.abrules as rules
 import orangecontrib.abml.argumentation as arg
 
-parser = argparse.ArgumentParser(description='Learn and test rules for prolog programs.')
-parser.add_argument('Name', type=str, help='Predicate name.')
+parser = argparse.ArgumentParser(description='Learn and evaluate rules for Prolog programs.')
+parser.add_argument('path', help='path to data directory')
 args = parser.parse_args()
-name = args.Name
+path = args.path
 
 # load data
-data = Orange.data.Table('data/{}/programs-train'.format(name))
+data = Orange.data.Table(os.path.join(path, 'programs-train'))
 
 # create learner
-rule_learner = rp.Rules4Prolog(name, 0.9)
-
-
+rule_learner = rp.Rules4Prolog(path, 0.9)
 
 # learn a classifier
 classifier = rule_learner(data)
 
 # save model
-fmodel = open("data/{}/model.txt".format(name), "wt")
+fmodel = open(os.path.join(path, 'model.txt'), "wt")
 for r in classifier.rule_list:
     print(r, r.curr_class_dist, r.quality)
     fmodel.write("{} dist={} quality={}\n".format(str(r), str(r.curr_class_dist), r.quality))
 
 # accuracy of model
-testdata = Orange.data.Table('data/{}/programs-test'.format(name))
+testdata = Orange.data.Table(os.path.join(path, 'programs-test'))
 predictions = classifier(testdata)
 acc = 0
 for i, p in enumerate(predictions):
@@ -64,12 +62,12 @@ scores += "CA\tAUC\tLogLoss\tMethod\n"
 for ni, n in enumerate(names):
     scores += "{}\t{}\t{}\t{}\n".format(ca[ni], auc[ni], ll[ni], n)
 print(scores)
-fscores = open("data/{}/scores.txt".format(name), "wt")
+fscores = open(os.path.join(path, 'scores.txt'), 'wt')
 fscores.write(scores)
 
 all_rules = classifier.rule_list
 all_rules.sort(key = lambda r: r.quality, reverse=True)
-rfile = open("data/{}/rules.txt".format(name), "wt")
+rfile = open(os.path.join(path, 'rules.txt'), 'wt')
 for r in all_rules:
     print(r, r.curr_class_dist, r.quality)
     rfile.write("{} {} {}\n".format(r, r.curr_class_dist, r.quality))
diff --git a/abml/learn_dist.py b/abml/learn_dist.py
index 58e4968..44e19c3 100644
--- a/abml/learn_dist.py
+++ b/abml/learn_dist.py
@@ -1,15 +1,16 @@
+import os.path
 import pickle
 import argparse
 from Orange.data import Table
 import abml.rules_prolog as rp
 
 parser = argparse.ArgumentParser(description='Learn and test rules for prolog programs.')
-parser.add_argument('Name', type=str, help='Predicate name.')
+parser.add_argument('path', help='path to data directory')
 args = parser.parse_args()
-name = args.Name
+path = args.path
 
-data = Table('data/{}/programs-train'.format(name))
+data = Table(os.path.join(path, 'programs-train'))
 
-rule_learner = rp.create_learner(name, evds=False)
+rule_learner = rp.create_learner(path, evds=False)
 rule_learner.calculate_evds(data)
-pickle.dump(rule_learner.evds, open("data/{}/evds.pickle".format(name), "wb"))
+pickle.dump(rule_learner.evds, open(os.path.join(path, 'evds.pickle'), "wb"))
diff --git a/abml/rules_prolog.py b/abml/rules_prolog.py
index 9edd674..f67ba93 100644
--- a/abml/rules_prolog.py
+++ b/abml/rules_prolog.py
@@ -1,6 +1,8 @@
-import numpy as np
-import pickle
 import itertools
+import os.path
+import pickle
+
+import numpy as np
 from Orange.classification.rules import _RuleClassifier, GuardianValidator
 import orangecontrib.abml.abrules as rules
 from Orange.classification.rules import Rule
@@ -92,14 +94,14 @@ class NegativeFirstClassifier(_RuleClassifier):
         return probabilities
 
 class Rules4Prolog:
-    def __init__(self, name, threshold):
+    def __init__(self, path, threshold):
         self.threshold = threshold
         self.learner = rules.ABRuleLearner(width=50, parent_alpha=0.05)
         self.learner.rule_finder.general_validator = TrueCondValidator(self.learner.rule_finder.general_validator.max_rule_length,
                                                                        self.learner.rule_finder.general_validator.min_covered_examples)
         self.learner.rule_validator = PureAccuracyValidator(0, self.threshold)
         self.learner.classifier = NegativeFirstClassifier
-        self.learner.evds = pickle.load(open("data/{}/evds.pickle".format(name), "rb"))
+        self.learner.evds = pickle.load(open(os.path.join(path, 'evds.pickle'), 'rb'))
 
     def __call__(self, data):
         # first learn rules for negative class (quality should be higher than
@@ -161,13 +163,13 @@ class Rules4Prolog:
 
 
 
-def create_learner(name, evds=True):
+def create_learner(path, evds=True):
     rule_learner = rules.ABRuleLearner(width=50, parent_alpha=0.05)
     rule_learner.rule_finder.general_validator = TrueCondValidator(rule_learner.rule_finder.general_validator.max_rule_length,
                                                                    rule_learner.rule_finder.general_validator.min_covered_examples)
     rule_learner.rule_validator = PureAccuracyValidator(0, 0.8)
     rule_learner.classifier = NegativeFirstClassifier
     if evds:
-        rule_learner.evds = pickle.load(open("data/{}/evds.pickle".format(name), "rb"))
+        rule_learner.evds = pickle.load(open(os.path.join(path, 'evds.pickle'), 'rb'))
     return rule_learner
 
diff --git a/test-rules.py b/test-rules.py
index d129eef..c47ca10 100755
--- a/test-rules.py
+++ b/test-rules.py
@@ -1,18 +1,20 @@
 #!/usr/bin/python3
 
+import argparse
 import collections
 import json
 import os.path
-import pickle
 import re
 from statistics import mean
-import sys
 
 from termcolor import colored
 
-from monkey.action import parse as parse_trace
 from monkey.patterns import get_patterns
-from prolog.util import parse as prolog_parse, rename_vars_list, stringify, tokenize
+
+parser = argparse.ArgumentParser(description='Evaluate rules on student programs.')
+parser.add_argument('path', help='path to data directory')
+args = parser.parse_args()
+data_dir = args.path
 
 # klass: T/F
 # condition: list of patterns
@@ -39,33 +41,13 @@ class Submission(collections.namedtuple('Submission', ['program', 'correct', 'pa
 class Hint(collections.namedtuple('Hint', ['ok', 'remove', 'add', 'add_alternatives'])):
     pass
 
-# script arguments
-solutions_file = sys.argv[1]
-data_dir = sys.argv[2]
-
-pid_file = os.path.join(data_dir, 'pid')
-attributes_file = os.path.join(data_dir, 'attributes.tab')
-rules_file = os.path.join(data_dir, 'rules.txt')
-users_file = os.path.join(data_dir, 'users-test.txt')
-programs_file = os.path.join(data_dir, 'programs.pickle')
-json_file = os.path.join(data_dir, 'bugs.json')
-
-pid = int(open(pid_file, 'r').read().strip())
-# read test results for known programs
-test = pickle.load(open(programs_file, 'rb'))
-
-# read traces
-users = [int(line.strip()) for line in open(users_file, 'r').readlines()]
-traces = {}
-for solution in pickle.load(open(solutions_file, 'rb')):
-    if solution.problem_id == pid and solution.codeq_user_id in users:
-        traces[solution.codeq_user_id] = solution.trace
-
 # read attributes
+attributes_file = os.path.join(data_dir, 'attributes.tab')
 attributes = dict([line.strip().split('\t') for line in open(attributes_file, 'r').readlines()])
 attributes_ordered = [line.strip().split('\t')[1] for line in open(attributes_file, 'r').readlines()]
 
 # read rules
+rules_file = os.path.join(data_dir, 'rules.txt')
 rules = []
 for line in open(rules_file, 'r').readlines():
     match = re.match(r'IF ((?:a[0-9]*!=F(?: AND )*)*) THEN correct=([TF]) *\[ *([0-9]*) *([0-9]*)\] *([0-9.]*)', line.strip())
@@ -76,6 +58,8 @@ for line in open(rules_file, 'r').readlines():
     else:
         print('Did not understand rule:', line.strip())
 
+# export rules for tutor
+json_file = os.path.join(data_dir, 'bugs.json')
 json_data = {
     'patterns': attributes_ordered,
     'rules': [{
@@ -85,8 +69,6 @@ json_data = {
         'quality': r.quality,
     } for r in rules],
 }
-
-# export rules for tutor
 with open(json_file, 'w') as f:
     json.dump(json_data, f, sort_keys=True, indent=2)
 
@@ -158,48 +140,42 @@ def suggest_true(rules, patterns):
 
     return None
 
+# read traces
+users_file = os.path.join(data_dir, 'users-test.txt')
+users = [int(line.strip()) for line in open(users_file, 'r').readlines()]
+
 # evaluate hints on student traces
 submissions = collections.defaultdict(list)
-for user, trace in traces.items():
-    # get submissions for this user
-    user_submissions = []
-    code = ''
-    for action in parse_trace(trace):
-        code = action.apply(code)
-        if action.type == 'test':
-            # skip syntactically incorrect submissions
-            if prolog_parse(code) is None:
-                continue
-
-            normalized_code = stringify(rename_vars_list(tokenize(code)))
-            # skip repeated submissions
-            if user_submissions and normalized_code == user_submissions[-1].program:
-                continue
-            # skip submissions without cached test results
-            if normalized_code not in test:
-                continue
-            correct = test[normalized_code]['n_tests'] == test[normalized_code]['n_passed']
-
-            # check rules for this submission
-            program_patterns = list(get_patterns(normalized_code))
-            hint = suggest_buggy(rules, program_patterns)
-            if not hint:
-                hint = suggest_true(rules, program_patterns)
-            user_submissions.append(Submission(normalized_code, correct, program_patterns, hint))
-
-            # skip submissions after the first correct program
-            if correct:
-                break
+for user in users:
+    user_subs = []
+    user_dir = os.path.join(data_dir, 'submissions', str(user))
+    # each submission is in a file named <seq. no>-<total tests>-<passed tests>
+    for submission in sorted(os.listdir(user_dir), key=lambda x: int(x.split('-')[0])):
+        seq, total, passed = submission.split('-')
+        correct = total == passed
+        with open(os.path.join(user_dir, submission), 'r') as f:
+            code = f.read()
+
+        # check rules for this submission
+        program_patterns = list(get_patterns(code))
+        hint = suggest_buggy(rules, program_patterns)
+        if not hint:
+            hint = suggest_true(rules, program_patterns)
+        user_subs.append(Submission(code, correct, program_patterns, hint))
+
+        # skip submissions after the first correct program
+        if correct:
+            break
 
     # ignore traces with no / only correct submissions
-    if (not any(s.correct for s in user_submissions) or
-        all(s.correct for s in user_submissions)):
+    if (not any(s.correct for s in user_subs) or
+        all(s.correct for s in user_subs)):
         continue
 
-    submissions[user] = user_submissions
+    submissions[user] = user_subs
 
     # print submissions with hints for debugging
-    for s in user_submissions:
+    for s in user_subs:
         print('PASS' if s.correct else 'FAIL', end='\t')
         marks = []
         if s.hint and s.hint.remove: