diff options
| -rw-r--r-- | abml/evaluate.py | 22 | ||||
| -rw-r--r-- | abml/learn_dist.py | 11 | ||||
| -rw-r--r-- | abml/rules_prolog.py | 14 | ||||
| -rwxr-xr-x | test-rules.py | 100 |
4 files changed, 62 insertions, 85 deletions
diff --git a/abml/evaluate.py b/abml/evaluate.py index 2e318fb..cb959ed 100644 --- a/abml/evaluate.py +++ b/abml/evaluate.py @@ -1,4 +1,4 @@ -import pickle +import os.path import argparse import Orange from Orange.evaluation import TestOnTestData, CA, AUC, LogLoss @@ -8,30 +8,28 @@ import orangecontrib.evcrules.logistic as logistic import orangecontrib.abml.abrules as rules import orangecontrib.abml.argumentation as arg -parser = argparse.ArgumentParser(description='Learn and test rules for prolog programs.') -parser.add_argument('Name', type=str, help='Predicate name.') +parser = argparse.ArgumentParser(description='Learn and evaluate rules for Prolog programs.') +parser.add_argument('path', help='path to data directory') args = parser.parse_args() -name = args.Name +path = args.path # load data -data = Orange.data.Table('data/{}/programs-train'.format(name)) +data = Orange.data.Table(os.path.join(path, 'programs-train')) # create learner -rule_learner = rp.Rules4Prolog(name, 0.9) - - +rule_learner = rp.Rules4Prolog(path, 0.9) # learn a classifier classifier = rule_learner(data) # save model -fmodel = open("data/{}/model.txt".format(name), "wt") +fmodel = open(os.path.join(path, 'model.txt'), "wt") for r in classifier.rule_list: print(r, r.curr_class_dist, r.quality) fmodel.write("{} dist={} quality={}\n".format(str(r), str(r.curr_class_dist), r.quality)) # accuracy of model -testdata = Orange.data.Table('data/{}/programs-test'.format(name)) +testdata = Orange.data.Table(os.path.join(path, 'programs-test')) predictions = classifier(testdata) acc = 0 for i, p in enumerate(predictions): @@ -64,12 +62,12 @@ scores += "CA\tAUC\tLogLoss\tMethod\n" for ni, n in enumerate(names): scores += "{}\t{}\t{}\t{}\n".format(ca[ni], auc[ni], ll[ni], n) print(scores) -fscores = open("data/{}/scores.txt".format(name), "wt") +fscores = open(os.path.join(path, 'scores.txt'), 'wt') fscores.write(scores) all_rules = classifier.rule_list all_rules.sort(key = lambda r: r.quality, reverse=True) -rfile = open("data/{}/rules.txt".format(name), "wt") +rfile = open(os.path.join(path, 'rules.txt'), 'wt') for r in all_rules: print(r, r.curr_class_dist, r.quality) rfile.write("{} {} {}\n".format(r, r.curr_class_dist, r.quality)) diff --git a/abml/learn_dist.py b/abml/learn_dist.py index 58e4968..44e19c3 100644 --- a/abml/learn_dist.py +++ b/abml/learn_dist.py @@ -1,15 +1,16 @@ +import os.path import pickle import argparse from Orange.data import Table import abml.rules_prolog as rp parser = argparse.ArgumentParser(description='Learn and test rules for prolog programs.') -parser.add_argument('Name', type=str, help='Predicate name.') +parser.add_argument('path', help='path to data directory') args = parser.parse_args() -name = args.Name +path = args.path -data = Table('data/{}/programs-train'.format(name)) +data = Table(os.path.join(path, 'programs-train')) -rule_learner = rp.create_learner(name, evds=False) +rule_learner = rp.create_learner(path, evds=False) rule_learner.calculate_evds(data) -pickle.dump(rule_learner.evds, open("data/{}/evds.pickle".format(name), "wb")) +pickle.dump(rule_learner.evds, open(os.path.join(path, 'evds.pickle'), "wb")) diff --git a/abml/rules_prolog.py b/abml/rules_prolog.py index 9edd674..f67ba93 100644 --- a/abml/rules_prolog.py +++ b/abml/rules_prolog.py @@ -1,6 +1,8 @@ -import numpy as np -import pickle import itertools +import os.path +import pickle + +import numpy as np from Orange.classification.rules import _RuleClassifier, GuardianValidator import orangecontrib.abml.abrules as rules from Orange.classification.rules import Rule @@ -92,14 +94,14 @@ class NegativeFirstClassifier(_RuleClassifier): return probabilities class Rules4Prolog: - def __init__(self, name, threshold): + def __init__(self, path, threshold): self.threshold = threshold self.learner = rules.ABRuleLearner(width=50, parent_alpha=0.05) self.learner.rule_finder.general_validator = TrueCondValidator(self.learner.rule_finder.general_validator.max_rule_length, self.learner.rule_finder.general_validator.min_covered_examples) self.learner.rule_validator = PureAccuracyValidator(0, self.threshold) self.learner.classifier = NegativeFirstClassifier - self.learner.evds = pickle.load(open("data/{}/evds.pickle".format(name), "rb")) + self.learner.evds = pickle.load(open(os.path.join(path, 'evds.pickle'), 'rb')) def __call__(self, data): # first learn rules for negative class (quality should be higher than @@ -161,13 +163,13 @@ class Rules4Prolog: -def create_learner(name, evds=True): +def create_learner(path, evds=True): rule_learner = rules.ABRuleLearner(width=50, parent_alpha=0.05) rule_learner.rule_finder.general_validator = TrueCondValidator(rule_learner.rule_finder.general_validator.max_rule_length, rule_learner.rule_finder.general_validator.min_covered_examples) rule_learner.rule_validator = PureAccuracyValidator(0, 0.8) rule_learner.classifier = NegativeFirstClassifier if evds: - rule_learner.evds = pickle.load(open("data/{}/evds.pickle".format(name), "rb")) + rule_learner.evds = pickle.load(open(os.path.join(path, 'evds.pickle'), 'rb')) return rule_learner diff --git a/test-rules.py b/test-rules.py index d129eef..c47ca10 100755 --- a/test-rules.py +++ b/test-rules.py @@ -1,18 +1,20 @@ #!/usr/bin/python3 +import argparse import collections import json import os.path -import pickle import re from statistics import mean -import sys from termcolor import colored -from monkey.action import parse as parse_trace from monkey.patterns import get_patterns -from prolog.util import parse as prolog_parse, rename_vars_list, stringify, tokenize + +parser = argparse.ArgumentParser(description='Evaluate rules on student programs.') +parser.add_argument('path', help='path to data directory') +args = parser.parse_args() +data_dir = args.path # klass: T/F # condition: list of patterns @@ -39,33 +41,13 @@ class Submission(collections.namedtuple('Submission', ['program', 'correct', 'pa class Hint(collections.namedtuple('Hint', ['ok', 'remove', 'add', 'add_alternatives'])): pass -# script arguments -solutions_file = sys.argv[1] -data_dir = sys.argv[2] - -pid_file = os.path.join(data_dir, 'pid') -attributes_file = os.path.join(data_dir, 'attributes.tab') -rules_file = os.path.join(data_dir, 'rules.txt') -users_file = os.path.join(data_dir, 'users-test.txt') -programs_file = os.path.join(data_dir, 'programs.pickle') -json_file = os.path.join(data_dir, 'bugs.json') - -pid = int(open(pid_file, 'r').read().strip()) -# read test results for known programs -test = pickle.load(open(programs_file, 'rb')) - -# read traces -users = [int(line.strip()) for line in open(users_file, 'r').readlines()] -traces = {} -for solution in pickle.load(open(solutions_file, 'rb')): - if solution.problem_id == pid and solution.codeq_user_id in users: - traces[solution.codeq_user_id] = solution.trace - # read attributes +attributes_file = os.path.join(data_dir, 'attributes.tab') attributes = dict([line.strip().split('\t') for line in open(attributes_file, 'r').readlines()]) attributes_ordered = [line.strip().split('\t')[1] for line in open(attributes_file, 'r').readlines()] # read rules +rules_file = os.path.join(data_dir, 'rules.txt') rules = [] for line in open(rules_file, 'r').readlines(): match = re.match(r'IF ((?:a[0-9]*!=F(?: AND )*)*) THEN correct=([TF]) *\[ *([0-9]*) *([0-9]*)\] *([0-9.]*)', line.strip()) @@ -76,6 +58,8 @@ for line in open(rules_file, 'r').readlines(): else: print('Did not understand rule:', line.strip()) +# export rules for tutor +json_file = os.path.join(data_dir, 'bugs.json') json_data = { 'patterns': attributes_ordered, 'rules': [{ @@ -85,8 +69,6 @@ json_data = { 'quality': r.quality, } for r in rules], } - -# export rules for tutor with open(json_file, 'w') as f: json.dump(json_data, f, sort_keys=True, indent=2) @@ -158,48 +140,42 @@ def suggest_true(rules, patterns): return None +# read traces +users_file = os.path.join(data_dir, 'users-test.txt') +users = [int(line.strip()) for line in open(users_file, 'r').readlines()] + # evaluate hints on student traces submissions = collections.defaultdict(list) -for user, trace in traces.items(): - # get submissions for this user - user_submissions = [] - code = '' - for action in parse_trace(trace): - code = action.apply(code) - if action.type == 'test': - # skip syntactically incorrect submissions - if prolog_parse(code) is None: - continue - - normalized_code = stringify(rename_vars_list(tokenize(code))) - # skip repeated submissions - if user_submissions and normalized_code == user_submissions[-1].program: - continue - # skip submissions without cached test results - if normalized_code not in test: - continue - correct = test[normalized_code]['n_tests'] == test[normalized_code]['n_passed'] - - # check rules for this submission - program_patterns = list(get_patterns(normalized_code)) - hint = suggest_buggy(rules, program_patterns) - if not hint: - hint = suggest_true(rules, program_patterns) - user_submissions.append(Submission(normalized_code, correct, program_patterns, hint)) - - # skip submissions after the first correct program - if correct: - break +for user in users: + user_subs = [] + user_dir = os.path.join(data_dir, 'submissions', str(user)) + # each submission is in a file named <seq. no>-<total tests>-<passed tests> + for submission in sorted(os.listdir(user_dir), key=lambda x: int(x.split('-')[0])): + seq, total, passed = submission.split('-') + correct = total == passed + with open(os.path.join(user_dir, submission), 'r') as f: + code = f.read() + + # check rules for this submission + program_patterns = list(get_patterns(code)) + hint = suggest_buggy(rules, program_patterns) + if not hint: + hint = suggest_true(rules, program_patterns) + user_subs.append(Submission(code, correct, program_patterns, hint)) + + # skip submissions after the first correct program + if correct: + break # ignore traces with no / only correct submissions - if (not any(s.correct for s in user_submissions) or - all(s.correct for s in user_submissions)): + if (not any(s.correct for s in user_subs) or + all(s.correct for s in user_subs)): continue - submissions[user] = user_submissions + submissions[user] = user_subs # print submissions with hints for debugging - for s in user_submissions: + for s in user_subs: print('PASS' if s.correct else 'FAIL', end='\t') marks = [] if s.hint and s.hint.remove: |