From cc41068e6b4566633ebc7a841290ce167671d2bf Mon Sep 17 00:00:00 2001 From: Timotej Lazar Date: Tue, 18 Apr 2017 17:25:18 +0200 Subject: Store student submissions in files instead of .pickle --- test-rules.py | 100 ++++++++++++++++++++++------------------------------------ 1 file changed, 38 insertions(+), 62 deletions(-) (limited to 'test-rules.py') diff --git a/test-rules.py b/test-rules.py index d129eef..c47ca10 100755 --- a/test-rules.py +++ b/test-rules.py @@ -1,18 +1,20 @@ #!/usr/bin/python3 +import argparse import collections import json import os.path -import pickle import re from statistics import mean -import sys from termcolor import colored -from monkey.action import parse as parse_trace from monkey.patterns import get_patterns -from prolog.util import parse as prolog_parse, rename_vars_list, stringify, tokenize + +parser = argparse.ArgumentParser(description='Evaluate rules on student programs.') +parser.add_argument('path', help='path to data directory') +args = parser.parse_args() +data_dir = args.path # klass: T/F # condition: list of patterns @@ -39,33 +41,13 @@ class Submission(collections.namedtuple('Submission', ['program', 'correct', 'pa class Hint(collections.namedtuple('Hint', ['ok', 'remove', 'add', 'add_alternatives'])): pass -# script arguments -solutions_file = sys.argv[1] -data_dir = sys.argv[2] - -pid_file = os.path.join(data_dir, 'pid') -attributes_file = os.path.join(data_dir, 'attributes.tab') -rules_file = os.path.join(data_dir, 'rules.txt') -users_file = os.path.join(data_dir, 'users-test.txt') -programs_file = os.path.join(data_dir, 'programs.pickle') -json_file = os.path.join(data_dir, 'bugs.json') - -pid = int(open(pid_file, 'r').read().strip()) -# read test results for known programs -test = pickle.load(open(programs_file, 'rb')) - -# read traces -users = [int(line.strip()) for line in open(users_file, 'r').readlines()] -traces = {} -for solution in pickle.load(open(solutions_file, 'rb')): - if solution.problem_id == pid and solution.codeq_user_id in users: - traces[solution.codeq_user_id] = solution.trace - # read attributes +attributes_file = os.path.join(data_dir, 'attributes.tab') attributes = dict([line.strip().split('\t') for line in open(attributes_file, 'r').readlines()]) attributes_ordered = [line.strip().split('\t')[1] for line in open(attributes_file, 'r').readlines()] # read rules +rules_file = os.path.join(data_dir, 'rules.txt') rules = [] for line in open(rules_file, 'r').readlines(): match = re.match(r'IF ((?:a[0-9]*!=F(?: AND )*)*) THEN correct=([TF]) *\[ *([0-9]*) *([0-9]*)\] *([0-9.]*)', line.strip()) @@ -76,6 +58,8 @@ for line in open(rules_file, 'r').readlines(): else: print('Did not understand rule:', line.strip()) +# export rules for tutor +json_file = os.path.join(data_dir, 'bugs.json') json_data = { 'patterns': attributes_ordered, 'rules': [{ @@ -85,8 +69,6 @@ json_data = { 'quality': r.quality, } for r in rules], } - -# export rules for tutor with open(json_file, 'w') as f: json.dump(json_data, f, sort_keys=True, indent=2) @@ -158,48 +140,42 @@ def suggest_true(rules, patterns): return None +# read traces +users_file = os.path.join(data_dir, 'users-test.txt') +users = [int(line.strip()) for line in open(users_file, 'r').readlines()] + # evaluate hints on student traces submissions = collections.defaultdict(list) -for user, trace in traces.items(): - # get submissions for this user - user_submissions = [] - code = '' - for action in parse_trace(trace): - code = action.apply(code) - if action.type == 'test': - # skip syntactically incorrect submissions - if prolog_parse(code) is None: - continue - - normalized_code = stringify(rename_vars_list(tokenize(code))) - # skip repeated submissions - if user_submissions and normalized_code == user_submissions[-1].program: - continue - # skip submissions without cached test results - if normalized_code not in test: - continue - correct = test[normalized_code]['n_tests'] == test[normalized_code]['n_passed'] - - # check rules for this submission - program_patterns = list(get_patterns(normalized_code)) - hint = suggest_buggy(rules, program_patterns) - if not hint: - hint = suggest_true(rules, program_patterns) - user_submissions.append(Submission(normalized_code, correct, program_patterns, hint)) - - # skip submissions after the first correct program - if correct: - break +for user in users: + user_subs = [] + user_dir = os.path.join(data_dir, 'submissions', str(user)) + # each submission is in a file named -- + for submission in sorted(os.listdir(user_dir), key=lambda x: int(x.split('-')[0])): + seq, total, passed = submission.split('-') + correct = total == passed + with open(os.path.join(user_dir, submission), 'r') as f: + code = f.read() + + # check rules for this submission + program_patterns = list(get_patterns(code)) + hint = suggest_buggy(rules, program_patterns) + if not hint: + hint = suggest_true(rules, program_patterns) + user_subs.append(Submission(code, correct, program_patterns, hint)) + + # skip submissions after the first correct program + if correct: + break # ignore traces with no / only correct submissions - if (not any(s.correct for s in user_submissions) or - all(s.correct for s in user_submissions)): + if (not any(s.correct for s in user_subs) or + all(s.correct for s in user_subs)): continue - submissions[user] = user_submissions + submissions[user] = user_subs # print submissions with hints for debugging - for s in user_submissions: + for s in user_subs: print('PASS' if s.correct else 'FAIL', end='\t') marks = [] if s.hint and s.hint.remove: -- cgit v1.2.1