From 9763725d157003b02417a37e49788b5a8683f46a Mon Sep 17 00:00:00 2001 From: Timotej Lazar Date: Fri, 16 Jan 2015 15:07:32 +0100 Subject: Move testing code from monkey.py to test.py Drop monkey/db.py and use Django models instead. --- monkey/db.py | 44 -------------- monkey/monkey.py | 170 +------------------------------------------------------ monkey/test.py | 170 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 171 insertions(+), 213 deletions(-) delete mode 100644 monkey/db.py create mode 100755 monkey/test.py diff --git a/monkey/db.py b/monkey/db.py deleted file mode 100644 index 0634098..0000000 --- a/monkey/db.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/python3 - -import sqlite3 - -db = sqlite3.connect('misc/solutions.db') -db.row_factory = sqlite3.Row -db.text_factory = bytes -cursor = db.cursor() - -def b_to_utf8(bytestring): - return str(bytestring, encoding='utf-8') - -def get_problem_ids(): - cursor.execute('SELECT id FROM problems ORDER BY id ASC') - return [row['id'] for row in cursor] - -def get_problem(pid): - cursor.execute('SELECT name, solution, library FROM problems WHERE id=?', (pid,)) - row = cursor.fetchone() - name = b_to_utf8(row['name']) - solution = b_to_utf8(row['solution']).replace('\r', '') - lib_id = row['library'] if row['library'] else None - return name, solution, lib_id - -def get_depends(pid): - cursor.execute('SELECT dependency FROM depends WHERE problem=?', (pid,)) - return [r['dependency'] for r in cursor.fetchall()] - -def get_library(lid): - cursor.execute('SELECT facts FROM libraries WHERE id=?', (lid,)) - row = cursor.fetchone() - return b_to_utf8(row['facts']).replace('\r', '') if row else None - -def get_tests(pid): - cursor.execute('SELECT query FROM tests WHERE problem=?', (pid,)) - return [b_to_utf8(row['query']) for row in cursor] - -def get_traces(pid): - cursor.execute('SELECT * FROM attempts WHERE problem=? AND done=1 ORDER BY id ASC', (pid,)) - return {(pid, attempt['user']): attempt['log'] for attempt in cursor} - -def get_solved(uid): - cursor.execute('SELECT problem FROM attempts WHERE user=? AND done=1 ORDER BY problem ASC', (uid,)) - return [row['problem'] for row in cursor.fetchall()] diff --git a/monkey/monkey.py b/monkey/monkey.py index 8e805f5..693aa67 100755 --- a/monkey/monkey.py +++ b/monkey/monkey.py @@ -8,11 +8,10 @@ import time from termcolor import colored -from . import db from .action import parse from .edits import classify_edits, clean_graph, edit_graph, get_edits_from_traces from .graph import Node, graphviz -from .prolog.engine import PrologEngine +from .prolog.engine import test from .prolog.util import compose, decompose, map_vars, rename_vars, stringify from .util import PQueue, Token, indent @@ -144,170 +143,3 @@ def fix(name, code, edits, timeout=30, debug=False): add_task(new_lines, new_rules, prev=task, step=new_step, cost=cost) return '', [], total_time, n_tested - -def print_hint(solution, steps, fix_time, n_tested): - if solution: - print(colored('Hint found! Tested {} programs in {:.1f} s.'.format(n_tested, fix_time), 'green')) - print(colored(' Edits', 'blue')) - for line, (before, after) in steps: - print(' {}:\t{} → {}'.format(line, stringify(before), stringify(after))) - print(colored(' Final version', 'blue')) - print(indent(compose(*decompose(solution)), 2)) - else: - print(colored('Hint not found! Tested {} programs in {:.1f} s.'.format(n_tested, fix_time), 'red')) - -# Find official solutions to all problems. -def init_problems(): - names = {} - codes = {} - libraries = {} - - pids = db.get_problem_ids() - for pid in pids: - names[pid], codes[pid], libraries[pid] = db.get_problem(pid) - - return names, codes, libraries - -# Submit code to Prolog server for testing. -def test(name, code): - # TODO also load fact library and solved predicates - engine = PrologEngine(code=code) - result = engine.ask("run_tests({}, '{}')".format(name, engine.id)) - engine.destroy() - return result['event'] == 'success' - -if __name__ == '__main__': - # Get problem id from commandline. - if len(sys.argv) < 2: - print('usage: ' + sys.argv[0] + ' ') - sys.exit(1) - pid = int(sys.argv[1]) - - names, codes, libraries = init_problems() - - # Analyze traces for this problem to get edits, submissions and queries. - traces = db.get_traces(pid) - edits, lines, submissions, queries = get_edits_from_traces(traces.values()) - - # Find incorrect submissions. - incorrect = [] - for submission, count in sorted(submissions.items()): - if not test(names[pid], submission): - # This incorrect submission appeared in [count] attempts. - incorrect += [submission]*count - - # XXX only for testing - try: - done = pickle.load(open('status-'+str(pid)+'.pickle', 'rb')) - except: - done = [] - - # test fix() on incorrect student submissions - if len(sys.argv) >= 3 and sys.argv[2] == 'test': - timeout = int(sys.argv[3]) if len(sys.argv) >= 4 else 10 - - print('Fixing {}/{} programs (timeout={})…'.format( - len([p for p in incorrect if p not in done]), len(incorrect), timeout)) - - for i, program in enumerate(incorrect): - if program in done: - continue - print(colored('Analyzing program {0}/{1}…'.format(i+1, len(incorrect)), 'yellow')) - print(indent(compose(*decompose(program)), 2)) - - solution, steps, fix_time, n_tested = fix(names[pid], program, edits, timeout=timeout) - if solution: - done.append(program) - print_hint(solution, steps, fix_time, n_tested) - print() - - pickle.dump(done, open('status-'+str(pid)+'.pickle', 'wb')) - - print('Found hints for ' + str(len(done)) + ' of ' + str(len(incorrect)) + ' incorrect programs') - - # print info for this problem - elif len(sys.argv) >= 3 and sys.argv[2] == 'info': - # with no additional arguments, print some stats - if len(sys.argv) == 3: - print('Problem {} ({}): {} edits in {} traces, fixed {}/{} ({}/{} unique)'.format( - pid, colored(names[pid], 'yellow'), - colored(str(len(edits)), 'yellow'), colored(str(len(traces)), 'yellow'), - colored(str(len([p for p in incorrect if p in done])), 'yellow'), - colored(str(len(incorrect)), 'yellow'), - colored(str(len(set(done))), 'yellow'), - colored(str(len(set(incorrect))), 'yellow'))) - else: - if sys.argv[3] == 'users': - print(' '.join([str(uid) for (pid, uid) in sorted(traces.keys())])) - # print all observed edits and their costs - elif sys.argv[3] == 'edits': - inserts, removes, changes = classify_edits(edits) - print('Inserts') - for after, cost in sorted(inserts.items(), key=lambda x: x[1]): - print(' {:.2f}\t{}'.format(cost, stringify(after))) - print('Removes') - for before, cost in sorted(removes.items(), key=lambda x: x[1]): - print(' {:.2f}\t{}'.format(cost, stringify(before))) - print('Changes') - for (before, after), cost in sorted(changes.items(), key=lambda x: x[1]): - print(' {:.2f}\t{} → {}'.format(cost, - stringify(before if before else [('INVALID', 'ε')]), - stringify(after if after else [('INVALID', 'ε')]))) - # print all student submissions not (yet) corrected - elif sys.argv[3] == 'unsolved': - for p in sorted(set(incorrect)): - if p in done: - continue - print(indent(compose(*decompose(p)), 2)) - print() - # print all student queries and their counts - elif sys.argv[3] == 'queries': - for query, count in queries.most_common(): - print(' ' + str(count) + '\t' + query) - - # Print the edit graph in graphviz dot syntax. - elif len(sys.argv) == 4 and sys.argv[2] == 'graph': - uid = int(sys.argv[3]) - actions = parse(traces[(pid, uid)]) - - nodes, submissions, queries = edit_graph(actions) - - def position(node): - return (node.data[1]*150, node.data[0]*-60) - - def label(node): - return stringify(node.data[2]) - - def node_attr(node): - if node.ein and node.data[2] == node.ein[0].data[2]: - return 'color="gray", shape="point"' - return '' - - def edge_attr(a, b): - if a.data[2] == b.data[2]: - return 'arrowhead="none"' - return '' - - graphviz_str = graphviz(nodes, pos=position, label=label, - node_attr=node_attr, edge_attr=edge_attr) - print(graphviz_str) - - # run interactive loop - else: - while True: - # read the program from stdin - print('Enter program, end with empty line:') - code = '' - try: - while True: - line = input() - if not line: - break - code += line + '\n' - except EOFError: - break - - # try finding a fix - print(colored('Analyzing program…', 'yellow')) - solution, steps, fix_time, n_tested = fix(names[pid], code, edits, debug=True) - print_hint(solution, steps, fix_time, n_tested) diff --git a/monkey/test.py b/monkey/test.py new file mode 100755 index 0000000..3143274 --- /dev/null +++ b/monkey/test.py @@ -0,0 +1,170 @@ +#!/usr/bin/python3 + +import os +import pickle +import sys + +import django +from termcolor import colored + +from .action import parse +from .edits import classify_edits, edit_graph, get_edits_from_traces +from .graph import graphviz +from .monkey import fix +from .prolog.engine import test +from .prolog.util import compose, decompose, stringify +from .util import indent + +# Load django models. +os.environ['DJANGO_SETTINGS_MODULE'] = 'webmonkey.settings' +django.setup() +from django.contrib.auth.models import User +from tutor.models import Attempt, Problem + +# Get problem id from commandline. +if len(sys.argv) < 2: + print('usage: ' + sys.argv[0] + ' ') + sys.exit(1) +pid = int(sys.argv[1]) + +# Analyze traces for this problem to get edits, submissions and queries. +problem = Problem.objects.get(pk=pid) +attempts = Attempt.objects.filter(problem=problem) + +traces = [a.trace for a in attempts] +edits, lines, submissions, queries = get_edits_from_traces(traces) + +# Find incorrect submissions. +incorrect = [] +for submission, count in sorted(submissions.items()): + if not test(problem.name, submission): + # This incorrect submission appeared in [count] attempts. + incorrect += [submission]*count + +# Load current status (programs for which a hint was found). +try: + done = pickle.load(open('status-'+str(problem.pk)+'.pickle', 'rb')) +except: + done = [] + +def print_hint(solution, steps, fix_time, n_tested): + if solution: + print(colored('Hint found! Tested {} programs in {:.1f} s.'.format(n_tested, fix_time), 'green')) + print(colored(' Edits', 'blue')) + for line, (before, after) in steps: + print(' {}:\t{} → {}'.format(line, stringify(before), stringify(after))) + print(colored(' Final version', 'blue')) + print(indent(compose(*decompose(solution)), 2)) + else: + print(colored('Hint not found! Tested {} programs in {:.1f} s.'.format(n_tested, fix_time), 'red')) + +# Test fix() on incorrect student submissions. +if len(sys.argv) >= 3 and sys.argv[2] == 'test': + timeout = int(sys.argv[3]) if len(sys.argv) >= 4 else 10 + + print('Fixing {}/{} programs (timeout={})…'.format( + len([p for p in incorrect if p not in done]), len(incorrect), timeout)) + + for i, program in enumerate(incorrect): + if program in done: + continue + print(colored('Analyzing program {0}/{1}…'.format(i+1, len(incorrect)), 'yellow')) + print(indent(compose(*decompose(program)), 2)) + + solution, steps, fix_time, n_tested = fix(problem.name, program, edits, timeout=timeout) + if solution: + done.append(program) + print_hint(solution, steps, fix_time, n_tested) + print() + + pickle.dump(done, open('status-'+str(problem.pk)+'.pickle', 'wb')) + + print('Found hints for ' + str(len(done)) + ' of ' + str(len(incorrect)) + ' incorrect programs') + +# Print info for this problem. +elif len(sys.argv) >= 3 and sys.argv[2] == 'info': + # With no additional arguments, print some stats. + if len(sys.argv) == 3: + print('Problem {} ({}): {} edits in {} traces, fixed {}/{} ({}/{} unique)'.format( + problem.pk, colored(problem.name, 'yellow'), + colored(str(len(edits)), 'yellow'), colored(str(len(traces)), 'yellow'), + colored(str(len([p for p in incorrect if p in done])), 'yellow'), + colored(str(len(incorrect)), 'yellow'), + colored(str(len(set(done))), 'yellow'), + colored(str(len(set(incorrect))), 'yellow'))) + else: + if sys.argv[3] == 'users': + print(' '.join([str(a.user.pk) for a in attempts])) + # Print all observed edits and their costs. + elif sys.argv[3] == 'edits': + inserts, removes, changes = classify_edits(edits) + print('Inserts') + for after, cost in sorted(inserts.items(), key=lambda x: x[1]): + print(' {:.2f}\t{}'.format(cost, stringify(after))) + print('Removes') + for before, cost in sorted(removes.items(), key=lambda x: x[1]): + print(' {:.2f}\t{}'.format(cost, stringify(before))) + print('Changes') + for (before, after), cost in sorted(changes.items(), key=lambda x: x[1]): + print(' {:.2f}\t{} → {}'.format(cost, + stringify(before if before else [('INVALID', 'ε')]), + stringify(after if after else [('INVALID', 'ε')]))) + # Print all student submissions not (yet) corrected. + elif sys.argv[3] == 'unsolved': + for p in sorted(set(incorrect)): + if p in done: + continue + print(indent(compose(*decompose(p)), 2)) + print() + # Print all student queries and their counts. + elif sys.argv[3] == 'queries': + for query, count in queries.most_common(): + print(' ' + str(count) + '\t' + query) + +# Print the edit graph in graphviz dot syntax. +elif len(sys.argv) == 4 and sys.argv[2] == 'graph': + uid = int(sys.argv[3]) + user = User.objects.get(pk=uid) + actions = parse(Attempt.objects.get(problem=problem, user=user).trace) + + nodes, submissions, queries = edit_graph(actions) + + def position(node): + return (node.data[1]*150, node.data[0]*-60) + + def label(node): + return stringify(node.data[2]) + + def node_attr(node): + if node.ein and node.data[2] == node.ein[0].data[2]: + return 'color="gray", shape="point"' + return '' + + def edge_attr(a, b): + if a.data[2] == b.data[2]: + return 'arrowhead="none"' + return '' + + graphviz_str = graphviz(nodes, pos=position, label=label, + node_attr=node_attr, edge_attr=edge_attr) + print(graphviz_str) + +# Run interactive loop. +else: + while True: + # Read the program from stdin. + print('Enter program, end with empty line:') + code = '' + try: + while True: + line = input() + if not line: + break + code += line + '\n' + except EOFError: + break + + # Try finding a fix. + print(colored('Analyzing program…', 'yellow')) + solution, steps, fix_time, n_tested = fix(problem.name, code, edits, debug=True) + print_hint(solution, steps, fix_time, n_tested) -- cgit v1.2.1