summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTimotej Lazar <timotej.lazar@araneo.org>2015-01-16 15:07:32 +0100
committerAleš Smodiš <aless@guru.si>2015-08-11 14:26:01 +0200
commit9763725d157003b02417a37e49788b5a8683f46a (patch)
tree3de64dd85eae57780e27760b8cfd1405eff24f6c
parentbacad47b87c5c2f9a90734ef13ca98a0961a08bb (diff)
Move testing code from monkey.py to test.py
Drop monkey/db.py and use Django models instead.
-rw-r--r--monkey/db.py44
-rwxr-xr-xmonkey/monkey.py170
-rwxr-xr-xmonkey/test.py170
3 files changed, 171 insertions, 213 deletions
diff --git a/monkey/db.py b/monkey/db.py
deleted file mode 100644
index 0634098..0000000
--- a/monkey/db.py
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/usr/bin/python3
-
-import sqlite3
-
-db = sqlite3.connect('misc/solutions.db')
-db.row_factory = sqlite3.Row
-db.text_factory = bytes
-cursor = db.cursor()
-
-def b_to_utf8(bytestring):
- return str(bytestring, encoding='utf-8')
-
-def get_problem_ids():
- cursor.execute('SELECT id FROM problems ORDER BY id ASC')
- return [row['id'] for row in cursor]
-
-def get_problem(pid):
- cursor.execute('SELECT name, solution, library FROM problems WHERE id=?', (pid,))
- row = cursor.fetchone()
- name = b_to_utf8(row['name'])
- solution = b_to_utf8(row['solution']).replace('\r', '')
- lib_id = row['library'] if row['library'] else None
- return name, solution, lib_id
-
-def get_depends(pid):
- cursor.execute('SELECT dependency FROM depends WHERE problem=?', (pid,))
- return [r['dependency'] for r in cursor.fetchall()]
-
-def get_library(lid):
- cursor.execute('SELECT facts FROM libraries WHERE id=?', (lid,))
- row = cursor.fetchone()
- return b_to_utf8(row['facts']).replace('\r', '') if row else None
-
-def get_tests(pid):
- cursor.execute('SELECT query FROM tests WHERE problem=?', (pid,))
- return [b_to_utf8(row['query']) for row in cursor]
-
-def get_traces(pid):
- cursor.execute('SELECT * FROM attempts WHERE problem=? AND done=1 ORDER BY id ASC', (pid,))
- return {(pid, attempt['user']): attempt['log'] for attempt in cursor}
-
-def get_solved(uid):
- cursor.execute('SELECT problem FROM attempts WHERE user=? AND done=1 ORDER BY problem ASC', (uid,))
- return [row['problem'] for row in cursor.fetchall()]
diff --git a/monkey/monkey.py b/monkey/monkey.py
index 8e805f5..693aa67 100755
--- a/monkey/monkey.py
+++ b/monkey/monkey.py
@@ -8,11 +8,10 @@ import time
from termcolor import colored
-from . import db
from .action import parse
from .edits import classify_edits, clean_graph, edit_graph, get_edits_from_traces
from .graph import Node, graphviz
-from .prolog.engine import PrologEngine
+from .prolog.engine import test
from .prolog.util import compose, decompose, map_vars, rename_vars, stringify
from .util import PQueue, Token, indent
@@ -144,170 +143,3 @@ def fix(name, code, edits, timeout=30, debug=False):
add_task(new_lines, new_rules, prev=task, step=new_step, cost=cost)
return '', [], total_time, n_tested
-
-def print_hint(solution, steps, fix_time, n_tested):
- if solution:
- print(colored('Hint found! Tested {} programs in {:.1f} s.'.format(n_tested, fix_time), 'green'))
- print(colored(' Edits', 'blue'))
- for line, (before, after) in steps:
- print(' {}:\t{} → {}'.format(line, stringify(before), stringify(after)))
- print(colored(' Final version', 'blue'))
- print(indent(compose(*decompose(solution)), 2))
- else:
- print(colored('Hint not found! Tested {} programs in {:.1f} s.'.format(n_tested, fix_time), 'red'))
-
-# Find official solutions to all problems.
-def init_problems():
- names = {}
- codes = {}
- libraries = {}
-
- pids = db.get_problem_ids()
- for pid in pids:
- names[pid], codes[pid], libraries[pid] = db.get_problem(pid)
-
- return names, codes, libraries
-
-# Submit code to Prolog server for testing.
-def test(name, code):
- # TODO also load fact library and solved predicates
- engine = PrologEngine(code=code)
- result = engine.ask("run_tests({}, '{}')".format(name, engine.id))
- engine.destroy()
- return result['event'] == 'success'
-
-if __name__ == '__main__':
- # Get problem id from commandline.
- if len(sys.argv) < 2:
- print('usage: ' + sys.argv[0] + ' <pid>')
- sys.exit(1)
- pid = int(sys.argv[1])
-
- names, codes, libraries = init_problems()
-
- # Analyze traces for this problem to get edits, submissions and queries.
- traces = db.get_traces(pid)
- edits, lines, submissions, queries = get_edits_from_traces(traces.values())
-
- # Find incorrect submissions.
- incorrect = []
- for submission, count in sorted(submissions.items()):
- if not test(names[pid], submission):
- # This incorrect submission appeared in [count] attempts.
- incorrect += [submission]*count
-
- # XXX only for testing
- try:
- done = pickle.load(open('status-'+str(pid)+'.pickle', 'rb'))
- except:
- done = []
-
- # test fix() on incorrect student submissions
- if len(sys.argv) >= 3 and sys.argv[2] == 'test':
- timeout = int(sys.argv[3]) if len(sys.argv) >= 4 else 10
-
- print('Fixing {}/{} programs (timeout={})…'.format(
- len([p for p in incorrect if p not in done]), len(incorrect), timeout))
-
- for i, program in enumerate(incorrect):
- if program in done:
- continue
- print(colored('Analyzing program {0}/{1}…'.format(i+1, len(incorrect)), 'yellow'))
- print(indent(compose(*decompose(program)), 2))
-
- solution, steps, fix_time, n_tested = fix(names[pid], program, edits, timeout=timeout)
- if solution:
- done.append(program)
- print_hint(solution, steps, fix_time, n_tested)
- print()
-
- pickle.dump(done, open('status-'+str(pid)+'.pickle', 'wb'))
-
- print('Found hints for ' + str(len(done)) + ' of ' + str(len(incorrect)) + ' incorrect programs')
-
- # print info for this problem
- elif len(sys.argv) >= 3 and sys.argv[2] == 'info':
- # with no additional arguments, print some stats
- if len(sys.argv) == 3:
- print('Problem {} ({}): {} edits in {} traces, fixed {}/{} ({}/{} unique)'.format(
- pid, colored(names[pid], 'yellow'),
- colored(str(len(edits)), 'yellow'), colored(str(len(traces)), 'yellow'),
- colored(str(len([p for p in incorrect if p in done])), 'yellow'),
- colored(str(len(incorrect)), 'yellow'),
- colored(str(len(set(done))), 'yellow'),
- colored(str(len(set(incorrect))), 'yellow')))
- else:
- if sys.argv[3] == 'users':
- print(' '.join([str(uid) for (pid, uid) in sorted(traces.keys())]))
- # print all observed edits and their costs
- elif sys.argv[3] == 'edits':
- inserts, removes, changes = classify_edits(edits)
- print('Inserts')
- for after, cost in sorted(inserts.items(), key=lambda x: x[1]):
- print(' {:.2f}\t{}'.format(cost, stringify(after)))
- print('Removes')
- for before, cost in sorted(removes.items(), key=lambda x: x[1]):
- print(' {:.2f}\t{}'.format(cost, stringify(before)))
- print('Changes')
- for (before, after), cost in sorted(changes.items(), key=lambda x: x[1]):
- print(' {:.2f}\t{} → {}'.format(cost,
- stringify(before if before else [('INVALID', 'ε')]),
- stringify(after if after else [('INVALID', 'ε')])))
- # print all student submissions not (yet) corrected
- elif sys.argv[3] == 'unsolved':
- for p in sorted(set(incorrect)):
- if p in done:
- continue
- print(indent(compose(*decompose(p)), 2))
- print()
- # print all student queries and their counts
- elif sys.argv[3] == 'queries':
- for query, count in queries.most_common():
- print(' ' + str(count) + '\t' + query)
-
- # Print the edit graph in graphviz dot syntax.
- elif len(sys.argv) == 4 and sys.argv[2] == 'graph':
- uid = int(sys.argv[3])
- actions = parse(traces[(pid, uid)])
-
- nodes, submissions, queries = edit_graph(actions)
-
- def position(node):
- return (node.data[1]*150, node.data[0]*-60)
-
- def label(node):
- return stringify(node.data[2])
-
- def node_attr(node):
- if node.ein and node.data[2] == node.ein[0].data[2]:
- return 'color="gray", shape="point"'
- return ''
-
- def edge_attr(a, b):
- if a.data[2] == b.data[2]:
- return 'arrowhead="none"'
- return ''
-
- graphviz_str = graphviz(nodes, pos=position, label=label,
- node_attr=node_attr, edge_attr=edge_attr)
- print(graphviz_str)
-
- # run interactive loop
- else:
- while True:
- # read the program from stdin
- print('Enter program, end with empty line:')
- code = ''
- try:
- while True:
- line = input()
- if not line:
- break
- code += line + '\n'
- except EOFError:
- break
-
- # try finding a fix
- print(colored('Analyzing program…', 'yellow'))
- solution, steps, fix_time, n_tested = fix(names[pid], code, edits, debug=True)
- print_hint(solution, steps, fix_time, n_tested)
diff --git a/monkey/test.py b/monkey/test.py
new file mode 100755
index 0000000..3143274
--- /dev/null
+++ b/monkey/test.py
@@ -0,0 +1,170 @@
+#!/usr/bin/python3
+
+import os
+import pickle
+import sys
+
+import django
+from termcolor import colored
+
+from .action import parse
+from .edits import classify_edits, edit_graph, get_edits_from_traces
+from .graph import graphviz
+from .monkey import fix
+from .prolog.engine import test
+from .prolog.util import compose, decompose, stringify
+from .util import indent
+
+# Load django models.
+os.environ['DJANGO_SETTINGS_MODULE'] = 'webmonkey.settings'
+django.setup()
+from django.contrib.auth.models import User
+from tutor.models import Attempt, Problem
+
+# Get problem id from commandline.
+if len(sys.argv) < 2:
+ print('usage: ' + sys.argv[0] + ' <pid>')
+ sys.exit(1)
+pid = int(sys.argv[1])
+
+# Analyze traces for this problem to get edits, submissions and queries.
+problem = Problem.objects.get(pk=pid)
+attempts = Attempt.objects.filter(problem=problem)
+
+traces = [a.trace for a in attempts]
+edits, lines, submissions, queries = get_edits_from_traces(traces)
+
+# Find incorrect submissions.
+incorrect = []
+for submission, count in sorted(submissions.items()):
+ if not test(problem.name, submission):
+ # This incorrect submission appeared in [count] attempts.
+ incorrect += [submission]*count
+
+# Load current status (programs for which a hint was found).
+try:
+ done = pickle.load(open('status-'+str(problem.pk)+'.pickle', 'rb'))
+except:
+ done = []
+
+def print_hint(solution, steps, fix_time, n_tested):
+ if solution:
+ print(colored('Hint found! Tested {} programs in {:.1f} s.'.format(n_tested, fix_time), 'green'))
+ print(colored(' Edits', 'blue'))
+ for line, (before, after) in steps:
+ print(' {}:\t{} → {}'.format(line, stringify(before), stringify(after)))
+ print(colored(' Final version', 'blue'))
+ print(indent(compose(*decompose(solution)), 2))
+ else:
+ print(colored('Hint not found! Tested {} programs in {:.1f} s.'.format(n_tested, fix_time), 'red'))
+
+# Test fix() on incorrect student submissions.
+if len(sys.argv) >= 3 and sys.argv[2] == 'test':
+ timeout = int(sys.argv[3]) if len(sys.argv) >= 4 else 10
+
+ print('Fixing {}/{} programs (timeout={})…'.format(
+ len([p for p in incorrect if p not in done]), len(incorrect), timeout))
+
+ for i, program in enumerate(incorrect):
+ if program in done:
+ continue
+ print(colored('Analyzing program {0}/{1}…'.format(i+1, len(incorrect)), 'yellow'))
+ print(indent(compose(*decompose(program)), 2))
+
+ solution, steps, fix_time, n_tested = fix(problem.name, program, edits, timeout=timeout)
+ if solution:
+ done.append(program)
+ print_hint(solution, steps, fix_time, n_tested)
+ print()
+
+ pickle.dump(done, open('status-'+str(problem.pk)+'.pickle', 'wb'))
+
+ print('Found hints for ' + str(len(done)) + ' of ' + str(len(incorrect)) + ' incorrect programs')
+
+# Print info for this problem.
+elif len(sys.argv) >= 3 and sys.argv[2] == 'info':
+ # With no additional arguments, print some stats.
+ if len(sys.argv) == 3:
+ print('Problem {} ({}): {} edits in {} traces, fixed {}/{} ({}/{} unique)'.format(
+ problem.pk, colored(problem.name, 'yellow'),
+ colored(str(len(edits)), 'yellow'), colored(str(len(traces)), 'yellow'),
+ colored(str(len([p for p in incorrect if p in done])), 'yellow'),
+ colored(str(len(incorrect)), 'yellow'),
+ colored(str(len(set(done))), 'yellow'),
+ colored(str(len(set(incorrect))), 'yellow')))
+ else:
+ if sys.argv[3] == 'users':
+ print(' '.join([str(a.user.pk) for a in attempts]))
+ # Print all observed edits and their costs.
+ elif sys.argv[3] == 'edits':
+ inserts, removes, changes = classify_edits(edits)
+ print('Inserts')
+ for after, cost in sorted(inserts.items(), key=lambda x: x[1]):
+ print(' {:.2f}\t{}'.format(cost, stringify(after)))
+ print('Removes')
+ for before, cost in sorted(removes.items(), key=lambda x: x[1]):
+ print(' {:.2f}\t{}'.format(cost, stringify(before)))
+ print('Changes')
+ for (before, after), cost in sorted(changes.items(), key=lambda x: x[1]):
+ print(' {:.2f}\t{} → {}'.format(cost,
+ stringify(before if before else [('INVALID', 'ε')]),
+ stringify(after if after else [('INVALID', 'ε')])))
+ # Print all student submissions not (yet) corrected.
+ elif sys.argv[3] == 'unsolved':
+ for p in sorted(set(incorrect)):
+ if p in done:
+ continue
+ print(indent(compose(*decompose(p)), 2))
+ print()
+ # Print all student queries and their counts.
+ elif sys.argv[3] == 'queries':
+ for query, count in queries.most_common():
+ print(' ' + str(count) + '\t' + query)
+
+# Print the edit graph in graphviz dot syntax.
+elif len(sys.argv) == 4 and sys.argv[2] == 'graph':
+ uid = int(sys.argv[3])
+ user = User.objects.get(pk=uid)
+ actions = parse(Attempt.objects.get(problem=problem, user=user).trace)
+
+ nodes, submissions, queries = edit_graph(actions)
+
+ def position(node):
+ return (node.data[1]*150, node.data[0]*-60)
+
+ def label(node):
+ return stringify(node.data[2])
+
+ def node_attr(node):
+ if node.ein and node.data[2] == node.ein[0].data[2]:
+ return 'color="gray", shape="point"'
+ return ''
+
+ def edge_attr(a, b):
+ if a.data[2] == b.data[2]:
+ return 'arrowhead="none"'
+ return ''
+
+ graphviz_str = graphviz(nodes, pos=position, label=label,
+ node_attr=node_attr, edge_attr=edge_attr)
+ print(graphviz_str)
+
+# Run interactive loop.
+else:
+ while True:
+ # Read the program from stdin.
+ print('Enter program, end with empty line:')
+ code = ''
+ try:
+ while True:
+ line = input()
+ if not line:
+ break
+ code += line + '\n'
+ except EOFError:
+ break
+
+ # Try finding a fix.
+ print(colored('Analyzing program…', 'yellow'))
+ solution, steps, fix_time, n_tested = fix(problem.name, code, edits, debug=True)
+ print_hint(solution, steps, fix_time, n_tested)