Update monkey.test to work with new DB

1 files changed, 38 insertions, 47 deletions

diff --git a/monkey/test.py b/monkey/test.py
index dfe20bc..38f37a0 100755
--- a/monkey/test.py
+++ b/monkey/test.py
@@ -4,47 +4,42 @@ import os
 import pickle
 import sys
 
-import django
 from termcolor import colored
 
+from db.models import CodeqUser, Problem, Solution
 from .edits import classify_edits, trace_graph
 from .graph import graphviz
-from .monkey import fix, fix_hints, test
+from .monkey import fix, fix_hints
 from prolog.util import annotate, compose, stringify
+import server.problems
 from .util import indent
 
-# Load django models.
-os.environ['DJANGO_SETTINGS_MODULE'] = 'webmonkey.settings'
-django.setup()
-from django.apps import apps as tutor_apps
-from django.contrib.auth.models import User
-from tutor.models import Attempt, Problem, get_aux_code
-
 # Get problem id from commandline.
 if len(sys.argv) < 2:
     print('usage: ' + sys.argv[0] + ' <pid>')
     sys.exit(1)
 pid = int(sys.argv[1])
 
-problem = Problem.objects.get(pk=pid)
-aux_code = get_aux_code(user=User.objects.get(pk=1), problem=problem)
+name = Problem.get(id=pid).name
+language, problem_group, problem = Problem.get_identifier(pid)
+
+solved_problems = [p for p in CodeqUser.solved_problems(1, language)
+                           if p != (problem_group, problem)]
+other_solutions = server.problems.solutions_for_problems(language, solved_problems)
+problem_module = server.problems.load_problem(language, problem_group, problem, 'common')
+problem_facts = server.problems.get_facts(language, problem_module) or ''
+aux_code = other_solutions + '\n' + problem_facts
+
+# Testing function.
+def test(code):
+    n_correct, n_all = problem_module.test(code + '\n' + aux_code)
+    return n_correct == n_all
 
-attempts = Attempt.objects.filter(problem=problem) \
-                          .exclude(user__groups=None)
+traces = [s.trace for s in Solution.filter(problem_id=pid)]
 
 # Load hint database stored in edits.pickle.
-edits = tutor_apps.get_app_config('tutor').edits[problem.pk]
-submissions = tutor_apps.get_app_config('tutor').submissions[problem.pk]
-queries = tutor_apps.get_app_config('tutor').queries[problem.pk]
-names = tutor_apps.get_app_config('tutor').names[problem.pk]
-
-# Find incorrect submissions.
-incorrect_all = []
-for submission, count in sorted(submissions.items()):
-    if not test(problem.name, submission + '\n' + aux_code):
-        # This incorrect submission appeared in [count] attempts.
-        incorrect_all += [submission]*count
-incorrect = set(incorrect_all)
+edits, submissions, queries, names = pickle.load(open('edits.pickle', 'rb'))
+edits, submissions, queries, names = edits[pid], submissions[pid], queries[pid], names[pid]
 
 # Load current status (programs for which a hint was found).
 try:
@@ -83,13 +78,21 @@ if len(sys.argv) == 2:
 
         # Try finding a fix.
         print(colored('Analyzing program…', 'yellow'))
-        solution, steps, fix_time, n_tested = fix(problem.name, code, edits, aux_code=aux_code, debug=True)
+        solution, steps, fix_time, n_tested = fix(name, code, edits, test, debug=True)
         print_hint(code, solution, steps, fix_time, n_tested)
 
 # Test fix() on incorrect student submissions.
 elif sys.argv[2] == 'test':
     timeout = int(sys.argv[3]) if len(sys.argv) == 4 else 10
 
+    # Find incorrect submissions.
+    incorrect_all = []
+    for submission, count in sorted(submissions.items()):
+        if not test(submission):
+            # This incorrect submission appeared in [count] traces.
+            incorrect_all += [submission]*count
+    incorrect = set(incorrect_all)
+
     print('Fixing {}/{} programs (timeout={})…'.format(
         len([p for p in incorrect if p not in done]), len(incorrect), timeout))
 
@@ -99,7 +102,7 @@ elif sys.argv[2] == 'test':
         print(colored('Analyzing program {0}/{1}…'.format(i+1, len(incorrect)), 'yellow'))
         print(indent(compose(annotate(program)), 2))
 
-        solution, steps, fix_time, n_tested = fix(problem.name, program, edits, aux_code=aux_code, timeout=timeout)
+        solution, steps, fix_time, n_tested = fix(name, program, edits, test, timeout=timeout)
         if solution:
             done.append(program)
         print_hint(program, solution, steps, fix_time, n_tested)
@@ -113,16 +116,12 @@ elif sys.argv[2] == 'test':
 elif sys.argv[2] == 'info':
     # With no additional arguments, print some stats.
     if len(sys.argv) == 3:
-        print('Problem {} ({}): {} edits in {} traces, fixed {}/{} ({}/{} unique)'.format(
-            problem.pk, colored(problem.name, 'yellow'),
-            colored(str(len(edits)), 'yellow'), colored(str(len([a.trace for a in attempts])), 'yellow'),
-            colored(str(len([p for p in incorrect_all if p in done])), 'yellow'),
-            colored(str(len(incorrect_all)), 'yellow'),
-            colored(str(len(set(done))), 'yellow'),
-            colored(str(len(set(incorrect))), 'yellow')))
-
-    elif sys.argv[3] == 'users':
-        print(' '.join([str(a.user.pk) for a in attempts]))
+        print('Problem {} ({}): {} edits and {} unique submissions in {} traces'.format(
+            pid, colored(name, 'yellow'),
+            colored(str(len(edits)), 'yellow'),
+            colored(str(len(submissions)), 'yellow'),
+            colored(str(len(traces)), 'yellow')))
+
     # Print all observed edits and their costs.
     elif sys.argv[3] == 'edits':
         inserts, removes, changes = classify_edits(edits)
@@ -141,13 +140,6 @@ elif sys.argv[2] == 'info':
         for name, count in sorted(names.items(), key=lambda x: x[1]):
             print(' {:.4f}\t{}'.format(count,  name))
 
-    # Print all student submissions not (yet) corrected.
-    elif sys.argv[3] == 'unsolved':
-        for p in sorted(incorrect):
-            if p in done:
-                continue
-            print(indent(compose(annotate(p)), 2))
-            print()
     # Print all student queries and their counts.
     elif sys.argv[3] == 'queries':
         for query, count in queries.most_common():
@@ -156,10 +148,9 @@ elif sys.argv[2] == 'info':
 # Print the edit graph in graphviz dot syntax.
 elif sys.argv[2] == 'graph' and len(sys.argv) == 4:
     uid = int(sys.argv[3])
-    user = User.objects.get(pk=uid)
-    attempt = Attempt.objects.get(problem=problem, user=user)
+    solution = Solution.get(problem_id=pid, codeq_user_id=uid)
 
-    nodes, submissions, queries = trace_graph(attempt.trace)
+    nodes, submissions, queries = trace_graph(solution.trace)
 
     def position(node):
         return (node.data[1]*150, node.data[0]*-60)