summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rwxr-xr-xmonkey/test.py85
1 files changed, 38 insertions, 47 deletions
diff --git a/monkey/test.py b/monkey/test.py
index dfe20bc..38f37a0 100755
--- a/monkey/test.py
+++ b/monkey/test.py
@@ -4,47 +4,42 @@ import os
import pickle
import sys
-import django
from termcolor import colored
+from db.models import CodeqUser, Problem, Solution
from .edits import classify_edits, trace_graph
from .graph import graphviz
-from .monkey import fix, fix_hints, test
+from .monkey import fix, fix_hints
from prolog.util import annotate, compose, stringify
+import server.problems
from .util import indent
-# Load django models.
-os.environ['DJANGO_SETTINGS_MODULE'] = 'webmonkey.settings'
-django.setup()
-from django.apps import apps as tutor_apps
-from django.contrib.auth.models import User
-from tutor.models import Attempt, Problem, get_aux_code
-
# Get problem id from commandline.
if len(sys.argv) < 2:
print('usage: ' + sys.argv[0] + ' <pid>')
sys.exit(1)
pid = int(sys.argv[1])
-problem = Problem.objects.get(pk=pid)
-aux_code = get_aux_code(user=User.objects.get(pk=1), problem=problem)
+name = Problem.get(id=pid).name
+language, problem_group, problem = Problem.get_identifier(pid)
+
+solved_problems = [p for p in CodeqUser.solved_problems(1, language)
+ if p != (problem_group, problem)]
+other_solutions = server.problems.solutions_for_problems(language, solved_problems)
+problem_module = server.problems.load_problem(language, problem_group, problem, 'common')
+problem_facts = server.problems.get_facts(language, problem_module) or ''
+aux_code = other_solutions + '\n' + problem_facts
+
+# Testing function.
+def test(code):
+ n_correct, n_all = problem_module.test(code + '\n' + aux_code)
+ return n_correct == n_all
-attempts = Attempt.objects.filter(problem=problem) \
- .exclude(user__groups=None)
+traces = [s.trace for s in Solution.filter(problem_id=pid)]
# Load hint database stored in edits.pickle.
-edits = tutor_apps.get_app_config('tutor').edits[problem.pk]
-submissions = tutor_apps.get_app_config('tutor').submissions[problem.pk]
-queries = tutor_apps.get_app_config('tutor').queries[problem.pk]
-names = tutor_apps.get_app_config('tutor').names[problem.pk]
-
-# Find incorrect submissions.
-incorrect_all = []
-for submission, count in sorted(submissions.items()):
- if not test(problem.name, submission + '\n' + aux_code):
- # This incorrect submission appeared in [count] attempts.
- incorrect_all += [submission]*count
-incorrect = set(incorrect_all)
+edits, submissions, queries, names = pickle.load(open('edits.pickle', 'rb'))
+edits, submissions, queries, names = edits[pid], submissions[pid], queries[pid], names[pid]
# Load current status (programs for which a hint was found).
try:
@@ -83,13 +78,21 @@ if len(sys.argv) == 2:
# Try finding a fix.
print(colored('Analyzing program…', 'yellow'))
- solution, steps, fix_time, n_tested = fix(problem.name, code, edits, aux_code=aux_code, debug=True)
+ solution, steps, fix_time, n_tested = fix(name, code, edits, test, debug=True)
print_hint(code, solution, steps, fix_time, n_tested)
# Test fix() on incorrect student submissions.
elif sys.argv[2] == 'test':
timeout = int(sys.argv[3]) if len(sys.argv) == 4 else 10
+ # Find incorrect submissions.
+ incorrect_all = []
+ for submission, count in sorted(submissions.items()):
+ if not test(submission):
+ # This incorrect submission appeared in [count] traces.
+ incorrect_all += [submission]*count
+ incorrect = set(incorrect_all)
+
print('Fixing {}/{} programs (timeout={})…'.format(
len([p for p in incorrect if p not in done]), len(incorrect), timeout))
@@ -99,7 +102,7 @@ elif sys.argv[2] == 'test':
print(colored('Analyzing program {0}/{1}…'.format(i+1, len(incorrect)), 'yellow'))
print(indent(compose(annotate(program)), 2))
- solution, steps, fix_time, n_tested = fix(problem.name, program, edits, aux_code=aux_code, timeout=timeout)
+ solution, steps, fix_time, n_tested = fix(name, program, edits, test, timeout=timeout)
if solution:
done.append(program)
print_hint(program, solution, steps, fix_time, n_tested)
@@ -113,16 +116,12 @@ elif sys.argv[2] == 'test':
elif sys.argv[2] == 'info':
# With no additional arguments, print some stats.
if len(sys.argv) == 3:
- print('Problem {} ({}): {} edits in {} traces, fixed {}/{} ({}/{} unique)'.format(
- problem.pk, colored(problem.name, 'yellow'),
- colored(str(len(edits)), 'yellow'), colored(str(len([a.trace for a in attempts])), 'yellow'),
- colored(str(len([p for p in incorrect_all if p in done])), 'yellow'),
- colored(str(len(incorrect_all)), 'yellow'),
- colored(str(len(set(done))), 'yellow'),
- colored(str(len(set(incorrect))), 'yellow')))
-
- elif sys.argv[3] == 'users':
- print(' '.join([str(a.user.pk) for a in attempts]))
+ print('Problem {} ({}): {} edits and {} unique submissions in {} traces'.format(
+ pid, colored(name, 'yellow'),
+ colored(str(len(edits)), 'yellow'),
+ colored(str(len(submissions)), 'yellow'),
+ colored(str(len(traces)), 'yellow')))
+
# Print all observed edits and their costs.
elif sys.argv[3] == 'edits':
inserts, removes, changes = classify_edits(edits)
@@ -141,13 +140,6 @@ elif sys.argv[2] == 'info':
for name, count in sorted(names.items(), key=lambda x: x[1]):
print(' {:.4f}\t{}'.format(count, name))
- # Print all student submissions not (yet) corrected.
- elif sys.argv[3] == 'unsolved':
- for p in sorted(incorrect):
- if p in done:
- continue
- print(indent(compose(annotate(p)), 2))
- print()
# Print all student queries and their counts.
elif sys.argv[3] == 'queries':
for query, count in queries.most_common():
@@ -156,10 +148,9 @@ elif sys.argv[2] == 'info':
# Print the edit graph in graphviz dot syntax.
elif sys.argv[2] == 'graph' and len(sys.argv) == 4:
uid = int(sys.argv[3])
- user = User.objects.get(pk=uid)
- attempt = Attempt.objects.get(problem=problem, user=user)
+ solution = Solution.get(problem_id=pid, codeq_user_id=uid)
- nodes, submissions, queries = trace_graph(attempt.trace)
+ nodes, submissions, queries = trace_graph(solution.trace)
def position(node):
return (node.data[1]*150, node.data[0]*-60)