diff options
| -rwxr-xr-x | monkey/test.py | 85 |
1 files changed, 38 insertions, 47 deletions
diff --git a/monkey/test.py b/monkey/test.py index dfe20bc..38f37a0 100755 --- a/monkey/test.py +++ b/monkey/test.py @@ -4,47 +4,42 @@ import os import pickle import sys -import django from termcolor import colored +from db.models import CodeqUser, Problem, Solution from .edits import classify_edits, trace_graph from .graph import graphviz -from .monkey import fix, fix_hints, test +from .monkey import fix, fix_hints from prolog.util import annotate, compose, stringify +import server.problems from .util import indent -# Load django models. -os.environ['DJANGO_SETTINGS_MODULE'] = 'webmonkey.settings' -django.setup() -from django.apps import apps as tutor_apps -from django.contrib.auth.models import User -from tutor.models import Attempt, Problem, get_aux_code - # Get problem id from commandline. if len(sys.argv) < 2: print('usage: ' + sys.argv[0] + ' <pid>') sys.exit(1) pid = int(sys.argv[1]) -problem = Problem.objects.get(pk=pid) -aux_code = get_aux_code(user=User.objects.get(pk=1), problem=problem) +name = Problem.get(id=pid).name +language, problem_group, problem = Problem.get_identifier(pid) + +solved_problems = [p for p in CodeqUser.solved_problems(1, language) + if p != (problem_group, problem)] +other_solutions = server.problems.solutions_for_problems(language, solved_problems) +problem_module = server.problems.load_problem(language, problem_group, problem, 'common') +problem_facts = server.problems.get_facts(language, problem_module) or '' +aux_code = other_solutions + '\n' + problem_facts + +# Testing function. +def test(code): + n_correct, n_all = problem_module.test(code + '\n' + aux_code) + return n_correct == n_all -attempts = Attempt.objects.filter(problem=problem) \ - .exclude(user__groups=None) +traces = [s.trace for s in Solution.filter(problem_id=pid)] # Load hint database stored in edits.pickle. -edits = tutor_apps.get_app_config('tutor').edits[problem.pk] -submissions = tutor_apps.get_app_config('tutor').submissions[problem.pk] -queries = tutor_apps.get_app_config('tutor').queries[problem.pk] -names = tutor_apps.get_app_config('tutor').names[problem.pk] - -# Find incorrect submissions. -incorrect_all = [] -for submission, count in sorted(submissions.items()): - if not test(problem.name, submission + '\n' + aux_code): - # This incorrect submission appeared in [count] attempts. - incorrect_all += [submission]*count -incorrect = set(incorrect_all) +edits, submissions, queries, names = pickle.load(open('edits.pickle', 'rb')) +edits, submissions, queries, names = edits[pid], submissions[pid], queries[pid], names[pid] # Load current status (programs for which a hint was found). try: @@ -83,13 +78,21 @@ if len(sys.argv) == 2: # Try finding a fix. print(colored('Analyzing program…', 'yellow')) - solution, steps, fix_time, n_tested = fix(problem.name, code, edits, aux_code=aux_code, debug=True) + solution, steps, fix_time, n_tested = fix(name, code, edits, test, debug=True) print_hint(code, solution, steps, fix_time, n_tested) # Test fix() on incorrect student submissions. elif sys.argv[2] == 'test': timeout = int(sys.argv[3]) if len(sys.argv) == 4 else 10 + # Find incorrect submissions. + incorrect_all = [] + for submission, count in sorted(submissions.items()): + if not test(submission): + # This incorrect submission appeared in [count] traces. + incorrect_all += [submission]*count + incorrect = set(incorrect_all) + print('Fixing {}/{} programs (timeout={})…'.format( len([p for p in incorrect if p not in done]), len(incorrect), timeout)) @@ -99,7 +102,7 @@ elif sys.argv[2] == 'test': print(colored('Analyzing program {0}/{1}…'.format(i+1, len(incorrect)), 'yellow')) print(indent(compose(annotate(program)), 2)) - solution, steps, fix_time, n_tested = fix(problem.name, program, edits, aux_code=aux_code, timeout=timeout) + solution, steps, fix_time, n_tested = fix(name, program, edits, test, timeout=timeout) if solution: done.append(program) print_hint(program, solution, steps, fix_time, n_tested) @@ -113,16 +116,12 @@ elif sys.argv[2] == 'test': elif sys.argv[2] == 'info': # With no additional arguments, print some stats. if len(sys.argv) == 3: - print('Problem {} ({}): {} edits in {} traces, fixed {}/{} ({}/{} unique)'.format( - problem.pk, colored(problem.name, 'yellow'), - colored(str(len(edits)), 'yellow'), colored(str(len([a.trace for a in attempts])), 'yellow'), - colored(str(len([p for p in incorrect_all if p in done])), 'yellow'), - colored(str(len(incorrect_all)), 'yellow'), - colored(str(len(set(done))), 'yellow'), - colored(str(len(set(incorrect))), 'yellow'))) - - elif sys.argv[3] == 'users': - print(' '.join([str(a.user.pk) for a in attempts])) + print('Problem {} ({}): {} edits and {} unique submissions in {} traces'.format( + pid, colored(name, 'yellow'), + colored(str(len(edits)), 'yellow'), + colored(str(len(submissions)), 'yellow'), + colored(str(len(traces)), 'yellow'))) + # Print all observed edits and their costs. elif sys.argv[3] == 'edits': inserts, removes, changes = classify_edits(edits) @@ -141,13 +140,6 @@ elif sys.argv[2] == 'info': for name, count in sorted(names.items(), key=lambda x: x[1]): print(' {:.4f}\t{}'.format(count, name)) - # Print all student submissions not (yet) corrected. - elif sys.argv[3] == 'unsolved': - for p in sorted(incorrect): - if p in done: - continue - print(indent(compose(annotate(p)), 2)) - print() # Print all student queries and their counts. elif sys.argv[3] == 'queries': for query, count in queries.most_common(): @@ -156,10 +148,9 @@ elif sys.argv[2] == 'info': # Print the edit graph in graphviz dot syntax. elif sys.argv[2] == 'graph' and len(sys.argv) == 4: uid = int(sys.argv[3]) - user = User.objects.get(pk=uid) - attempt = Attempt.objects.get(problem=problem, user=user) + solution = Solution.get(problem_id=pid, codeq_user_id=uid) - nodes, submissions, queries = trace_graph(attempt.trace) + nodes, submissions, queries = trace_graph(solution.trace) def position(node): return (node.data[1]*150, node.data[0]*-60) |