diff options
| -rwxr-xr-x | monkey/test.py | 25 |
1 files changed, 13 insertions, 12 deletions
diff --git a/monkey/test.py b/monkey/test.py index b5701a2..16d635f 100755 --- a/monkey/test.py +++ b/monkey/test.py @@ -12,7 +12,7 @@ from .edits import classify_edits, edit_graph, get_edits_from_traces from .graph import graphviz from .monkey import fix from prolog.engine import test -from prolog.util import compose, decompose, stringify +from prolog.util import Token, compose, decompose, stringify from .util import indent # Load django models. @@ -40,11 +40,12 @@ submissions = tutor_apps.get_app_config('tutor').submissions[problem.pk] queries = tutor_apps.get_app_config('tutor').queries[problem.pk] # Find incorrect submissions. -incorrect = [] +incorrect_all = [] for submission, count in sorted(submissions.items()): if not test(problem.name, submission + '\n' + aux_code): # This incorrect submission appeared in [count] attempts. - incorrect += [submission]*count + incorrect_all += [submission]*count +incorrect = set(incorrect_all) # Load current status (programs for which a hint was found). try: @@ -70,7 +71,7 @@ if len(sys.argv) >= 3 and sys.argv[2] == 'test': print('Fixing {}/{} programs (timeout={})…'.format( len([p for p in incorrect if p not in done]), len(incorrect), timeout)) - for i, program in enumerate(incorrect): + for i, program in enumerate(sorted(incorrect)): if program in done: continue print(colored('Analyzing program {0}/{1}…'.format(i+1, len(incorrect)), 'yellow')) @@ -93,8 +94,8 @@ elif len(sys.argv) >= 3 and sys.argv[2] == 'info': print('Problem {} ({}): {} edits in {} traces, fixed {}/{} ({}/{} unique)'.format( problem.pk, colored(problem.name, 'yellow'), colored(str(len(edits)), 'yellow'), colored(str(len([a.trace for a in attempts])), 'yellow'), - colored(str(len([p for p in incorrect if p in done])), 'yellow'), - colored(str(len(incorrect)), 'yellow'), + colored(str(len([p for p in incorrect_all if p in done])), 'yellow'), + colored(str(len(incorrect_all)), 'yellow'), colored(str(len(set(done))), 'yellow'), colored(str(len(set(incorrect))), 'yellow'))) else: @@ -105,18 +106,18 @@ elif len(sys.argv) >= 3 and sys.argv[2] == 'info': inserts, removes, changes = classify_edits(edits) print('Inserts') for after, cost in sorted(inserts.items(), key=lambda x: x[1]): - print(' {:.2f}\t{}'.format(cost, stringify(after))) + print(' {:.4f}\t{}'.format(cost, stringify(after))) print('Removes') for before, cost in sorted(removes.items(), key=lambda x: x[1]): - print(' {:.2f}\t{}'.format(cost, stringify(before))) + print(' {:.4f}\t{}'.format(cost, stringify(before))) print('Changes') for (before, after), cost in sorted(changes.items(), key=lambda x: x[1]): - print(' {:.2f}\t{} → {}'.format(cost, - stringify(before if before else [('INVALID', 'ε')]), - stringify(after if after else [('INVALID', 'ε')]))) + print(' {:.4f}\t{} → {}'.format(cost, + stringify(before if before else [Token('INVALID', 'ε')]), + stringify(after if after else [Token('INVALID', 'ε')]))) # Print all student submissions not (yet) corrected. elif sys.argv[3] == 'unsolved': - for p in sorted(set(incorrect)): + for p in sorted(incorrect): if p in done: continue print(indent(compose(*decompose(p)), 2)) |