summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rwxr-xr-xmonkey/test.py25
1 files changed, 13 insertions, 12 deletions
diff --git a/monkey/test.py b/monkey/test.py
index b5701a2..16d635f 100755
--- a/monkey/test.py
+++ b/monkey/test.py
@@ -12,7 +12,7 @@ from .edits import classify_edits, edit_graph, get_edits_from_traces
from .graph import graphviz
from .monkey import fix
from prolog.engine import test
-from prolog.util import compose, decompose, stringify
+from prolog.util import Token, compose, decompose, stringify
from .util import indent
# Load django models.
@@ -40,11 +40,12 @@ submissions = tutor_apps.get_app_config('tutor').submissions[problem.pk]
queries = tutor_apps.get_app_config('tutor').queries[problem.pk]
# Find incorrect submissions.
-incorrect = []
+incorrect_all = []
for submission, count in sorted(submissions.items()):
if not test(problem.name, submission + '\n' + aux_code):
# This incorrect submission appeared in [count] attempts.
- incorrect += [submission]*count
+ incorrect_all += [submission]*count
+incorrect = set(incorrect_all)
# Load current status (programs for which a hint was found).
try:
@@ -70,7 +71,7 @@ if len(sys.argv) >= 3 and sys.argv[2] == 'test':
print('Fixing {}/{} programs (timeout={})…'.format(
len([p for p in incorrect if p not in done]), len(incorrect), timeout))
- for i, program in enumerate(incorrect):
+ for i, program in enumerate(sorted(incorrect)):
if program in done:
continue
print(colored('Analyzing program {0}/{1}…'.format(i+1, len(incorrect)), 'yellow'))
@@ -93,8 +94,8 @@ elif len(sys.argv) >= 3 and sys.argv[2] == 'info':
print('Problem {} ({}): {} edits in {} traces, fixed {}/{} ({}/{} unique)'.format(
problem.pk, colored(problem.name, 'yellow'),
colored(str(len(edits)), 'yellow'), colored(str(len([a.trace for a in attempts])), 'yellow'),
- colored(str(len([p for p in incorrect if p in done])), 'yellow'),
- colored(str(len(incorrect)), 'yellow'),
+ colored(str(len([p for p in incorrect_all if p in done])), 'yellow'),
+ colored(str(len(incorrect_all)), 'yellow'),
colored(str(len(set(done))), 'yellow'),
colored(str(len(set(incorrect))), 'yellow')))
else:
@@ -105,18 +106,18 @@ elif len(sys.argv) >= 3 and sys.argv[2] == 'info':
inserts, removes, changes = classify_edits(edits)
print('Inserts')
for after, cost in sorted(inserts.items(), key=lambda x: x[1]):
- print(' {:.2f}\t{}'.format(cost, stringify(after)))
+ print(' {:.4f}\t{}'.format(cost, stringify(after)))
print('Removes')
for before, cost in sorted(removes.items(), key=lambda x: x[1]):
- print(' {:.2f}\t{}'.format(cost, stringify(before)))
+ print(' {:.4f}\t{}'.format(cost, stringify(before)))
print('Changes')
for (before, after), cost in sorted(changes.items(), key=lambda x: x[1]):
- print(' {:.2f}\t{} → {}'.format(cost,
- stringify(before if before else [('INVALID', 'ε')]),
- stringify(after if after else [('INVALID', 'ε')])))
+ print(' {:.4f}\t{} → {}'.format(cost,
+ stringify(before if before else [Token('INVALID', 'ε')]),
+ stringify(after if after else [Token('INVALID', 'ε')])))
# Print all student submissions not (yet) corrected.
elif sys.argv[3] == 'unsolved':
- for p in sorted(set(incorrect)):
+ for p in sorted(incorrect):
if p in done:
continue
print(indent(compose(*decompose(p)), 2))