Get number of passed/total tests from Prolog test functions

4 files changed, 15 insertions, 16 deletions

diff --git a/monkey/__init__.py b/monkey/__init__.py
index 83ea215..fbb4535 100644
--- a/monkey/__init__.py
+++ b/monkey/__init__.py
@@ -83,7 +83,8 @@ def fix(code, edits, test, timeout=30, debug=False):
                 print('{}: {} → {}'.format(idx, stringify(a), stringify(b)))
 
         # If the code is correct, we are done.
-        if test(code):
+        n_correct, n_all = test(code)
+        if n_correct == n_all:
             return code, path, total_time, n_tested
         n_tested += 1
 
diff --git a/monkey/edits.py b/monkey/edits.py
index ae44301..73f692d 100644
--- a/monkey/edits.py
+++ b/monkey/edits.py
@@ -234,13 +234,11 @@ if __name__ == '__main__':
             # Check for cached results.
             normal_code = stringify(rename_vars(tokenize(code)))
             code_key = (normal_code, tuple(dependencies))
-            if code_key in test_results[pid]:
-                return test_results[pid][code_key]
-
-            aux_code = '\n' + solutions_for_problems(problem.language, dependencies) + '\n' + facts
-            correct, hints = problem_module.test(code, aux_code)
-            test_results[pid][code_key] = correct
-            return correct
+            if code_key not in test_results[pid]:
+                aux_code = '\n' + solutions_for_problems(problem.language, dependencies) + '\n' + facts
+                n_correct, n_all, _ = problem_module.test(code, aux_code)
+                test_results[pid][code_key] = (n_correct, n_all)
+            return test_results[pid][code_key]
 
         print('Analyzing traces for {}… '.format(problem.identifier), end='', flush=True)
         print('{} traces… '.format(len(solutions)), end='', flush=True)
diff --git a/monkey/test.py b/monkey/test.py
index 1d82c33..bb49948 100755
--- a/monkey/test.py
+++ b/monkey/test.py
@@ -47,8 +47,8 @@ def test(code):
     dependencies = sorted([p[2:] for p in other_problems
                                  if p.identifier in used_predicate_identifiers])
     aux_code = '\n' + solutions_for_problems('prolog', dependencies) + '\n' + facts
-    correct, hints = problem_module.test(code, aux_code)
-    return correct
+    n_correct, n_all, _ = problem_module.test(code, aux_code)
+    return n_correct, n_all
 
 traces = [s.trace for s in Solution.filter(problem_id=problem.id)]
 
diff --git a/server/prolog_session.py b/server/prolog_session.py
index 8a5b6e3..121f802 100644
--- a/server/prolog_session.py
+++ b/server/prolog_session.py
@@ -122,8 +122,8 @@ class PrologSession(server.LanguageSession):
         hints = []
         try:
             # check if the program is already correct
-            passed, _ = problem_module.test(program, aux_code=aux_code)
-            if passed:
+            n_correct, n_all, _ = problem_module.test(program, aux_code=aux_code)
+            if n_correct == n_all:
                 hints = [{'id': 'program_already_correct'}]
 
             if not hints and hasattr(language_module, 'hint'):
@@ -133,8 +133,8 @@ class PrologSession(server.LanguageSession):
             if not hints and problem_id in _edits:
                 # Testing function for monkey.
                 def tester(code):
-                    passed, hints = problem_module.test(code, aux_code=aux_code)
-                    return passed
+                    n_correct, n_all, _ = problem_module.test(code, aux_code=aux_code)
+                    return n_correct, n_all
                 solution, steps, fix_time, n_tested = monkey.fix(
                         program, _edits[problem_id], tester, timeout=5, debug=True)
                 if solution and steps:
@@ -156,8 +156,8 @@ class PrologSession(server.LanguageSession):
         aux_code = self._aux_code(session.get_uid(), problem, program)
 
         try:
-            passed, hints = problem_module.test(program, aux_code=aux_code)
-            if passed:
+            n_correct, n_all, hints = problem_module.test(program, aux_code=aux_code)
+            if n_correct == n_all:
                 session.update_solution(problem_id, done=True)
         except AttributeError as ex:
             hints = [{'id': 'system_error', 'args': {'message': 'test function does not exist'}}]