Fix test functions for Prolog problems

Old version would return wrong values for programs with syntax errors.

12 files changed, 100 insertions, 120 deletions

diff --git a/prolog/problems/family_relations/ancestor_2/common.py b/prolog/problems/family_relations/ancestor_2/common.py
index 94b7e68..150e2f0 100644
--- a/prolog/problems/family_relations/ancestor_2/common.py
+++ b/prolog/problems/family_relations/ancestor_2/common.py
@@ -45,19 +45,17 @@ def test(program, solved_problems):
             server.problems.solutions_for_problems('prolog', solved_problems) + '\n' +
             server.problems.load_facts('prolog', facts).facts)
 
+    n_correct = 0
     engine_id = None
     try:
         engine_id, output = prolog.engine.create(code=code, timeout=1.0)
-        if not engine_id or 'error' in map(itemgetter(0), output):
-            # Engine creation failed, or syntax error in code.
-            return 0, len(test_cases)
-
-        n_correct = 0
-        for query, answers in test_cases:
-            # Limit inferences for each solution to curb unbounded recursion.
-            limited = 'call_with_inference_limit(({}), 100000, _)'.format(query)
-            if prolog.engine.check_answers(engine_id, query=limited, answers=answers, timeout=1.0):
-                n_correct += 1
+        if engine_id is not None and 'error' not in map(itemgetter(0), output):
+            # Engine successfully created, and no syntax error in program.
+            for query, answers in test_cases:
+                # Limit inferences for each solution to curb unbounded recursion.
+                limited = 'call_with_inference_limit(({}), 100000, _)'.format(query)
+                if prolog.engine.check_answers(engine_id, query=limited, answers=answers, timeout=1.0):
+                    n_correct += 1
     finally:
         if engine_id:
             prolog.engine.destroy(engine_id)
diff --git a/prolog/problems/family_relations/aunt_2/common.py b/prolog/problems/family_relations/aunt_2/common.py
index 4f5ea26..9ac91ea 100644
--- a/prolog/problems/family_relations/aunt_2/common.py
+++ b/prolog/problems/family_relations/aunt_2/common.py
@@ -49,19 +49,17 @@ def test(program, solved_problems):
             server.problems.solutions_for_problems('prolog', solved_problems) + '\n' +
             server.problems.load_facts('prolog', facts).facts)
 
+    n_correct = 0
     engine_id = None
     try:
         engine_id, output = prolog.engine.create(code=code, timeout=1.0)
-        if not engine_id or 'error' in map(itemgetter(0), output):
-            # Engine creation failed, or syntax error in code.
-            return 0, len(test_cases)
-
-        n_correct = 0
-        for query, answers in test_cases:
-            # Limit inferences for each solution to curb unbounded recursion.
-            limited = 'call_with_inference_limit(({}), 100000, _)'.format(query)
-            if prolog.engine.check_answers(engine_id, query=limited, answers=answers, timeout=1.0):
-                n_correct += 1
+        if engine_id is not None and 'error' not in map(itemgetter(0), output):
+            # Engine successfully created, and no syntax error in program.
+            for query, answers in test_cases:
+                # Limit inferences for each solution to curb unbounded recursion.
+                limited = 'call_with_inference_limit(({}), 100000, _)'.format(query)
+                if prolog.engine.check_answers(engine_id, query=limited, answers=answers, timeout=1.0):
+                    n_correct += 1
     finally:
         if engine_id:
             prolog.engine.destroy(engine_id)
diff --git a/prolog/problems/family_relations/brother_2/common.py b/prolog/problems/family_relations/brother_2/common.py
index 8f2b73e..a370baa 100644
--- a/prolog/problems/family_relations/brother_2/common.py
+++ b/prolog/problems/family_relations/brother_2/common.py
@@ -43,19 +43,17 @@ def test(program, solved_problems):
             server.problems.solutions_for_problems('prolog', solved_problems) + '\n' +
             server.problems.load_facts('prolog', facts).facts)
 
+    n_correct = 0
     engine_id = None
     try:
         engine_id, output = prolog.engine.create(code=code, timeout=1.0)
-        if not engine_id or 'error' in map(itemgetter(0), output):
-            # Engine creation failed, or syntax error in code.
-            return 0, len(test_cases)
-
-        n_correct = 0
-        for query, answers in test_cases:
-            # Limit inferences for each solution to curb unbounded recursion.
-            limited = 'call_with_inference_limit(({}), 100000, _)'.format(query)
-            if prolog.engine.check_answers(engine_id, query=limited, answers=answers, timeout=1.0):
-                n_correct += 1
+        if engine_id is not None and 'error' not in map(itemgetter(0), output):
+            # Engine successfully created, and no syntax error in program.
+            for query, answers in test_cases:
+                # Limit inferences for each solution to curb unbounded recursion.
+                limited = 'call_with_inference_limit(({}), 100000, _)'.format(query)
+                if prolog.engine.check_answers(engine_id, query=limited, answers=answers, timeout=1.0):
+                    n_correct += 1
     finally:
         if engine_id:
             prolog.engine.destroy(engine_id)
diff --git a/prolog/problems/family_relations/connected_3/common.py b/prolog/problems/family_relations/connected_3/common.py
index e6aa6dc..8b462f5 100644
--- a/prolog/problems/family_relations/connected_3/common.py
+++ b/prolog/problems/family_relations/connected_3/common.py
@@ -1,5 +1,9 @@
 # coding=utf-8
 
+from operator import itemgetter
+import prolog.engine
+import server.problems
+
 id = 102
 number = 9
 visible = True
@@ -30,19 +34,17 @@ def test(program, solved_problems):
             server.problems.solutions_for_problems('prolog', solved_problems) + '\n' +
             server.problems.load_facts('prolog', facts).facts)
 
+    n_correct = 0
     engine_id = None
     try:
         engine_id, output = prolog.engine.create(code=code, timeout=1.0)
-        if not engine_id or 'error' in map(itemgetter(0), output):
-            # Engine creation failed, or syntax error in code.
-            return 0, len(test_cases)
-
-        n_correct = 0
-        for query, answers in test_cases:
-            # Limit inferences for each solution to curb unbounded recursion.
-            limited = 'call_with_inference_limit(({}), 100000, _)'.format(query)
-            if prolog.engine.check_answers(engine_id, query=limited, answers=answers, timeout=1.0):
-                n_correct += 1
+        if engine_id is not None and 'error' not in map(itemgetter(0), output):
+            # Engine successfully created, and no syntax error in program.
+            for query, answers in test_cases:
+                # Limit inferences for each solution to curb unbounded recursion.
+                limited = 'call_with_inference_limit(({}), 100000, _)'.format(query)
+                if prolog.engine.check_answers(engine_id, query=limited, answers=answers, timeout=1.0):
+                    n_correct += 1
     finally:
         if engine_id:
             prolog.engine.destroy(engine_id)
diff --git a/prolog/problems/family_relations/cousin_2/common.py b/prolog/problems/family_relations/cousin_2/common.py
index c556ffd..16a639c 100644
--- a/prolog/problems/family_relations/cousin_2/common.py
+++ b/prolog/problems/family_relations/cousin_2/common.py
@@ -56,19 +56,17 @@ def test(program, solved_problems):
             server.problems.solutions_for_problems('prolog', solved_problems) + '\n' +
             server.problems.load_facts('prolog', facts).facts)
 
+    n_correct = 0
     engine_id = None
     try:
         engine_id, output = prolog.engine.create(code=code, timeout=1.0)
-        if not engine_id or 'error' in map(itemgetter(0), output):
-            # Engine creation failed, or syntax error in code.
-            return 0, len(test_cases)
-
-        n_correct = 0
-        for query, answers in test_cases:
-            # Limit inferences for each solution to curb unbounded recursion.
-            limited = 'call_with_inference_limit(({}), 100000, _)'.format(query)
-            if prolog.engine.check_answers(engine_id, query=limited, answers=answers, timeout=1.0):
-                n_correct += 1
+        if engine_id is not None and 'error' not in map(itemgetter(0), output):
+            # Engine successfully created, and no syntax error in program.
+            for query, answers in test_cases:
+                # Limit inferences for each solution to curb unbounded recursion.
+                limited = 'call_with_inference_limit(({}), 100000, _)'.format(query)
+                if prolog.engine.check_answers(engine_id, query=limited, answers=answers, timeout=1.0):
+                    n_correct += 1
     finally:
         if engine_id:
             prolog.engine.destroy(engine_id)
diff --git a/prolog/problems/family_relations/descendant_2/common.py b/prolog/problems/family_relations/descendant_2/common.py
index 49e7e5c..44eb605 100644
--- a/prolog/problems/family_relations/descendant_2/common.py
+++ b/prolog/problems/family_relations/descendant_2/common.py
@@ -45,19 +45,17 @@ def test(program, solved_problems):
             server.problems.solutions_for_problems('prolog', solved_problems) + '\n' +
             server.problems.load_facts('prolog', facts).facts)
 
+    n_correct = 0
     engine_id = None
     try:
         engine_id, output = prolog.engine.create(code=code, timeout=1.0)
-        if not engine_id or 'error' in map(itemgetter(0), output):
-            # Engine creation failed, or syntax error in code.
-            return 0, len(test_cases)
-
-        n_correct = 0
-        for query, answers in test_cases:
-            # Limit inferences for each solution to curb unbounded recursion.
-            limited = 'call_with_inference_limit(({}), 100000, _)'.format(query)
-            if prolog.engine.check_answers(engine_id, query=limited, answers=answers, timeout=1.0):
-                n_correct += 1
+        if engine_id is not None and 'error' not in map(itemgetter(0), output):
+            # Engine successfully created, and no syntax error in program.
+            for query, answers in test_cases:
+                # Limit inferences for each solution to curb unbounded recursion.
+                limited = 'call_with_inference_limit(({}), 100000, _)'.format(query)
+                if prolog.engine.check_answers(engine_id, query=limited, answers=answers, timeout=1.0):
+                    n_correct += 1
     finally:
         if engine_id:
             prolog.engine.destroy(engine_id)
diff --git a/prolog/problems/family_relations/father_2/common.py b/prolog/problems/family_relations/father_2/common.py
index ef94bc4..400c91b 100644
--- a/prolog/problems/family_relations/father_2/common.py
+++ b/prolog/problems/family_relations/father_2/common.py
@@ -46,19 +46,17 @@ def test(program, solved_problems):
             server.problems.solutions_for_problems('prolog', solved_problems) + '\n' +
             server.problems.load_facts('prolog', facts).facts)
 
+    n_correct = 0
     engine_id = None
     try:
         engine_id, output = prolog.engine.create(code=code, timeout=1.0)
-        if not engine_id or 'error' in map(itemgetter(0), output):
-            # Engine creation failed, or syntax error in code.
-            return 0, len(test_cases)
-
-        n_correct = 0
-        for query, answers in test_cases:
-            # Limit inferences for each solution to curb unbounded recursion.
-            limited = 'call_with_inference_limit(({}), 100000, _)'.format(query)
-            if prolog.engine.check_answers(engine_id, query=limited, answers=answers, timeout=1.0):
-                n_correct += 1
+        if engine_id is not None and 'error' not in map(itemgetter(0), output):
+            # Engine successfully created, and no syntax error in program.
+            for query, answers in test_cases:
+                # Limit inferences for each solution to curb unbounded recursion.
+                limited = 'call_with_inference_limit(({}), 100000, _)'.format(query)
+                if prolog.engine.check_answers(engine_id, query=limited, answers=answers, timeout=1.0):
+                    n_correct += 1
     finally:
         if engine_id:
             prolog.engine.destroy(engine_id)
diff --git a/prolog/problems/family_relations/grandparent_2/common.py b/prolog/problems/family_relations/grandparent_2/common.py
index f7b81b6..1657509 100644
--- a/prolog/problems/family_relations/grandparent_2/common.py
+++ b/prolog/problems/family_relations/grandparent_2/common.py
@@ -47,19 +47,17 @@ def test(program, solved_problems):
             server.problems.solutions_for_problems('prolog', solved_problems) + '\n' +
             server.problems.load_facts('prolog', facts).facts)
 
+    n_correct = 0
     engine_id = None
     try:
         engine_id, output = prolog.engine.create(code=code, timeout=1.0)
-        if not engine_id or 'error' in map(itemgetter(0), output):
-            # Engine creation failed, or syntax error in code.
-            return 0, len(test_cases)
-
-        n_correct = 0
-        for query, answers in test_cases:
-            # Limit inferences for each solution to curb unbounded recursion.
-            limited = 'call_with_inference_limit(({}), 100000, _)'.format(query)
-            if prolog.engine.check_answers(engine_id, query=limited, answers=answers, timeout=1.0):
-                n_correct += 1
+        if engine_id is not None and 'error' not in map(itemgetter(0), output):
+            # Engine successfully created, and no syntax error in program.
+            for query, answers in test_cases:
+                # Limit inferences for each solution to curb unbounded recursion.
+                limited = 'call_with_inference_limit(({}), 100000, _)'.format(query)
+                if prolog.engine.check_answers(engine_id, query=limited, answers=answers, timeout=1.0):
+                    n_correct += 1
     finally:
         if engine_id:
             prolog.engine.destroy(engine_id)
diff --git a/prolog/problems/family_relations/mother_2/common.py b/prolog/problems/family_relations/mother_2/common.py
index 06aefef..21bf300 100644
--- a/prolog/problems/family_relations/mother_2/common.py
+++ b/prolog/problems/family_relations/mother_2/common.py
@@ -47,19 +47,17 @@ def test(program, solved_problems):
             server.problems.solutions_for_problems('prolog', solved_problems) + '\n' +
             server.problems.load_facts('prolog', facts).facts)
 
+    n_correct = 0
     engine_id = None
     try:
         engine_id, output = prolog.engine.create(code=code, timeout=1.0)
-        if not engine_id or 'error' in map(itemgetter(0), output):
-            # Engine creation failed, or syntax error in code.
-            return 0, len(test_cases)
-
-        n_correct = 0
-        for query, answers in test_cases:
-            # Limit inferences for each solution to curb unbounded recursion.
-            limited = 'call_with_inference_limit(({}), 100000, _)'.format(query)
-            if prolog.engine.check_answers(engine_id, query=limited, answers=answers, timeout=1.0):
-                n_correct += 1
+        if engine_id is not None and 'error' not in map(itemgetter(0), output):
+            # Engine successfully created, and no syntax error in program.
+            for query, answers in test_cases:
+                # Limit inferences for each solution to curb unbounded recursion.
+                limited = 'call_with_inference_limit(({}), 100000, _)'.format(query)
+                if prolog.engine.check_answers(engine_id, query=limited, answers=answers, timeout=1.0):
+                    n_correct += 1
     finally:
         if engine_id:
             prolog.engine.destroy(engine_id)
diff --git a/prolog/problems/family_relations/sister_2/common.py b/prolog/problems/family_relations/sister_2/common.py
index 56eba55..b09b29a 100644
--- a/prolog/problems/family_relations/sister_2/common.py
+++ b/prolog/problems/family_relations/sister_2/common.py
@@ -44,19 +44,17 @@ def test(program, solved_problems):
             server.problems.solutions_for_problems('prolog', solved_problems) + '\n' +
             server.problems.load_facts('prolog', facts).facts)
 
+    n_correct = 0
     engine_id = None
     try:
         engine_id, output = prolog.engine.create(code=code, timeout=1.0)
-        if not engine_id or 'error' in map(itemgetter(0), output):
-            # Engine creation failed, or syntax error in code.
-            return 0, len(test_cases)
-
-        n_correct = 0
-        for query, answers in test_cases:
-            # Limit inferences for each solution to curb unbounded recursion.
-            limited = 'call_with_inference_limit(({}), 100000, _)'.format(query)
-            if prolog.engine.check_answers(engine_id, query=limited, answers=answers, timeout=1.0):
-                n_correct += 1
+        if engine_id is not None and 'error' not in map(itemgetter(0), output):
+            # Engine successfully created, and no syntax error in program.
+            for query, answers in test_cases:
+                # Limit inferences for each solution to curb unbounded recursion.
+                limited = 'call_with_inference_limit(({}), 100000, _)'.format(query)
+                if prolog.engine.check_answers(engine_id, query=limited, answers=answers, timeout=1.0):
+                    n_correct += 1
     finally:
         if engine_id:
             prolog.engine.destroy(engine_id)
diff --git a/prolog/problems/lists/conc_3/common.py b/prolog/problems/lists/conc_3/common.py
index be4f118..a36d097 100644
--- a/prolog/problems/lists/conc_3/common.py
+++ b/prolog/problems/lists/conc_3/common.py
@@ -30,19 +30,17 @@ def test(program, solved_problems):
     code = (program + '\n' +
             server.problems.solutions_for_problems('prolog', solved_problems))
 
+    n_correct = 0
     engine_id = None
     try:
         engine_id, output = prolog.engine.create(code=code, timeout=1.0)
-        if not engine_id or 'error' in map(itemgetter(0), output):
-            # Engine creation failed, or syntax error in code.
-            return 0, len(test_cases)
-
-        n_correct = 0
-        for query, answers in test_cases:
-            # Limit inferences for each solution to curb unbounded recursion.
-            limited = 'call_with_inference_limit(({}), 100000, _)'.format(query)
-            if prolog.engine.check_answers(engine_id, query=limited, answers=answers, timeout=1.0):
-                n_correct += 1
+        if engine_id is not None and 'error' not in map(itemgetter(0), output):
+            # Engine successfully created, and no syntax error in program.
+            for query, answers in test_cases:
+                # Limit inferences for each solution to curb unbounded recursion.
+                limited = 'call_with_inference_limit(({}), 100000, _)'.format(query)
+                if prolog.engine.check_answers(engine_id, query=limited, answers=answers, timeout=1.0):
+                    n_correct += 1
     finally:
         if engine_id:
             prolog.engine.destroy(engine_id)
diff --git a/prolog/problems/lists/len_2/common.py b/prolog/problems/lists/len_2/common.py
index 6737f33..d30588c 100644
--- a/prolog/problems/lists/len_2/common.py
+++ b/prolog/problems/lists/len_2/common.py
@@ -29,19 +29,17 @@ def test(program, solved_problems):
     code = (program + '\n' +
             server.problems.solutions_for_problems('prolog', solved_problems))
 
+    n_correct = 0
     engine_id = None
     try:
         engine_id, output = prolog.engine.create(code=code, timeout=1.0)
-        if not engine_id or 'error' in map(itemgetter(0), output):
-            # Engine creation failed, or syntax error in code.
-            return 0, len(test_cases)
-
-        n_correct = 0
-        for query, answers in test_cases:
-            # Limit inferences for each solution to curb unbounded recursion.
-            limited = 'call_with_inference_limit(({}), 100000, _)'.format(query)
-            if prolog.engine.check_answers(engine_id, query=limited, answers=answers, timeout=1.0):
-                n_correct += 1
+        if engine_id is not None and 'error' not in map(itemgetter(0), output):
+            # Engine successfully created, and no syntax error in program.
+            for query, answers in test_cases:
+                # Limit inferences for each solution to curb unbounded recursion.
+                limited = 'call_with_inference_limit(({}), 100000, _)'.format(query)
+                if prolog.engine.check_answers(engine_id, query=limited, answers=answers, timeout=1.0):
+                    n_correct += 1
     finally:
         if engine_id:
             prolog.engine.destroy(engine_id)