Use check_answers instead of ask_all for testing

Simplify the testing function and add correct test cases for Prolog
problems mother/2 and brother/2.

4 files changed, 78 insertions, 67 deletions

diff --git a/prolog/problems/family_relations/brother_2/common.py b/prolog/problems/family_relations/brother_2/common.py
index 48bc519..e725682 100644
--- a/prolog/problems/family_relations/brother_2/common.py
+++ b/prolog/problems/family_relations/brother_2/common.py
@@ -1,5 +1,6 @@
 # coding=utf-8
 
+from operator import itemgetter
 import socket
 import prolog.engine
 import prolog.util
@@ -28,15 +29,15 @@ hint_typs = {
     'x_y_must_be_different': Hint('x_y_must_be_different'),
 }
 
-def test(program, solved_problems):
-    # TODO: change queries and answers (these are for sister/2)
-    # Test queries and expected answers (values of X).
-    queries = [
-        ('sister(melanie, X)', set(['andrew'])),
-        ('sister(X, andrew)', set(['melanie'])),
-        ('sister(X, _)', set(['michelle', 'daniela', 'patricia', 'luana', 'sally', 'melanie', 'nevia', 'vanessa', 'anna'])),
-    ]
+test_cases = [
+    ('brother(X, _)',
+        [{'X': 'alessandro'}, {'X': 'andrew'}, {'X': 'jeffrey'},
+         {'X': 'kramer'}, {'X': 'michael'}, {'X': 'william'}]),
+    ('brother(andrew, X)',
+        [{'X': 'melanie'}]),
+]
 
+def test(program, solved_problems):
     code = (program + '\n' +
             server.problems.solutions_for_problems('prolog', solved_problems) + '\n' +
             server.problems.load_facts('prolog', facts).facts)
@@ -44,21 +45,21 @@ def test(program, solved_problems):
     engine_id = None
     try:
         engine_id, output = prolog.engine.create(code=code, timeout=1.0)
-        if not engine_id:
-            return 0, len(queries)
+        if not engine_id or 'error' in map(itemgetter(0), output):
+            # Engine creation failed, or syntax error in code.
+            return 0, len(test_cases)
 
         n_correct = 0
-        for query, solution in queries:
-            answers, error = prolog.engine.ask_all(engine_id, query=query, timeout=1.0)
-            if answers and answers[-1] != 'timed out':
-                values_x = set([ans.get('X') for ans, constraints in answers])
-                if values_x == solution:
-                    n_correct += 1
+        for query, answers in test_cases:
+            # Limit inferences for each solution to curb unbounded recursion.
+            limited = 'call_with_inference_limit(({}), 100000, _)'.format(query)
+            if prolog.engine.check_answers(engine_id, query=limited, answers=answers, timeout=1.0):
+                n_correct += 1
     finally:
         if engine_id:
             prolog.engine.destroy(engine_id)
 
-    return n_correct, len(queries)
+    return n_correct, len(test_cases)
 
 def hint(program, solved_problems):
     # tokens = prolog.util.tokenize(program)
diff --git a/prolog/problems/family_relations/father_2/common.py b/prolog/problems/family_relations/father_2/common.py
index dd799e7..1816d56 100644
--- a/prolog/problems/family_relations/father_2/common.py
+++ b/prolog/problems/family_relations/father_2/common.py
@@ -1,5 +1,6 @@
 # coding=utf-8
 
+from operator import itemgetter
 import socket
 import prolog.engine
 import prolog.util
@@ -27,15 +28,11 @@ hint_type = {
     'y_need_not_be_parent': Hint('y_need_not_be_parent'),
 }
 
-def test(program, solved_problems):
-    # TODO: change queries and answers (these are for sister/2)
-    # Test queries and expected answers (values of X).
-    queries = [
-        ('sister(melanie, X)', set(['andrew'])),
-        ('sister(X, andrew)', set(['melanie'])),
-        ('sister(X, _)', set(['michelle', 'daniela', 'patricia', 'luana', 'sally', 'melanie', 'nevia', 'vanessa', 'anna'])),
-    ]
+test_cases = [
+    # TODO
+]
 
+def test(program, solved_problems):
     code = (program + '\n' +
             server.problems.solutions_for_problems('prolog', solved_problems) + '\n' +
             server.problems.load_facts('prolog', facts).facts)
@@ -43,21 +40,21 @@ def test(program, solved_problems):
     engine_id = None
     try:
         engine_id, output = prolog.engine.create(code=code, timeout=1.0)
-        if not engine_id:
-            return 0, len(queries)
+        if not engine_id or 'error' in map(itemgetter(0), output):
+            # Engine creation failed, or syntax error in code.
+            return 0, len(test_cases)
 
         n_correct = 0
-        for query, solution in queries:
-            answers, error = prolog.engine.ask_all(engine_id, query=query, timeout=1.0)
-            if answers and answers[-1] != 'timed out':
-                values_x = set([ans.get('X') for ans, constraints in answers])
-                if values_x == solution:
-                    n_correct += 1
+        for query, answers in test_cases:
+            # Limit inferences for each solution to curb unbounded recursion.
+            limited = 'call_with_inference_limit(({}), 100000, _)'.format(query)
+            if prolog.engine.check_answers(engine_id, query=limited, answers=answers, timeout=1.0):
+                n_correct += 1
     finally:
         if engine_id:
             prolog.engine.destroy(engine_id)
 
-    return n_correct, len(queries)
+    return n_correct, len(test_cases)
 
 def hint(program, solved_problems):
     tokens = prolog.util.tokenize(program)
diff --git a/prolog/problems/family_relations/mother_2/common.py b/prolog/problems/family_relations/mother_2/common.py
index 2b74f08..3a7e196 100644
--- a/prolog/problems/family_relations/mother_2/common.py
+++ b/prolog/problems/family_relations/mother_2/common.py
@@ -1,5 +1,6 @@
 # coding=utf-8
 
+from operator import itemgetter
 import socket
 import prolog.engine
 import prolog.util
@@ -27,15 +28,22 @@ hint_type = {
     'y_need_not_be_parent': Hint('y_need_not_be_parent'),
 }
 
-def test(program, solved_problems):
-    # TODO: change queries and answers (these are for sister/2)
-    # Test queries and expected answers (values of X).
-    queries = [
-        ('sister(melanie, X)', set(['andrew'])),
-        ('sister(X, andrew)', set(['melanie'])),
-        ('sister(X, _)', set(['michelle', 'daniela', 'patricia', 'luana', 'sally', 'melanie', 'nevia', 'vanessa', 'anna'])),
-    ]
+test_cases = [
+    ('mother(X, _)',
+        [{'X': 'ana'}, {'X': 'elaine'}, {'X': 'estelle'}, {'X': 'helen'},
+         {'X': 'jill'}, {'X': 'joanne'}, {'X': 'margaret'}, {'X': 'nevia'},
+         {'X': 'patricia'}, {'X': 'sally'}, {'X': 'tina'}, {'X': 'vanessa'}]),
+    ('mother(_, X)',
+        [{'X': 'aleksander'}, {'X': 'alessandro'}, {'X': 'andrew'},
+         {'X': 'anna'}, {'X': 'daniela'}, {'X': 'george'}, {'X': 'jerry'},
+         {'X': 'joanne'}, {'X': 'john'}, {'X': 'kramer'}, {'X': 'luana'},
+         {'X': 'melanie'}, {'X': 'nevia'}, {'X': 'steve'}, {'X': 'susan'},
+         {'X': 'william'}]),
+    ('mother(joanne, X)',
+        [{'X': 'steve'}]),
+]
 
+def test(program, solved_problems):
     code = (program + '\n' +
             server.problems.solutions_for_problems('prolog', solved_problems) + '\n' +
             server.problems.load_facts('prolog', facts).facts)
@@ -43,21 +51,21 @@ def test(program, solved_problems):
     engine_id = None
     try:
         engine_id, output = prolog.engine.create(code=code, timeout=1.0)
-        if not engine_id:
-            return 0, len(queries)
+        if not engine_id or 'error' in map(itemgetter(0), output):
+            # Engine creation failed, or syntax error in code.
+            return 0, len(test_cases)
 
         n_correct = 0
-        for query, solution in queries:
-            answers, error = prolog.engine.ask_all(engine_id, query=query, timeout=1.0)
-            if answers and answers[-1] != 'timed out':
-                values_x = set([ans.get('X') for ans, constraints in answers])
-                if values_x == solution:
-                    n_correct += 1
+        for query, answers in test_cases:
+            # Limit inferences for each solution to curb unbounded recursion.
+            limited = 'call_with_inference_limit(({}), 100000, _)'.format(query)
+            if prolog.engine.check_answers(engine_id, query=limited, answers=answers, timeout=1.0):
+                n_correct += 1
     finally:
         if engine_id:
             prolog.engine.destroy(engine_id)
 
-    return n_correct, len(queries)
+    return n_correct, len(test_cases)
 
 def hint(program, solved_problems):
     tokens = prolog.util.tokenize(program)
diff --git a/prolog/problems/family_relations/sister_2/common.py b/prolog/problems/family_relations/sister_2/common.py
index 9c63b18..a1375f2 100644
--- a/prolog/problems/family_relations/sister_2/common.py
+++ b/prolog/problems/family_relations/sister_2/common.py
@@ -1,5 +1,6 @@
 # coding=utf-8
 
+from operator import itemgetter
 import socket
 import prolog.engine
 import prolog.util
@@ -28,14 +29,18 @@ hint_type = {
     'x_y_must_be_different': Hint('x_y_must_be_different'),
 }
 
-def test(program, solved_problems):
-    # Test queries and expected answers (values of X).
-    queries = [
-        ('sister(melanie, X)', set(['andrew'])),
-        ('sister(X, andrew)', set(['melanie'])),
-        ('sister(X, _)', set(['michelle', 'daniela', 'patricia', 'luana', 'sally', 'melanie', 'nevia', 'vanessa', 'anna'])),
-    ]
+test_cases = [
+    ('sister(X, _)',
+        [{'X': 'anna'}, {'X': 'daniela'}, {'X': 'luana'}, {'X': 'melanie'},
+         {'X': 'michelle'}, {'X': 'nevia'}, {'X': 'patricia'}, {'X': 'sally'},
+         {'X': 'vanessa'}]),
+    ('sister(melanie, X)',
+        [{'X': 'andrew'}]),
+    ('sister(X, andrew)',
+        [{'X': 'melanie'}]),
+]
 
+def test(program, solved_problems):
     code = (program + '\n' +
             server.problems.solutions_for_problems('prolog', solved_problems) + '\n' +
             server.problems.load_facts('prolog', facts).facts)
@@ -43,21 +48,21 @@ def test(program, solved_problems):
     engine_id = None
     try:
         engine_id, output = prolog.engine.create(code=code, timeout=1.0)
-        if not engine_id:
-            return 0, len(queries)
+        if not engine_id or 'error' in map(itemgetter(0), output):
+            # Engine creation failed, or syntax error in code.
+            return 0, len(test_cases)
 
         n_correct = 0
-        for query, solution in queries:
-            answers, error = prolog.engine.ask_all(engine_id, query=query, timeout=1.0)
-            if answers and answers[-1] != 'timed out':
-                values_x = set([ans.get('X') for ans, constraints in answers])
-                if values_x == solution:
-                    n_correct += 1
+        for query, answers in test_cases:
+            # Limit inferences for each solution to curb unbounded recursion.
+            limited = 'call_with_inference_limit(({}), 100000, _)'.format(query)
+            if prolog.engine.check_answers(engine_id, query=limited, answers=answers, timeout=1.0):
+                n_correct += 1
     finally:
         if engine_id:
             prolog.engine.destroy(engine_id)
 
-    return n_correct, len(queries)
+    return n_correct, len(test_cases)
 
 def hint(program, solved_problems):
     # how do I know which general hints were already shown?