Improve testing procedure

For each query:
  - generate a single solution
    - incorrect solution / timeout / other error → FAIL
  - generate (up to) ten solutions
    - more than one distinct solution found → FAIL
    - timeout is OK (a correct solution was found above)
  - if reached this point → PASS

1 files changed, 17 insertions, 2 deletions

diff --git a/prolog/engine.py b/prolog/engine.py
index 53307cb..ea9b19a 100644
--- a/prolog/engine.py
+++ b/prolog/engine.py
@@ -217,8 +217,23 @@ class PrologEngine(object):
                 predicates.add(self.predicate_indicator(rule))
 
             for query in queries:
-                result = self.query(query, m_user)
-                correct &= (len(result) == 1 and result[0]['X'] == self.answers[(pid, query)])
+                result = self.query(query, m_user, n=1)
+                if len(result) != 1 or result[0]['X'] != self.answers[(pid, query)]:
+                    correct = False
+                    break
+
+                # If a correct solution was found, see if another (incorrect)
+                # solution is found in the first 10 answers.
+                try:
+                    result = self.query(query, m_user, n=10)
+                    unique = set([r['X'] for r in result])
+                    if len(unique) != 1:
+                        correct = False
+                        break
+                except Exception as ex:
+                    # Only a timeout exception can occur here; in this case, we
+                    # consider [code] correct.
+                    pass
         except Exception as ex:
             correct = False