summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTimotej Lazar <timotej.lazar@araneo.org>2014-12-17 15:41:49 +0100
committerAleš Smodiš <aless@guru.si>2015-08-11 14:26:00 +0200
commite95aa102182957ccd35d0489dacd073bd76c3351 (patch)
treebb06694631be72af8feefd311a1f8fb3e13fbdd9
parentd7b48d4250153aa21879609b5707af9aad434ad1 (diff)
Improve testing procedure
For each query: - generate a single solution - incorrect solution / timeout / other error → FAIL - generate (up to) ten solutions - more than one distinct solution found → FAIL - timeout is OK (a correct solution was found above) - if reached this point → PASS
-rw-r--r--prolog/engine.py19
1 files changed, 17 insertions, 2 deletions
diff --git a/prolog/engine.py b/prolog/engine.py
index 53307cb..ea9b19a 100644
--- a/prolog/engine.py
+++ b/prolog/engine.py
@@ -217,8 +217,23 @@ class PrologEngine(object):
predicates.add(self.predicate_indicator(rule))
for query in queries:
- result = self.query(query, m_user)
- correct &= (len(result) == 1 and result[0]['X'] == self.answers[(pid, query)])
+ result = self.query(query, m_user, n=1)
+ if len(result) != 1 or result[0]['X'] != self.answers[(pid, query)]:
+ correct = False
+ break
+
+ # If a correct solution was found, see if another (incorrect)
+ # solution is found in the first 10 answers.
+ try:
+ result = self.query(query, m_user, n=10)
+ unique = set([r['X'] for r in result])
+ if len(unique) != 1:
+ correct = False
+ break
+ except Exception as ex:
+ # Only a timeout exception can occur here; in this case, we
+ # consider [code] correct.
+ pass
except Exception as ex:
correct = False