From e95aa102182957ccd35d0489dacd073bd76c3351 Mon Sep 17 00:00:00 2001 From: Timotej Lazar Date: Wed, 17 Dec 2014 15:41:49 +0100 Subject: Improve testing procedure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For each query: - generate a single solution - incorrect solution / timeout / other error → FAIL - generate (up to) ten solutions - more than one distinct solution found → FAIL - timeout is OK (a correct solution was found above) - if reached this point → PASS --- prolog/engine.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) (limited to 'prolog/engine.py') diff --git a/prolog/engine.py b/prolog/engine.py index 53307cb..ea9b19a 100644 --- a/prolog/engine.py +++ b/prolog/engine.py @@ -217,8 +217,23 @@ class PrologEngine(object): predicates.add(self.predicate_indicator(rule)) for query in queries: - result = self.query(query, m_user) - correct &= (len(result) == 1 and result[0]['X'] == self.answers[(pid, query)]) + result = self.query(query, m_user, n=1) + if len(result) != 1 or result[0]['X'] != self.answers[(pid, query)]: + correct = False + break + + # If a correct solution was found, see if another (incorrect) + # solution is found in the first 10 answers. + try: + result = self.query(query, m_user, n=10) + unique = set([r['X'] for r in result]) + if len(unique) != 1: + correct = False + break + except Exception as ex: + # Only a timeout exception can occur here; in this case, we + # consider [code] correct. + pass except Exception as ex: correct = False -- cgit v1.2.1