Start refactoring testing code

Testing logic now lives in PrologEngine. The engine now has some notion
of problems and users, which is necessary to avoid repeatedly loading
code into Prolog.

TODO:
  - support library loading
  - fix PrologEngine.test for unusual cases (more than one solution, …)
  - memoization of correct answers

3 files changed, 78 insertions, 73 deletions

diff --git a/prolog/engine.py b/prolog/engine.py
index 299d2eb..ec27e27 100755..100644
--- a/prolog/engine.py
+++ b/prolog/engine.py
@@ -1,7 +1,7 @@
 #!/usr/bin/python3
 
 from prolog.core import *
-from prolog.util import *
+import prolog.util
 
 class Atom(object):
     __slots__ = 'ref'
@@ -80,29 +80,25 @@ class PrologEngine(object):
 
         # construct some predicates
         self.p = {
-            'assertz': PL_predicate(b'assertz', 2, None),
-            'erase': PL_predicate(b'erase', 1, None),
-            'call_with_depth_limit': PL_predicate(b'call_with_depth_limit', 3, None),
-            'call_with_time_limit': PL_predicate(b'call_with_time_limit', 2, None),
-            'consult': PL_predicate(b'consult', 1, None),
-            'findnsols': PL_predicate(b'findnsols', 4, None),
-            'message_to_string': PL_predicate(b'message_to_string', 2, None),
-            'read_term_from_atom': PL_predicate(b'read_term_from_atom', 3, None),
-            'safe_goal': PL_predicate(b'safe_goal', 1, None),
-            'set_prolog_stack': PL_predicate(b'set_prolog_stack', 2, None)
+            'add_import_module/3': PL_predicate(b'add_import_module', 3, None),
+            'assertz/1': PL_predicate(b'assertz', 1, None),
+            'assertz/2': PL_predicate(b'assertz', 2, None),
+            'erase/1': PL_predicate(b'erase', 1, None),
+            'call_with_time_limit/2': PL_predicate(b'call_with_time_limit', 2, None),
+            'consult/1': PL_predicate(b'consult', 1, None),
+            'message_to_string/2': PL_predicate(b'message_to_string', 2, None),
+            'read_term_from_atom/3': PL_predicate(b'read_term_from_atom', 3, None),
+            'safe_goal/1': PL_predicate(b'safe_goal', 1, None),
+            'set_prolog_stack/2': PL_predicate(b'set_prolog_stack', 2, None)
         }
         self.err_flags = PL_Q_NODEBUG|PL_Q_CATCH_EXCEPTION
 
-        # database of current asserts; each load gets a new index
-        self.refs = {}
-        self.max_index = 0
+        # Increase memory limits.
+        self.call('set_prolog_stack/2', [Term('global'), Term('limit(2*10**9)')])
+        self.call('set_prolog_stack/2', [Term('local'), Term('limit(2*10**9)')])
 
-        # increase memory limits
-        self.call('set_prolog_stack', [Term('global'), Term('limit(2*10**9)')])
-        self.call('set_prolog_stack', [Term('local'), Term('limit(2*10**9)')])
-
-        # discard messages from swipl library
-        self.load('message_hook(_, _, _). ')
+        # Discard messages from swipl library.
+        self.call('assertz/1', [Term('message_hook(_, _, _)')])
 
     def exception(self, qid):
         ref = PL_exception(qid)
@@ -113,7 +109,7 @@ class PrologEngine(object):
             if ex is not None:
                 msg = Term()
                 if PL_call_predicate(None, self.err_flags,
-                                     self.p['message_to_string'], Termv([ex, msg]).ref):
+                                     self.p['message_to_string/2'], Termv([ex, msg]).ref):
                     raise Exception(str(msg))
                 raise Exception('Unknown error')
         finally:
@@ -131,43 +127,7 @@ class PrologEngine(object):
         return True
 
     def consult(self, filename):
-        return self.call('consult', [Term(Atom(filename))])
-
-    def load(self, program, module=None):
-        tokens = tokenize(program)
-        refs = []
-        try:
-            start = 0
-            for idx in range(len(tokens)):
-                if tokens[idx].type != 'PERIOD' or idx - start <= 1:
-                    continue
-                rule = stringify(tokens[start:idx])
-                orig_rule = rule
-                start = idx + 1
-
-                if module is not None:
-                    rule = module + ':(' + rule + ')'
-                ref = Term()
-                if self.call('assertz', [Term(rule), ref]):
-                    refs.append(ref)
-        except Exception as ex:
-            # unload already loaded rules (hacky, mustfix, todo)
-            for ref in refs:
-                self.call('erase', [ref])
-            raise Exception('Error loading program: {}\nBad rule: {}'.format(ex, orig_rule))
-
-        self.max_index += 1
-        self.refs[self.max_index] = refs
-        return self.max_index
-
-    def unload(self, index):
-        try:
-            for ref in self.refs[index]:
-                self.call('erase', [ref])
-            del self.refs[index]
-        except Exception as ex:
-            # This should never happen.
-            sys.stderr.write(str(ex) + '\n')
+        return self.call('consult/1', [Term(Atom(filename))])
 
     # Get up to [n] solutions to query [q].
     def query(self, q, module=None, n=1):
@@ -181,17 +141,17 @@ class PrologEngine(object):
         goal = Term()
         try:
             # Parse term and store variable names.
-            if not self.call('read_term_from_atom', [Term(Atom(q)), goal, options]):
+            if not self.call('read_term_from_atom/3', [Term(Atom(q)), goal, options]):
                 raise Exception('Warning: Could not read term from {}\n'.format(q))
             # Check if goal is safe with currently loaded rules.
-            if not self.call('safe_goal', [goal]):
+            if not self.call('safe_goal/1', [goal]):
                 raise Exception('Warning: Unsafe goal: {}\n'.format(goal))
         except Exception as ex:
             raise
 
         solutions = Term()
         goal_aux = Term('findnsols', [Term(n), goal, goal, solutions])
-        qid = PL_open_query(None, self.err_flags, self.p['call_with_time_limit'],
+        qid = PL_open_query(None, self.err_flags, self.p['call_with_time_limit/2'],
                             Termv([Term(0.01), goal_aux]).ref)
 
         result = []
@@ -215,15 +175,51 @@ class PrologEngine(object):
 
         return result
 
-# Basic sanity check.
-if __name__ == '__main__':
-    prolog = PrologEngine()
-    prolog.load('a(1). a(2). a(3).', 'test')
-    n = 1
-    while True:
-        solution = prolog.query('a(X), a(Y), X\=Y', 'test', n=n)
-        if solution is None or len(solution) != n:
-            print('{} solutions found!'.format(n-1))
-            break
-        print(solution[-1])
-        n += 1
+    # Loads the correct solution [code] to problem [pid].
+    # TODO handle library loading.
+    def load_solution(self, pid, code):
+        module = 'solution{}'.format(pid)
+        for rule in prolog.util.split(code):
+            self.call('assertz/1', [Term('{}:({})'.format(module, rule))])
+
+    # Import the correct solution for problem [pid] into module for user [uid].
+    def mark_solved(self, uid, pid):
+        m_user = 'user{}'.format(uid)
+        m_solution = 'solution{}'.format(pid)
+        return self.call('add_import_module/3', [Term(m_user), Term(m_solution), Term('end')])
+
+    # Test whether [code] gives the same answer to [query] as the correct
+    # solution.
+    def test(self, uid, pid, code, queries):
+        m_user = 'user{}'.format(uid)
+        m_solution = 'solution{}'.format(pid)
+
+        # Find the correct answers. Code for [pid] should be loaded beforehand.
+        # TODO Memoize correct answers.
+        answers = []
+        for query in queries:
+            result = self.query(query, m_solution)
+            if result is None or len(result) < 1 or 'X' not in result[0]:
+                raise Exception('Error finding correct answer to query "{}"'.format(query))
+            answers.append(result[0]['X'])  # TODO maybe we could check all vars?
+
+        correct = True
+        refs = []
+        try:
+            # Load the user program [code].
+            for rule in prolog.util.split(code):
+                ref = Term()
+                if self.call('assertz/2', [Term('{}:({})'.format(m_user, rule)), ref]):
+                    refs.append(ref)
+
+            for i, query in enumerate(queries):
+                result = self.query(query, m_user)
+                correct &= (len(result) == 1 and result[0]['X'] == answers[i])
+        except Exception as ex:
+            correct = False
+        finally:
+            # Unload all loaded rules.
+            for ref in refs:
+                self.call('erase/1', [ref])
+
+        return correct
diff --git a/prolog/lexer.py b/prolog/lexer.py
index 971e8a6..971e8a6 100755..100644
--- a/prolog/lexer.py
+++ b/prolog/lexer.py
diff --git a/prolog/util.py b/prolog/util.py
index b7e536b..0ab3c8b 100644
--- a/prolog/util.py
+++ b/prolog/util.py
@@ -27,6 +27,15 @@ def stringify(tokens):
         return str(t)
     return ''.join(map(token_str, tokens))
 
+# Yield the sequence of rules in [code].
+def split(code):
+    tokens = tokenize(code)
+    start = 0
+    for idx, token in enumerate(tokens):
+        if token.type == 'PERIOD' and idx - start > 1:
+            yield stringify(tokens[start:idx])
+            start = idx + 1
+
 # return a list of lines in 'code', and a list of rule indexes
 def decompose(code):
     lines = []