Merge branch 'master' of ssh://212.235.189.51:22122/codeq-server

9 files changed, 222 insertions, 215 deletions

diff --git a/prolog/engine.py b/prolog/engine.py
index 50582c8..eca0826 100644
--- a/prolog/engine.py
+++ b/prolog/engine.py
@@ -6,7 +6,6 @@ import http.client
 import json
 from operator import itemgetter
 import re
-import socket
 import time
 import urllib
 
@@ -112,46 +111,45 @@ def pretty_vars(data):
         result += [strip_html(b) for b in data['residuals']]
     return ',\n'.join(result) if result else 'true'
 
-# Run [query] in the pengine with id [engine] and return the list of answers
-# found within [timeout] seconds. If a timeout occurs before the query is done,
-# 'timed out' is appended as the last answer.
-def ask_all(engine, query, timeout=10):
-    # Returns a tuple ((bindings, constraints), error, more?) for one answer.
+# Run [query] in pengine with ID [engine] and check whether all solutions
+# specified by [answers] (a list of binding dictionaries) are returned. This
+# function succeeds if [query] finds each solution in [answers] at least once
+# within [timeout] seconds, and fails when it finds any other solution.
+def check_answers(engine, query, answers, timeout=10):
+    seen = []
     start = time.monotonic()
-    answers, messages = [], []
     try:
         # Run the query.
         reply, output = ask(engine, query, timeout)
-        messages += output
-        if 'error' in map(itemgetter(0), output):
-            return answers, messages
         answer, error, more = process_answer(reply)
         if answer:
-            answers.append(answer)
-        if error:
-            messages.append(error)
+            bindings, constraints = answer
+            if bindings not in answers:
+                return False
+            if bindings not in seen:
+                seen.append(bindings)
 
         # Continue while there are more potential answers and time remaining.
         while more:
             real_timeout = timeout - (time.monotonic()-start)
             if real_timeout <= 0:
-                raise socket.timeout()
+                break
             reply, output = next(engine, timeout=real_timeout)
-            messages += output
             answer, error, more = process_answer(reply)
             if answer:
-                answers.append(answer)
-            if error:
-                messages.append(error)
-    except socket.timeout as ex:
-        answers.append('timed out')
-    return answers, messages
+                bindings, constraints = answer
+                if bindings not in answers:
+                    return False
+                if bindings not in seen:
+                    seen.append(bindings)
+    except:
+        pass
+    return len(seen) == len(answers)
 
 # Run [query] in the pengine with id [engine] and return the first answer only
 # found within [timeout] seconds.
 # used for quick hint debugging by Sasha
 def ask_one(engine, query, timeout=1):
-    # quicker than ask_all as there could be many hint-triggering tests
     # Returns either an error message, true, false, timeout (see below), or bindings
     # Timeout is "returned" as an unhandled exception -- deliberately so
 
@@ -175,7 +173,6 @@ def ask_one(engine, query, timeout=1):
         return 'false'
 
 def ask_truth(engine, query, timeout=1):
-    # quicker than ask_all as there could be many hint-triggering tests
     # Returns either True or False
     # (Runtime) error is False!
     # Timeout is an unhandled exception -- deliberately so
diff --git a/python/interpreter.py b/python/interpreter.py
new file mode 100755
index 0000000..87de3aa
--- /dev/null
+++ b/python/interpreter.py
@@ -0,0 +1,40 @@
+#!/usr/bin/python3
+
+import code
+import sys
+
+import seccomp
+
+f = seccomp.SyscallFilter(defaction=seccomp.KILL)
+# Necessary for Python.
+f.add_rule(seccomp.ALLOW, "exit_group")
+f.add_rule(seccomp.ALLOW, "rt_sigaction")
+f.add_rule(seccomp.ALLOW, "brk")
+
+# Mostly harmless.
+f.add_rule(seccomp.ALLOW, "mprotect")
+
+# Allow reading from stdin and writing to stdout/stderr.
+f.add_rule(seccomp.ALLOW, "read", seccomp.Arg(0, seccomp.EQ, sys.stdin.fileno()))
+f.add_rule(seccomp.ALLOW, "write", seccomp.Arg(0, seccomp.EQ, sys.stdout.fileno()))
+f.add_rule(seccomp.ALLOW, "write", seccomp.Arg(0, seccomp.EQ, sys.stderr.fileno()))
+
+f.add_rule(seccomp.ALLOW, "ioctl")
+f.add_rule(seccomp.ALLOW, "mmap")
+f.add_rule(seccomp.ALLOW, "munmap")
+
+# Needed for finding source code for exceptions.
+f.add_rule(seccomp.ALLOW, "stat")
+f.add_rule(seccomp.ALLOW, "open", seccomp.Arg(1, seccomp.MASKED_EQ, 0x3, 0))
+f.add_rule(seccomp.ALLOW, "fcntl")
+f.add_rule(seccomp.ALLOW, "fstat")
+f.add_rule(seccomp.ALLOW, "lseek")
+f.add_rule(seccomp.ALLOW, "read")
+f.add_rule(seccomp.ALLOW, "close")
+
+# Needed for code.InteractiveConsole.
+f.add_rule(seccomp.ALLOW, "access")
+f.add_rule(seccomp.ALLOW, "select")
+f.load()
+
+code.interact(banner='')
diff --git a/python/runner/interpreter.py b/python/runner/interpreter.py
deleted file mode 100755
index 5fa320a..0000000
--- a/python/runner/interpreter.py
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/usr/bin/python3
-
-# Apparently there is no (working) way to get a non-blocking stdout if we call
-# the Python interpreter directly with subprocess.Popen. For some reason, this
-# works.
-
-import code
-code.interact(banner='')
diff --git a/python/runner/main.py b/python/runner/main.py
deleted file mode 100755
index 4e1af53..0000000
--- a/python/runner/main.py
+++ /dev/null
@@ -1,117 +0,0 @@
-#!/usr/bin/python3
-
-from fcntl import fcntl, F_GETFL, F_SETFL
-import io
-import multiprocessing
-import multiprocessing.managers
-import os
-import subprocess
-import sys
-import threading
-import time
-import uuid
-
-interpreters = {}
-module_access_lock = threading.Lock()
-
-# Execute [code] and evaluate [expr]. Input is given by the string [stdin].
-# Return result of evaluation, the contents of stdout and stderr, and the
-# exception traceback.
-def _run_exec(conn, code, expr=None, stdin=''):
-    result, out, err, exc = None, None, None, None
-    sys.stdin = io.StringIO(stdin)
-    sys.stdout = io.StringIO()
-    sys.stderr = io.StringIO()
-    try:
-        env = {}
-        if code:
-            exec(code, env)
-        if expr:
-            result = eval(expr, env)
-    except Exception as ex:
-        # Exception is not JSON serializable, so return traceback as string.
-        import traceback
-        exc = traceback.format_exc()
-    finally:
-        out = sys.stdout.getvalue()
-        err = sys.stderr.getvalue()
-        sys.stdin.close()
-        sys.stdout.close()
-        sys.stderr.close()
-        conn.send((result, out, err, exc))
-
-class Python(object):
-    # Call run_exec in a separate process for each input and return a list of
-    # results from those calls. If a call times out, 'timed out' is returned in
-    # place of exception traceback.
-    def run(self, code, inputs, timeout):
-        # Launch processes.
-        futures = []
-        for expr, stdin in inputs:
-            conn_parent, conn_child = multiprocessing.Pipe()
-            p = multiprocessing.Process(target=_run_exec, args=(conn_child, code, expr, stdin))
-            p.start()
-            futures.append((p, conn_parent))
-
-        # Wait for results.
-        results = []
-        start = time.monotonic()
-        for p, conn in futures:
-            now = time.monotonic()
-            real_timeout = max(0, timeout - (now - start))
-            if conn.poll(real_timeout):
-                results.append(conn.recv())
-            else:
-                results.append('timed out')
-            p.terminate()
-        return results
-
-    # Start and return a new Python interpreter process.
-    def create(self):
-        directory = os.path.dirname(os.path.realpath(__file__))
-        script = os.path.join(directory, 'interpreter.py')
-        p = subprocess.Popen([script],
-                stdin=subprocess.PIPE,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.STDOUT)
-        # Set the non-blocking flag for stdout.
-        flags = fcntl(p.stdout, F_GETFL)
-        fcntl(p.stdout, F_SETFL, flags | os.O_NONBLOCK)
-
-        interpreter_id = uuid.uuid4().hex
-        with module_access_lock:
-            interpreters[interpreter_id] = p
-        return interpreter_id
-
-    # Read any available bytes from the interpreter's stdout.
-    def pull(self, interpreter_id):
-        with module_access_lock:
-            interpreter = interpreters[interpreter_id]
-        stdout = interpreter.stdout.read()
-        if stdout:
-            stdout = stdout.decode('utf-8')
-        return stdout
-
-    # Push a string to the interpreter's stdin.
-    def push(self, interpreter_id, stdin):
-        with module_access_lock:
-            interpreter = interpreters[interpreter_id]
-        interpreter.stdin.write(stdin.encode('utf-8'))
-        interpreter.stdin.flush()
-
-    # Kill an interpreter process.
-    def destroy(self, interpreter_id):
-        with module_access_lock:
-            interpreter = interpreters[interpreter_id]
-            del interpreters[interpreter_id]
-        interpreter.kill()
-
-class PythonManager(multiprocessing.managers.BaseManager):
-    pass
-
-PythonManager.register('Python', callable=Python)
-
-if __name__ == '__main__':
-    print('Python engine started.')
-    m = PythonManager(address=('localhost', 3031), authkey=b'c0d3q3y-python')
-    m.get_server().serve_forever()
diff --git a/readme.md b/readme.md
index a7e11ca..0b16214 100644
--- a/readme.md
+++ b/readme.md
@@ -1,5 +1,18 @@
-Deployment
-==========
+Installation
+============
+
+Install the following packages:
+
+  - apache2
+  - nodejs (>= 10.0.22)
+  - python3 (>= 3.4)
+  - python3-ply
+  - python3-psycopg2
+  - python3-termcolor
+  - swi-prolog-nox (>= 7.2)
+
+SWI prolog
+----------
 
 To use the correct SWI prolog package (>= 7.2) on Debian, add a custom
 repository by creating the file /etc/apt/sources.list.d/prolog.list
@@ -16,17 +29,29 @@ apt-get update
 apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EF8406856DBFCA18
 apt-get update
 
-Install the following packages:
+libseccomp
+----------
 
-  - apache2
-  - python3 (>= 3.4)
-  - python3-ply
-  - python3-psycopg2
-  - python3-termcolor
-  - swi-prolog-nox (>= 7.2)
-  - nodejs (>= 10.0.22)
+A new version of libseccomp (≥ 2.2) with Python bindings is needed to sandbox
+Python interpreters. This is not available yet in most distributions, so fetch
+the latest sources with
+
+    git clone https://github.com/seccomp/libseccomp
+
+then build and install it with
+
+    ./configure --enable-python
+    make
+    make install
+
+nodejs
+------
+
+Run "npm install" inside the "web" directory to install all dependencies (they
+will be installed inside the "web" directory)
 
-Settings:
+Settings
+========
 
   - point webroot to codeq-web
   - set up reverse proxy for /ws/ to the node server:
@@ -40,13 +65,11 @@ Settings:
       ProxyPassReverse /ws/ http://localhost:8083/ws/
   - set _path_prefix in server.problems
   - set DB settings in db
-  - run "npm install" inside the "web" directory to install all dependencies
-    (they will be installed inside the "web" directory)
 
-Running:
+Running
+=======
 
   - run the prolog interpreter server: swipl prolog/runner/main.pl
-  - run the python interpreter server: python3 python/runner/main.py
   - run the session daemon: python3 daemon.py
   - run the web server for client communication: node web/main.js
 
diff --git a/server/handlers.py b/server/handlers.py
index fb50ff4..0cc474f 100644
--- a/server/handlers.py
+++ b/server/handlers.py
@@ -101,14 +101,6 @@ class Query(CodeqService):
                 request.reply(result)
 
 
-# Pull stdout/stderr from the session's Python interpreter. TODO: convert to async handling
-class PythonPull(CodeqService):
-    def process(self, request):
-        python = request.session.get_python()
-        output = python.pull()
-        request.reply({'code': 0, 'message': 'ok', 'terminal': {'text': output if output else ''}})
-
-
 # Push stdin to the session's Python interpreter. TODO: convert to async handling
 class PythonPush(CodeqService):
     def process(self, request):
@@ -198,7 +190,6 @@ incoming_handlers = {
     'activity': Activity(),
     'query': Query(),
     'python_push': PythonPush(),
-    'python_pull': PythonPull(),
     'hint': Hint(),
     'test': Test()
 }
diff --git a/server/python_session.py b/server/python_session.py
index 4af455f..62fcbf8 100644
--- a/server/python_session.py
+++ b/server/python_session.py
@@ -1,8 +1,15 @@
 # coding=utf-8
 
 import ast
-import multiprocessing.managers
+from fcntl import fcntl, F_GETFL, F_SETFL
+import io
+import multiprocessing
+import os
+import queue
+import subprocess
+import sys
 import threading
+import time
 
 import server.user_session
 from db.models import Problem
@@ -17,53 +24,45 @@ class PythonSession(object):
     No properties are accessible; use getters and setters instead.
     Values are passed by value instead of by reference (deep copy!).
     """
-    def __init__(self):
+    def __init__(self, output_cb=None):
         self._access_lock = threading.Lock()
         self._sent_hints = []
-        self._interpreter = None
 
-        # Proxy for calling the Python runner. We use a separate connection for
-        # each session so the runner can be restarted without affecting the
-        # server.
-        _m = multiprocessing.managers.BaseManager(address=('localhost', 3031),
-                                                  authkey=b'c0d3q3y-python')
-        _m.register('Python')
-        _m.connect()
-        self._python = _m.Python()
-
-        self.create()
+        self._control = queue.Queue()
+        self._interpreter = threading.Thread(target=_interpreter,
+                                         kwargs={'control': self._control, 'callback': output_cb})
+        self._interpreter.start()
 
     def run(self, code=None, inputs=None, timeout=1.0):
-        with self._access_lock:
-            return self._python.run(code, inputs, timeout)
-
-    def create(self):
-        with self._access_lock:
-            if self._interpreter is None:
-                self._interpreter = self._python.create()
-
-    def pull(self):
-        with self._access_lock:
-            if self._interpreter is None:
-                return None
-            return self._python.pull(self._interpreter)
+        # Launch processes.
+        futures = []
+        for expr, stdin in inputs:
+            conn_parent, conn_child = multiprocessing.Pipe()
+            p = multiprocessing.Process(target=_run_exec, args=(conn_child, code, expr, stdin))
+            p.start()
+            futures.append((p, conn_parent))
+
+        # Wait for results.
+        results = []
+        start = time.monotonic()
+        for p, conn in futures:
+            now = time.monotonic()
+            real_timeout = max(0, timeout - (now - start))
+            if conn.poll(real_timeout):
+                results.append(conn.recv())
+            else:
+                results.append('timed out')
+            p.terminate()
+        return results
 
     def push(self, stdin):
-        with self._access_lock:
-            if self._interpreter is not None:
-                self._python.push(self._interpreter, stdin)
+        self._control.put_nowait(('push', stdin))
 
     def destroy(self):
-        with self._access_lock:
-            if self._interpreter is not None:
-                self._python.destroy(self._interpreter)
-                self._interpreter = None
+        self._control.put_nowait(('done', None))
 
     def __del__(self):
-        # no locking needed if GC is removing us, as there cannot be any concurrent access by definition
-        if hasattr(self, '_interpreter') and self._interpreter is not None:
-            self._python.destroy(self._interpreter)
-            self._interpreter = None
+        self.destroy()
 
     def hint(self, sid, problem_id, program):
         language, problem_group, problem = Problem.get_identifier(problem_id)
@@ -105,3 +104,89 @@ class PythonSession(object):
                         hint_type = mod.hint_type[hint['id']]
                         hint_type.instantiate(hint, self._sent_hints)
             self._sent_hints.extend(hints)
+
+def _interpreter(control, callback):
+    directory = os.path.dirname(os.path.realpath(__file__))
+    # TODO drop privileges using a wrapper
+    script = os.path.join(directory, '..', 'python', 'interpreter.py')
+
+    proc = None
+    while True:
+        # Ensure the interpreter process is running.
+        if proc is None:
+            proc = subprocess.Popen([script],
+                    stdin=subprocess.PIPE,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.STDOUT)
+            # Set the non-blocking flag for stdout.
+            flags = fcntl(proc.stdout.fileno(), F_GETFL)
+            fcntl(proc.stdout.fileno(), F_SETFL, flags | os.O_NONBLOCK)
+
+        # Get a control command.
+        try:
+            cmd, data = control.get_nowait()
+            if cmd == 'push':
+                proc.stdin.write(data.encode('utf-8'))
+                proc.stdin.flush()
+            elif cmd == 'done':
+                break
+        except:
+            pass
+
+        # Communicate with child process.
+        retcode = proc.poll()
+        if retcode is None:
+            data = proc.stdout.read()
+            if data:
+                # NOTE this might fail if read() stops in the middle of utf8 sequence
+                text = data.decode('utf-8')
+                if text:
+                    callback(text)
+        else:
+            if retcode == -31:
+                callback('Child killed due to sandbox misbehavior.\n')
+            else:
+                callback('Child exited with status "{}".\n'.format(retcode))
+            proc = None
+
+        # TODO we should select() on control and proc.stdout instead of polling
+        time.sleep(0.1)
+
+    # We are done, kill the child.
+    if proc is not None:
+        proc.kill()
+
+# Execute [code] and evaluate [expr]. Input is given by the string [stdin].
+# Return result of evaluation, the contents of stdout and stderr, and the
+# exception traceback.
+# TODO sandbox this
+def _run_exec(conn, code, expr=None, stdin=''):
+    result, out, err, exc = None, None, None, None
+    sys.stdin = io.StringIO(stdin)
+    sys.stdout = io.StringIO()
+    sys.stderr = io.StringIO()
+    try:
+        env = {}
+        if code:
+            exec(code, env)
+        if expr:
+            result = eval(expr, env)
+    except Exception as ex:
+        # Exception is not JSON serializable, so return traceback as string
+        # (without the first entry, which is this function).
+        import traceback
+        e_type, e_value, e_tb = sys.exc_info()
+        stack = traceback.extract_tb(e_tb)
+        exc = ''.join(
+                ['Traceback (most recent call last):\n'] +
+                ['  line {}, in {}\n'.format(lineno, name) + (line+'\n' if line else '')
+                    for filename, lineno, name, line in stack[1:]] +
+                traceback.format_exception_only(e_type, e_value)
+            ).rstrip()
+    finally:
+        out = sys.stdout.getvalue()
+        err = sys.stderr.getvalue()
+        sys.stdin.close()
+        sys.stdout.close()
+        sys.stderr.close()
+        conn.send((result, out, err, exc))
diff --git a/server/user_session.py b/server/user_session.py
index 2cc629c..0d8535c 100644
--- a/server/user_session.py
+++ b/server/user_session.py
@@ -62,7 +62,8 @@ class UserSession(object):
     def get_python(self):
         with self._access_lock:
             if self.python_session is None:
-                self.python_session = python_session.PythonSession()  # lazy init
+                self.python_session = python_session.PythonSession(
+                    output_cb=lambda text: self.send({'event': 'terminal_output', 'text': text}))
             return self.python_session
 
     def get_problem_data(self, language, problem_group, problem):
diff --git a/web/main.js b/web/main.js
index c4c8691..03e9dad 100644
--- a/web/main.js
+++ b/web/main.js
@@ -137,11 +137,6 @@ var guiHandlers = {
         sendDataToPython(message).then(session.send, session.end).done();
     },
 
-    'python_pull': function actionPythonPull(session, message) {
-        logger.debug('Received python_pull from GUI');
-        sendDataToPython(message).then(session.send, session.end).done();
-    },
-
     'hint': function actionHint(session, message) {
         logger.debug('Received hint from GUI');
         sendDataToPython(message).then(session.send, session.end).done();