1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
|
# coding=utf-8
from operator import itemgetter
import socket
import prolog.engine
import prolog.util
from server.hints import Hint, HintSequence
import server.problems
id = 96
group = 'family_relations'
number = 3
visible = True
facts = 'family_relations'
solution = '''\
sister(X, Y) :-
parent(P, X),
parent(P, Y),
female(X),
X \== Y.
'''
hint_type = {
'x_must_be_female': Hint('x_must_be_female'),
'common_parent_needed': Hint('common_parent_needed'),
'y_can_be_of_any_gender': Hint('Y_can_be_of_any_gender'),
'x_y_must_be_different': Hint('x_y_must_be_different'),
}
test_cases = [
('sister(X, _)',
[{'X': 'anna'}, {'X': 'daniela'}, {'X': 'luana'}, {'X': 'melanie'},
{'X': 'michelle'}, {'X': 'nevia'}, {'X': 'patricia'}, {'X': 'sally'},
{'X': 'vanessa'}]),
('sister(melanie, X)',
[{'X': 'andrew'}]),
('sister(X, andrew)',
[{'X': 'melanie'}]),
]
def test(program, solved_problems):
code = (program + '\n' +
server.problems.solutions_for_problems('prolog', solved_problems) + '\n' +
server.problems.load_facts('prolog', facts).facts)
engine_id = None
try:
engine_id, output = prolog.engine.create(code=code, timeout=1.0)
if not engine_id or 'error' in map(itemgetter(0), output):
# Engine creation failed, or syntax error in code.
return 0, len(test_cases)
n_correct = 0
for query, answers in test_cases:
# Limit inferences for each solution to curb unbounded recursion.
limited = 'call_with_inference_limit(({}), 100000, _)'.format(query)
if prolog.engine.check_answers(engine_id, query=limited, answers=answers, timeout=1.0):
n_correct += 1
finally:
if engine_id:
prolog.engine.destroy(engine_id)
passed = n_correct == len(test_cases)
hints = [{'id': 'test_results', 'args': {'passed': n_correct, 'total': len(test_cases)}}]
return passed, hints
def hint(program, solved_problems):
# how do I know which general hints were already shown?
# how do I know enough time has elapsed for general hints to be shown?
# how do I know whether the hint button was pressed?
# should we have another hint button which appears in hint window (like more...)
# when a hint is available after testing the current version of the program
# and this button triggers code-specific hints (i.e. non-general ones)
# one way to do hints is to use a hierarchy of hints...
# similar to how some rule-based systems work
# the trigger-testing could be easier that way as well
# what I have in mind is:
# e.g. when I test if X is sister to itself, I've already done other trigger-tests
# and I don't need to do them again
# first general hints (triggered by the hint button)
# do we do it like this? this button triggers general hints only?
# not sure this is ok.. discuss with others (include Janez)
# if hbutton.pressed:
# trigger generalhint(#buttonpress)
# code-specific hints next
# tokenize (and similar) only if *needed* for a given exercise
# to reduce server processor load
tokens = prolog.util.tokenize(program)
code = (program + '\n' +
server.problems.solutions_for_problems('prolog', solved_problems) + '\n' +
server.problems.load_facts('prolog', facts).facts)
engine_id = None
try:
engine_id, output = prolog.engine.create(code=code, timeout=1.0)
if not engine_id:
raise Exception('Prolog engine failed to create.')
# X must be female
if prolog.engine.ask_truth(engine_id, 'male(X), sister(X, _)'):
return [{'id': 'x_must_be_female'}]
# X and Y must have a common parent
if prolog.engine.ask_truth(engine_id,
'sister(X, Y), \+ (parent(P, X), parent(P, Y))'):
return [{'id': 'common_parent_needed'}]
# Y can be of any gender, incl. male
# as this is after previous hints, by this stage the code probably
# does return some sensible results already
# this is an example of using the ordering of hints for easier trigger checks
if prolog.engine.ask_one(engine_id,
'sister(_, Y), male(Y)') == 'false':
return [{'id': 'y_can_be_of_any_gender'}]
# X and Y must be different
# this is the last code-specific hint (when everything else doesn't trigger anymore)
if prolog.engine.ask_truth(engine_id,
'setof(X, (member(X, [sally, nevia, vanessa]), sister(X, X)), L), length(L, 3)'):
return [{'id': 'x_y_must_be_different'}]
# last hints are connected with programming style
# here, this means detecting such code:
# parent(P1, X), parent(P2, Y), P1 == P2 (or P1 = P2)
# I guess program tokens can be used for this
# do we require that the program is correct for this?
# or is it enough that this is after ALL code-specific hints AND it has to trigger?
# regular expressions, anyone?
# QUESTION to discuss:
# where do AI hints come into play?
# only after all code-specific and non stylistic hints are exhausted?
# or randomly before? or?
# QUESTION to discuss:
# I suggest hint triggers (if code-based or also token-based?)
# are used as CLASSES of unit tests
# although.. this makes random counterexamples harder to implement
# (does it?)
except socket.timeout as ex:
pass
finally:
if engine_id:
prolog.engine.destroy(engine_id)
return []
|
