Mercurial > ~astiob > upreckon > hgweb
annotate problem.py @ 76:0e5ae28e0b2b
Points are now weighted on a test context basis
In particular, this has allowed for simple extensions to the format
of testconf to award points to whole test groups without at the same time
compromising the future ability of giving partial score for correct
but slow solutions. Specifically, the groupweight configuration variable
has been added and normally has the format {groupindex: points} where
groupindex is the group's index in the tests configuration variable.
The backwards incompatible change is that test contexts are no longer
guaranteed to learn the score awarded or the maximum possible score
for every test case and may instead be notified about them in batches.
In other news, the pointmap and groupweight configuration variables can
(now) be given as sequences in addition to mappings. (Technically,
the distinction currently made is dict versus everything else.) Items
of a sequence pointmap/groupweight correspond directly to the test cases/
groups defined in the tests configuration variable; in particular,
when groups are used, tests=[1],[2,3];pointmap={1:1,2:2,3:3} can now be
written as pointmap=tests=[1],[2,3]. Missing items are handled in the same
way in which they are handled when the variable is a mapping. Note
that the items of groupweight correspond to whole test groups rather
than individual test cases.
In other news again, the wording of problem total lines has been changed
from '<unweighted> points; weighted score: <weighted>' to '<weighted>
points (<unweighted> before weighting)', and group total lines now
properly report fractional numbers of points (this is a bug fix).
author | Oleg Oshmyan <chortos@inbox.lv> |
---|---|
date | Sat, 08 Jan 2011 16:03:35 +0200 |
parents | 007f7eb6fb2b |
children | 69eadc60f4e2 |
rev | line source |
---|---|
21 | 1 #! /usr/bin/env python |
16 | 2 # Copyright (c) 2010 Chortos-2 <chortos@inbox.lv> |
3 | |
21 | 4 from __future__ import division, with_statement |
5 | |
6 try: | |
7 from compat import * | |
8 import config, testcases | |
9 except ImportError: | |
10 import __main__ | |
11 __main__.import_error(sys.exc_info()[1]) | |
12 else: | |
22 | 13 from __main__ import clock, options |
21 | 14 |
22 | 15 import os, re, sys |
21 | 16 |
16 | 17 try: |
75
007f7eb6fb2b
The test context stack is now a deque.
Oleg Oshmyan <chortos@inbox.lv>
parents:
74
diff
changeset
|
18 from collections import deque |
007f7eb6fb2b
The test context stack is now a deque.
Oleg Oshmyan <chortos@inbox.lv>
parents:
74
diff
changeset
|
19 except ImportError: |
007f7eb6fb2b
The test context stack is now a deque.
Oleg Oshmyan <chortos@inbox.lv>
parents:
74
diff
changeset
|
20 deque = list |
007f7eb6fb2b
The test context stack is now a deque.
Oleg Oshmyan <chortos@inbox.lv>
parents:
74
diff
changeset
|
21 |
007f7eb6fb2b
The test context stack is now a deque.
Oleg Oshmyan <chortos@inbox.lv>
parents:
74
diff
changeset
|
22 try: |
21 | 23 import signal |
24 except ImportError: | |
25 signalnames = () | |
26 else: | |
27 # Construct a cache of all signal names available on the current | |
28 # platform. Prefer names from the UNIX standards over other versions. | |
29 unixnames = frozenset(('HUP', 'INT', 'QUIT', 'ILL', 'ABRT', 'FPE', 'KILL', 'SEGV', 'PIPE', 'ALRM', 'TERM', 'USR1', 'USR2', 'CHLD', 'CONT', 'STOP', 'TSTP', 'TTIN', 'TTOU', 'BUS', 'POLL', 'PROF', 'SYS', 'TRAP', 'URG', 'VTALRM', 'XCPU', 'XFSZ')) | |
30 signalnames = {} | |
31 for name in dir(signal): | |
32 if re.match('SIG[A-Z]+$', name): | |
33 value = signal.__dict__[name] | |
22 | 34 if isinstance(value, int) and (value not in signalnames or name[3:] in unixnames): |
21 | 35 signalnames[value] = name |
36 del unixnames | |
16 | 37 |
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
38 __all__ = 'Problem', 'TestContext', 'test_context_end', 'TestGroup' |
21 | 39 |
26 | 40 def strerror(e): |
41 s = getattr(e, 'strerror') | |
42 if not s: s = str(e) | |
43 return ' (%s%s)' % (s[0].lower(), s[1:]) if s else '' | |
21 | 44 |
45 class Cache(object): | |
46 def __init__(self, mydict): | |
47 self.__dict__ = mydict | |
16 | 48 |
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
49 class TestContext(object): |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
50 pass |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
51 |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
52 test_context_end = object() |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
53 |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
54 class TestGroup(TestContext): |
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
75
diff
changeset
|
55 __slots__ = 'points', 'case', 'log', 'correct', 'allcorrect', 'real', 'max', 'ntotal', 'nvalued', 'ncorrect', 'ncorrectvalued' |
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
56 |
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
75
diff
changeset
|
57 def __init__(self, points=None): |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
75
diff
changeset
|
58 self.points = points |
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
59 self.real = self.max = self.ntotal = self.nvalued = self.ncorrect = self.ncorrectvalued = 0 |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
60 self.allcorrect = True |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
61 self.log = [] |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
62 |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
63 def case_start(self, case): |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
64 self.case = case |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
65 self.correct = False |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
66 self.ntotal += 1 |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
67 if case.points: |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
68 self.nvalued += 1 |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
69 |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
70 def case_correct(self): |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
71 self.correct = True |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
72 self.ncorrect += 1 |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
73 if self.case.points: |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
74 self.ncorrectvalued += 1 |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
75 |
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
75
diff
changeset
|
76 def case_end(self): |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
75
diff
changeset
|
77 self.log.append((self.case, self.correct)) |
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
78 del self.case |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
79 if not self.correct: |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
80 self.allcorrect = False |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
81 |
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
75
diff
changeset
|
82 def score(self, real, max): |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
75
diff
changeset
|
83 self.real += real |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
75
diff
changeset
|
84 self.max += max |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
75
diff
changeset
|
85 |
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
86 def end(self): |
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
75
diff
changeset
|
87 if not self.allcorrect: |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
75
diff
changeset
|
88 self.real = 0 |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
75
diff
changeset
|
89 if self.points is not None and self.points != self.max: |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
75
diff
changeset
|
90 max, weighted = self.points, self.real * self.points / self.max if self.max else 0 |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
75
diff
changeset
|
91 before_weighting = ' (%g/%g before weighting)' % (self.real, self.max) |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
75
diff
changeset
|
92 else: |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
75
diff
changeset
|
93 max, weighted = self.max, self.real |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
75
diff
changeset
|
94 before_weighting = '' |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
75
diff
changeset
|
95 say('Group total: %d/%d tests, %g/%g points%s' % (self.ncorrect, self.ntotal, weighted, max, before_weighting)) |
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
96 # No real need to flush stdout, as it will anyway be flushed in a moment, |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
97 # when either the problem total or the next test case's ID is printed |
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
75
diff
changeset
|
98 return weighted, max, self.log |
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
99 |
16 | 100 class Problem(object): |
101 __slots__ = 'name', 'config', 'cache', 'testcases' | |
102 | |
103 def __init__(prob, name): | |
104 if not isinstance(name, basestring): | |
105 # This shouldn't happen, of course | |
21 | 106 raise TypeError('Problem() argument 1 must be string, not ' + type(name).__name__) |
16 | 107 prob.name = name |
21 | 108 prob.config = config.load_problem(name) |
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
109 prob.cache = Cache({'padoutput': 0}) |
21 | 110 prob.testcases = testcases.load_problem(prob) |
111 | |
112 # TODO | |
113 def build(prob): | |
114 raise NotImplementedError | |
16 | 115 |
116 def test(prob): | |
23 | 117 case = None |
22 | 118 try: |
75
007f7eb6fb2b
The test context stack is now a deque.
Oleg Oshmyan <chortos@inbox.lv>
parents:
74
diff
changeset
|
119 contexts = deque((TestGroup(),)) |
22 | 120 for case in prob.testcases: |
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
121 if case is test_context_end: |
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
75
diff
changeset
|
122 real, max, log = contexts.pop().end() |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
75
diff
changeset
|
123 for case, correct in log: |
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
124 contexts[-1].case_start(case) |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
125 if correct: |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
126 contexts[-1].case_correct() |
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
75
diff
changeset
|
127 contexts[-1].case_end() |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
75
diff
changeset
|
128 contexts[-1].score(real, max) |
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
129 continue |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
130 elif isinstance(case, TestContext): |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
131 contexts.append(case) |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
132 continue |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
133 contexts[-1].case_start(case) |
22 | 134 granted = 0 |
135 id = str(case.id) | |
136 if case.isdummy: | |
137 id = 'sample ' + id | |
138 say('%*s: ' % (prob.cache.padoutput, id), end='') | |
139 sys.stdout.flush() | |
140 try: | |
141 granted = case(lambda: (say('%7.3f%s s, ' % (case.time_stopped - case.time_started, case.time_limit_string), end=''), sys.stdout.flush())) | |
142 except testcases.CanceledByUser: | |
143 verdict = 'canceled by the user' | |
144 except testcases.TimeLimitExceeded: | |
145 verdict = 'time limit exceeded' | |
146 except testcases.WrongAnswer: | |
147 e = sys.exc_info()[1] | |
148 if e.comment: | |
149 verdict = 'wrong answer (%s)' % e.comment | |
150 else: | |
151 verdict = 'wrong answer' | |
152 except testcases.NonZeroExitCode: | |
153 e = sys.exc_info()[1] | |
154 if e.exitcode < 0: | |
155 if sys.platform == 'win32': | |
156 verdict = 'terminated with error 0x%X' % (e.exitcode + 0x100000000) | |
157 elif -e.exitcode in signalnames: | |
158 verdict = 'terminated by signal %d (%s)' % (-e.exitcode, signalnames[-e.exitcode]) | |
159 else: | |
160 verdict = 'terminated by signal %d' % -e.exitcode | |
21 | 161 else: |
22 | 162 verdict = 'non-zero return code %d' % e.exitcode |
163 except testcases.CannotStartTestee: | |
26 | 164 verdict = 'cannot launch the program to test%s' % strerror(sys.exc_info()[1].upstream) |
22 | 165 except testcases.CannotStartValidator: |
26 | 166 verdict = 'cannot launch the validator%s' % strerror(sys.exc_info()[1].upstream) |
22 | 167 except testcases.CannotReadOutputFile: |
26 | 168 verdict = 'cannot read the output file%s' % strerror(sys.exc_info()[1].upstream) |
22 | 169 except testcases.CannotReadInputFile: |
26 | 170 verdict = 'cannot read the input file%s' % strerror(sys.exc_info()[1].upstream) |
22 | 171 except testcases.CannotReadAnswerFile: |
26 | 172 verdict = 'cannot read the reference output file%s' % strerror(sys.exc_info()[1].upstream) |
22 | 173 except testcases.TestCaseNotPassed: |
26 | 174 verdict = 'unspecified reason [this may be a bug in test.py]%s' % strerror(sys.exc_info()[1]) |
22 | 175 #except Exception: |
26 | 176 # verdict = 'unknown error [this may be a bug in test.py]%s' % strerror(sys.exc_info()[1]) |
21 | 177 else: |
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
178 try: |
22 | 179 granted, comment = granted |
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
180 except TypeError: |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
181 comment = '' |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
182 else: |
22 | 183 if comment: |
184 comment = ' (%s)' % comment | |
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
185 if granted >= 1: |
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
186 contexts[-1].case_correct() |
22 | 187 verdict = 'OK' + comment |
188 elif not granted: | |
189 verdict = 'wrong answer' + comment | |
190 else: | |
191 verdict = 'partly correct' + comment | |
26 | 192 granted *= case.points |
22 | 193 say('%g/%g, %s' % (granted, case.points, verdict)) |
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
75
diff
changeset
|
194 contexts[-1].case_end() |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
75
diff
changeset
|
195 contexts[-1].score(granted, case.points) |
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
196 weighted = contexts[0].real * prob.config.taskweight / contexts[0].max if contexts[0].max else 0 |
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
75
diff
changeset
|
197 before_weighting = valued = '' |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
75
diff
changeset
|
198 if prob.config.taskweight != contexts[0].max: |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
75
diff
changeset
|
199 before_weighting = ' (%g/%g before weighting)' % (contexts[0].real, contexts[0].max) |
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
200 if contexts[0].nvalued != contexts[0].ntotal: |
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
75
diff
changeset
|
201 valued = ' (%d/%d valued)' % (contexts[0].ncorrectvalued, contexts[0].nvalued) |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
75
diff
changeset
|
202 say('Problem total: %d/%d tests%s, %g/%g points%s' % (contexts[0].ncorrect, contexts[0].ntotal, valued, weighted, prob.config.taskweight, before_weighting)) |
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
203 sys.stdout.flush() |
22 | 204 return weighted, prob.config.taskweight |
205 finally: | |
23 | 206 if options.erase and (not prob.config.stdio or case and case.validator): |
22 | 207 for var in 'in', 'out': |
208 name = getattr(prob.config, var + 'name') | |
209 if name: | |
210 try: | |
211 os.remove(name) | |
212 except Exception: | |
213 pass | |
214 if case.validator and not callable(case.validator): | |
215 if prob.config.ansname: | |
216 try: | |
217 os.remove(prob.config.ansname) | |
218 except Exception: | |
219 pass |