Mercurial > ~astiob > upreckon > hgweb
annotate 2.00/testcases.py @ 24:c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
TestCase.__call__() now returns the fraction (a number from 0 to 1) of case.points that is to be awarded.
Bug fix: %-patterns in configuration variables common to all problems are now substituted.
author | Oleg Oshmyan <chortos@inbox.lv> |
---|---|
date | Thu, 23 Sep 2010 00:11:24 +0000 |
parents | c1f52b5d80d6 |
children | b500e117080e |
rev | line source |
---|---|
21 | 1 #! /usr/bin/env python |
16 | 2 # Copyright (c) 2010 Chortos-2 <chortos@inbox.lv> |
3 | |
21 | 4 from __future__ import division, with_statement |
5 | |
6 try: | |
7 from compat import * | |
8 import files, problem, config | |
9 except ImportError: | |
10 import __main__ | |
11 __main__.import_error(sys.exc_info()[1]) | |
12 else: | |
13 from __main__ import clock, options | |
14 | |
15 import glob, re, sys, tempfile, time | |
16 from subprocess import Popen, PIPE, STDOUT | |
17 | |
18 import os | |
19 devnull = open(os.path.devnull, 'w+') | |
20 | |
21 try: | |
22 from signal import SIGTERM, SIGKILL | |
23 except ImportError: | |
24 SIGTERM = 15 | |
25 SIGKILL = 9 | |
26 | |
16 | 27 try: |
21 | 28 from _subprocess import TerminateProcess |
29 except ImportError: | |
30 # CPython 2.5 does define _subprocess.TerminateProcess even though it is | |
31 # not used in the subprocess module, but maybe something else does not | |
32 try: | |
33 import ctypes | |
34 TerminateProcess = ctypes.windll.kernel32.TerminateProcess | |
35 except (ImportError, AttributeError): | |
36 TerminateProcess = None | |
37 | |
22 | 38 |
39 # Do the hacky-wacky dark magic needed to catch presses of the Escape button. | |
40 # If only Python supported forcible termination of threads... | |
41 if not sys.stdin.isatty(): | |
42 canceled = init_canceled = lambda: False | |
43 pause = None | |
44 else: | |
45 try: | |
46 # Windows has select() too, but it is not the select() we want | |
47 import msvcrt | |
48 except ImportError: | |
49 try: | |
50 import select, termios, tty, atexit | |
51 except ImportError: | |
52 # It cannot be helped! | |
53 # Silently disable support for killing the program being tested | |
54 canceled = init_canceled = lambda: False | |
55 pause = None | |
56 else: | |
57 def cleanup(old=termios.tcgetattr(sys.stdin.fileno())): | |
58 termios.tcsetattr(sys.stdin.fileno(), termios.TCSAFLUSH, old) | |
59 atexit.register(cleanup) | |
60 del cleanup | |
61 tty.setcbreak(sys.stdin.fileno()) | |
62 def canceled(): | |
63 while select.select((sys.stdin,), (), (), 0)[0]: | |
64 if sys.stdin.read(1) == '\33': | |
65 return True | |
66 return False | |
67 def init_canceled(): | |
68 while select.select((sys.stdin,), (), (), 0)[0]: | |
69 sys.stdin.read(1) | |
70 def pause(): | |
71 sys.stdin.read(1) | |
72 else: | |
73 def canceled(): | |
74 while msvcrt.kbhit(): | |
75 c = msvcrt.getch() | |
76 if c == '\33': | |
77 return True | |
78 elif c == '\0': | |
79 # Let's hope no-one is fiddling with this | |
80 msvcrt.getch() | |
81 return False | |
82 def init_canceled(): | |
83 while msvcrt.kbhit(): | |
84 msvcrt.getch() | |
85 def pause(): | |
86 msvcrt.getch() | |
87 | |
88 | |
21 | 89 __all__ = ('TestCase', 'load_problem', 'TestCaseNotPassed', |
22 | 90 'TimeLimitExceeded', 'CanceledByUser', 'WrongAnswer', |
91 'NonZeroExitCode', 'CannotStartTestee', | |
92 'CannotStartValidator', 'CannotReadOutputFile', | |
93 'CannotReadInputFile', 'CannotReadAnswerFile') | |
21 | 94 |
95 | |
96 | |
97 # Exceptions | |
98 | |
99 class TestCaseNotPassed(Exception): __slots__ = () | |
100 class TimeLimitExceeded(TestCaseNotPassed): __slots__ = () | |
22 | 101 class CanceledByUser(TestCaseNotPassed): __slots__ = () |
21 | 102 |
103 class WrongAnswer(TestCaseNotPassed): | |
104 __slots__ = 'comment' | |
105 def __init__(self, comment=''): | |
106 self.comment = comment | |
107 | |
108 class NonZeroExitCode(TestCaseNotPassed): | |
109 __slots__ = 'exitcode' | |
110 def __init__(self, exitcode): | |
111 self.exitcode = exitcode | |
112 | |
113 class ExceptionWrapper(TestCaseNotPassed): | |
114 __slots__ = 'upstream' | |
115 def __init__(self, upstream): | |
116 self.upstream = upstream | |
117 | |
118 class CannotStartTestee(ExceptionWrapper): __slots__ = () | |
119 class CannotStartValidator(ExceptionWrapper): __slots__ = () | |
120 class CannotReadOutputFile(ExceptionWrapper): __slots__ = () | |
121 class CannotReadInputFile(ExceptionWrapper): __slots__ = () | |
122 class CannotReadAnswerFile(ExceptionWrapper): __slots__ = () | |
123 | |
124 | |
125 | |
22 | 126 # Helper context managers |
127 | |
128 class CopyDeleting(object): | |
129 __slots__ = 'case', 'file', 'name' | |
130 | |
131 def __init__(self, case, file, name): | |
132 self.case = case | |
133 self.file = file | |
134 self.name = name | |
135 | |
136 def __enter__(self): | |
137 if self.name: | |
138 try: | |
139 self.file.copy(self.name) | |
140 except: | |
141 try: | |
142 self.__exit__(None, None, None) | |
143 except: | |
144 pass | |
145 raise | |
146 | |
147 def __exit__(self, exc_type, exc_val, exc_tb): | |
148 if self.name: | |
149 self.case.files_to_delete.append(self.name) | |
150 | |
151 | |
152 class Copying(object): | |
153 __slots__ = 'file', 'name' | |
154 | |
155 def __init__(self, file, name): | |
156 self.file = file | |
157 self.name = name | |
158 | |
159 def __enter__(self): | |
160 if self.name: | |
161 self.file.copy(self.name) | |
162 | |
163 def __exit__(self, exc_type, exc_val, exc_tb): | |
164 pass | |
165 | |
166 | |
167 | |
21 | 168 # Test case types |
16 | 169 |
170 class TestCase(object): | |
21 | 171 __slots__ = ('problem', 'id', 'isdummy', 'infile', 'outfile', 'points', |
172 'process', 'time_started', 'time_stopped', 'time_limit_string', | |
22 | 173 'realinname', 'realoutname', 'maxtime', 'maxmemory', |
174 'has_called_back', 'files_to_delete') | |
21 | 175 |
176 if ABCMeta: | |
177 __metaclass__ = ABCMeta | |
16 | 178 |
21 | 179 def __init__(case, prob, id, isdummy, points): |
16 | 180 case.problem = prob |
21 | 181 case.id = id |
182 case.isdummy = isdummy | |
183 case.points = points | |
184 case.maxtime = case.problem.config.maxtime | |
185 case.maxmemory = case.problem.config.maxmemory | |
186 if case.maxtime: | |
187 case.time_limit_string = '/%.3f' % case.maxtime | |
188 else: | |
189 case.time_limit_string = '' | |
190 if not isdummy: | |
191 case.realinname = case.problem.config.testcaseinname | |
192 case.realoutname = case.problem.config.testcaseoutname | |
193 else: | |
194 case.realinname = case.problem.config.dummyinname | |
195 case.realoutname = case.problem.config.dummyoutname | |
196 | |
197 @abstractmethod | |
198 def test(case): raise NotImplementedError | |
16 | 199 |
22 | 200 def __call__(case, callback): |
201 case.has_called_back = False | |
202 case.files_to_delete = [] | |
21 | 203 try: |
22 | 204 return case.test(callback) |
21 | 205 finally: |
22 | 206 now = clock() |
207 if not getattr(case, 'time_started', None): | |
208 case.time_started = case.time_stopped = now | |
209 elif not getattr(case, 'time_stopped', None): | |
210 case.time_stopped = now | |
211 if not case.has_called_back: | |
212 callback() | |
21 | 213 case.cleanup() |
214 | |
215 def cleanup(case): | |
216 #if getattr(case, 'infile', None): | |
217 # case.infile.close() | |
218 #if getattr(case, 'outfile', None): | |
219 # case.outfile.close() | |
220 if getattr(case, 'process', None): | |
221 # Try killing after three unsuccessful TERM attempts in a row | |
222 # (except on Windows, where TERMing is killing) | |
223 for i in range(3): | |
224 try: | |
225 try: | |
226 case.process.terminate() | |
227 except AttributeError: | |
228 # Python 2.5 | |
229 if TerminateProcess and hasattr(proc, '_handle'): | |
230 # Windows API | |
231 TerminateProcess(proc._handle, 1) | |
232 else: | |
233 # POSIX | |
234 os.kill(proc.pid, SIGTERM) | |
235 except Exception: | |
236 time.sleep(0) | |
237 case.process.poll() | |
238 else: | |
22 | 239 case.process.wait() |
21 | 240 break |
241 else: | |
242 # If killing the process is unsuccessful three times in a row, | |
243 # just silently stop trying | |
244 for i in range(3): | |
245 try: | |
246 try: | |
247 case.process.kill() | |
248 except AttributeError: | |
249 # Python 2.5 | |
250 if TerminateProcess and hasattr(proc, '_handle'): | |
251 # Windows API | |
252 TerminateProcess(proc._handle, 1) | |
253 else: | |
254 # POSIX | |
255 os.kill(proc.pid, SIGKILL) | |
256 except Exception: | |
257 time.sleep(0) | |
258 case.process.poll() | |
259 else: | |
22 | 260 case.process.wait() |
21 | 261 break |
22 | 262 if case.files_to_delete: |
263 for name in case.files_to_delete: | |
264 try: | |
265 os.remove(name) | |
266 except Exception: | |
267 # It can't be helped | |
268 pass | |
21 | 269 |
270 def open_infile(case): | |
271 try: | |
272 case.infile = files.File('/'.join((case.problem.name, case.realinname.replace('$', case.id)))) | |
273 except IOError: | |
274 e = sys.exc_info()[1] | |
275 raise CannotReadInputFile(e) | |
276 | |
277 def open_outfile(case): | |
278 try: | |
279 case.outfile = files.File('/'.join((case.problem.name, case.realoutname.replace('$', case.id)))) | |
280 except IOError: | |
281 e = sys.exc_info()[1] | |
282 raise CannotReadAnswerFile(e) | |
283 | |
16 | 284 |
21 | 285 class ValidatedTestCase(TestCase): |
286 __slots__ = 'validator' | |
287 | |
288 def __init__(case, *args): | |
289 TestCase.__init__(case, *args) | |
290 if not case.problem.config.tester: | |
291 case.validator = None | |
292 else: | |
293 case.validator = case.problem.config.tester | |
294 | |
295 # TODO | |
296 def validate(case, output): | |
297 if not case.validator: | |
298 # Compare the output with the reference output | |
299 case.open_outfile() | |
300 with case.outfile.open() as refoutput: | |
301 for line, refline in zip(output, refoutput): | |
302 if not isinstance(refline, basestring): | |
303 line = bytes(line, sys.getdefaultencoding()) | |
304 if line != refline: | |
22 | 305 raise WrongAnswer |
21 | 306 try: |
307 try: | |
308 next(output) | |
309 except NameError: | |
310 output.next() | |
311 except StopIteration: | |
312 pass | |
313 else: | |
22 | 314 raise WrongAnswer |
21 | 315 try: |
316 try: | |
317 next(refoutput) | |
318 except NameError: | |
319 refoutput.next() | |
320 except StopIteration: | |
321 pass | |
322 else: | |
22 | 323 raise WrongAnswer |
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
324 return 1 |
21 | 325 elif callable(case.validator): |
326 return case.validator(output) | |
327 else: | |
328 # Call the validator program | |
329 output.close() | |
23 | 330 if case.problem.config.ansname: |
331 case.open_outfile() | |
332 case.outfile.copy(case.problem.config.ansname) | |
21 | 333 case.process = Popen(case.validator, stdin=devnull, stdout=PIPE, stderr=STDOUT, universal_newlines=True, bufsize=-1) |
334 comment = case.process.communicate()[0].strip() | |
335 lower = comment.lower() | |
336 match = re.match(r'(ok|correct|wrong(?:(?:\s|_)*answer)?)(?:$|\s+|[.,!:]+\s*)', lower) | |
337 if match: | |
338 comment = comment[match.end():] | |
339 if not case.problem.config.maxexitcode: | |
340 if case.process.returncode: | |
341 raise WrongAnswer(comment) | |
342 else: | |
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
343 return 1, comment |
21 | 344 else: |
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
345 return case.process.returncode / case.problem.config.maxexitcode, comment |
21 | 346 |
347 | |
348 class BatchTestCase(ValidatedTestCase): | |
349 __slots__ = () | |
350 | |
22 | 351 def test(case, callback): |
352 init_canceled() | |
21 | 353 if sys.platform == 'win32' or not case.maxmemory: |
354 preexec_fn = None | |
355 else: | |
356 def preexec_fn(): | |
357 try: | |
358 import resource | |
359 maxmemory = int(case.maxmemory * 1048576) | |
360 resource.setrlimit(resource.RLIMIT_AS, (maxmemory, maxmemory)) | |
361 # I would also set a CPU time limit but I do not want the time | |
362 # that passes between the calls to fork and exec to be counted in | |
363 except MemoryError: | |
364 # We do not have enough memory for ourselves; | |
365 # let the parent know about this | |
366 raise | |
367 except Exception: | |
368 # Well, at least we tried | |
369 pass | |
370 case.open_infile() | |
371 case.time_started = None | |
372 if case.problem.config.stdio: | |
373 if options.erase and not case.validator: | |
22 | 374 # TODO: re-use the same file name if possible |
21 | 375 # FIXME: 2.5 lacks the delete parameter |
376 with tempfile.NamedTemporaryFile(delete=False) as f: | |
22 | 377 inputdatafname = f.name |
378 context = CopyDeleting(case, case.infile, inputdatafname) | |
21 | 379 else: |
380 inputdatafname = case.problem.config.inname | |
22 | 381 context = Copying(case.infile, inputdatafname) |
382 with context: | |
383 with open(inputdatafname, 'rU') as infile: | |
384 with tempfile.TemporaryFile('w+') if options.erase and not case.validator else open(case.problem.config.outname, 'w+') as outfile: | |
23 | 385 # TODO: make sure outfile.file is passed to Popen if needed |
21 | 386 try: |
22 | 387 try: |
388 case.process = Popen(case.problem.config.path, stdin=infile, stdout=outfile, stderr=devnull, universal_newlines=True, bufsize=-1, preexec_fn=preexec_fn) | |
389 except MemoryError: | |
390 # If there is not enough memory for the forked test.py, | |
391 # opt for silent dropping of the limit | |
392 case.process = Popen(case.problem.config.path, stdin=infile, stdout=outfile, stderr=devnull, universal_newlines=True, bufsize=-1) | |
393 except OSError: | |
394 raise CannotStartTestee(sys.exc_info()[1]) | |
395 case.time_started = clock() | |
396 if not case.maxtime: | |
397 while True: | |
398 exitcode, now = case.process.poll(), clock() | |
399 if exitcode is not None: | |
400 case.time_stopped = now | |
401 break | |
402 elif canceled(): | |
403 raise CanceledByUser | |
404 else: | |
405 time_end = case.time_started + case.maxtime | |
406 while True: | |
407 exitcode, now = case.process.poll(), clock() | |
408 if exitcode is not None: | |
409 case.time_stopped = now | |
410 break | |
411 elif now >= time_end: | |
412 raise TimeLimitExceeded | |
413 elif canceled(): | |
414 raise CanceledByUser | |
415 if config.globalconf.force_zero_exitcode and case.process.returncode: | |
416 raise NonZeroExitCode(case.process.returncode) | |
417 callback() | |
418 case.has_called_back = True | |
419 outfile.seek(0) | |
420 return case.validate(outfile) | |
21 | 421 else: |
22 | 422 case.infile.copy(case.problem.config.inname) |
21 | 423 try: |
424 try: | |
425 case.process = Popen(case.problem.config.path, stdin=devnull, stdout=devnull, stderr=STDOUT, preexec_fn=preexec_fn) | |
426 except MemoryError: | |
427 # If there is not enough memory for the forked test.py, | |
428 # opt for silent dropping of the limit | |
429 case.process = Popen(case.problem.config.path, stdin=devnull, stdout=devnull, stderr=STDOUT) | |
430 except OSError: | |
431 raise CannotStartTestee(sys.exc_info()[1]) | |
432 case.time_started = clock() | |
433 if not case.maxtime: | |
22 | 434 while True: |
435 exitcode, now = case.process.poll(), clock() | |
436 if exitcode is not None: | |
437 case.time_stopped = now | |
438 break | |
439 elif canceled(): | |
440 raise CanceledByUser | |
21 | 441 else: |
442 time_end = case.time_started + case.maxtime | |
443 while True: | |
22 | 444 exitcode, now = case.process.poll(), clock() |
21 | 445 if exitcode is not None: |
446 case.time_stopped = now | |
447 break | |
448 elif now >= time_end: | |
22 | 449 raise TimeLimitExceeded |
450 elif canceled(): | |
451 raise CanceledByUser | |
21 | 452 if config.globalconf.force_zero_exitcode and case.process.returncode: |
453 raise NonZeroExitCode(case.process.returncode) | |
22 | 454 callback() |
455 case.has_called_back = True | |
21 | 456 with open(case.problem.config.outname, 'rU') as output: |
457 return case.validate(output) | |
458 | |
459 | |
460 # This is the only test case type not executing any programs to be tested | |
461 class OutputOnlyTestCase(ValidatedTestCase): | |
462 __slots__ = () | |
463 def cleanup(case): pass | |
464 | |
465 class BestOutputTestCase(ValidatedTestCase): | |
466 __slots__ = () | |
467 | |
468 # This is the only test case type executing two programs simultaneously | |
469 class ReactiveTestCase(TestCase): | |
470 __slots__ = () | |
471 # The basic idea is to launch the program to be tested and the grader | |
472 # and to pipe their standard I/O from and to each other, | |
473 # and then to capture the grader's exit code and use it | |
474 # like the exit code of a test validator is used. | |
475 | |
476 | |
477 def load_problem(prob, _types={'batch' : BatchTestCase, | |
478 'outonly' : OutputOnlyTestCase, | |
479 'bestout' : BestOutputTestCase, | |
480 'reactive': ReactiveTestCase}): | |
23 | 481 if options.legacy: |
482 prob.config.usegroups = False | |
483 prob.config.tests = list(prob.config.tests) | |
484 for i, name in enumerate(prob.config.tests): | |
485 try: | |
486 if len(name) > 1: | |
487 prob.config.usegroups = True | |
488 break | |
489 elif len(name): | |
490 prob.config.tests[i] = name[0] | |
491 except Exception: | |
492 try: | |
493 # Try to retrieve the first two test case ID's and cache them on success | |
494 prob.config.tests[i] = name = iter(name) | |
495 except TypeError: | |
496 continue | |
497 try: | |
498 try: | |
499 first = next(name) | |
500 except NameError: | |
501 # Python 2.5 lacks the next() built-in | |
502 first = name.next() | |
503 except StopIteration: | |
504 prob.config.tests[i] = () | |
505 else: | |
506 try: | |
507 try: | |
508 second = next(name) | |
509 except NameError: | |
510 second = name.next() | |
511 except StopIteration: | |
512 prob.config.tests[i] = first | |
513 else: | |
514 prob.config.tests[i] = itertools.chain((first, second), name) | |
515 prob.config.usegroups = True | |
516 break | |
16 | 517 if prob.config.usegroups: |
22 | 518 # FIXME: test groups should again be supported! |
16 | 519 pass |
520 else: | |
21 | 521 # We will need to iterate over these configuration variables twice |
522 try: | |
523 len(prob.config.dummies) | |
524 except Exception: | |
525 prob.config.dummies = tuple(prob.config.dummies) | |
526 try: | |
527 len(prob.config.tests) | |
528 except Exception: | |
23 | 529 prob.config.tests = tuple(prob.config.tests) |
21 | 530 # First get prob.cache.padoutput right |
531 for i in prob.config.dummies: | |
532 s = 'sample ' + str(i).zfill(prob.config.paddummies) | |
533 prob.cache.padoutput = max(prob.cache.padoutput, len(s)) | |
16 | 534 for i in prob.config.tests: |
21 | 535 s = str(i).zfill(prob.config.padtests) |
536 prob.cache.padoutput = max(prob.cache.padoutput, len(s)) | |
537 # Now yield the actual test cases | |
538 for i in prob.config.dummies: | |
539 s = str(i).zfill(prob.config.paddummies) | |
540 yield _types[prob.config.kind](prob, s, True, 0) | |
541 for i in prob.config.tests: | |
542 s = str(i).zfill(prob.config.padtests) | |
543 yield _types[prob.config.kind](prob, s, False, prob.config.pointmap.get(i, prob.config.pointmap.get(None, prob.config.maxexitcode if prob.config.maxexitcode else 1))) |