Mercurial > ~astiob > upreckon > hgweb

comparison testcases.py @ 50:4ea7133ac25c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Converted 2.00 into the default branch
author Oleg Oshmyan <chortos@inbox.lv>
date Sun, 19 Dec 2010 23:25:13 +0200
parents 2.00/testcases.py@81f58c938ec5
children 1914ae9cfdce
comparison
equal deleted inserted replaced
47:06f1683c8db9 50:4ea7133ac25c
1 #! /usr/bin/env python
2 # Copyright (c) 2010 Chortos-2 <chortos@inbox.lv>
3
4 # TODO: copy the ansfile if not options.erase even if no validator is used
5
6 from __future__ import division, with_statement
7
8 try:
9 from compat import *
10 import files, problem, config
11 except ImportError:
12 import __main__
13 __main__.import_error(sys.exc_info()[1])
14 else:
15 from __main__ import clock, options
16
17 import glob, re, sys, tempfile, time
18 from subprocess import Popen, PIPE, STDOUT
19
20 import os
21 devnull = open(os.path.devnull, 'w+')
22
23 try:
24 from signal import SIGTERM, SIGKILL
25 except ImportError:
26 SIGTERM = 15
27 SIGKILL = 9
28
29 try:
30 from _subprocess import TerminateProcess
31 except ImportError:
32 # CPython 2.5 does define _subprocess.TerminateProcess even though it is
33 # not used in the subprocess module, but maybe something else does not
34 try:
35 import ctypes
36 TerminateProcess = ctypes.windll.kernel32.TerminateProcess
37 except (ImportError, AttributeError):
38 TerminateProcess = None
39
40
41 # Do the hacky-wacky dark magic needed to catch presses of the Escape button.
42 # If only Python supported forcible termination of threads...
43 if not sys.stdin.isatty():
44 canceled = init_canceled = lambda: False
45 pause = None
46 else:
47 try:
48 # Windows has select() too, but it is not the select() we want
49 import msvcrt
50 except ImportError:
51 try:
52 import select, termios, tty, atexit
53 except ImportError:
54 # It cannot be helped!
55 # Silently disable support for killing the program being tested
56 canceled = init_canceled = lambda: False
57 pause = None
58 else:
59 def cleanup(old=termios.tcgetattr(sys.stdin.fileno())):
60 termios.tcsetattr(sys.stdin.fileno(), termios.TCSAFLUSH, old)
61 atexit.register(cleanup)
62 del cleanup
63 tty.setcbreak(sys.stdin.fileno())
64 def canceled(select=select.select, stdin=sys.stdin, read=sys.stdin.read):
65 while select((stdin,), (), (), 0)[0]:
66 if read(1) == '\33':
67 return True
68 return False
69 def init_canceled():
70 while select.select((sys.stdin,), (), (), 0)[0]:
71 sys.stdin.read(1)
72 def pause():
73 sys.stdin.read(1)
74 else:
75 def canceled(kbhit=msvcrt.kbhit, getch=msvcrt.getch):
76 while kbhit():
77 c = getch()
78 if c == '\33':
79 return True
80 elif c == '\0':
81 # Let's hope no-one is fiddling with this
82 getch()
83 return False
84 def init_canceled():
85 while msvcrt.kbhit():
86 msvcrt.getch()
87 def pause():
88 msvcrt.getch()
89
90
91 __all__ = ('TestCase', 'load_problem', 'TestCaseNotPassed',
92 'TimeLimitExceeded', 'CanceledByUser', 'WrongAnswer',
93 'NonZeroExitCode', 'CannotStartTestee',
94 'CannotStartValidator', 'CannotReadOutputFile',
95 'CannotReadInputFile', 'CannotReadAnswerFile')
96
97
98
99 # Exceptions
100
101 class TestCaseNotPassed(Exception): __slots__ = ()
102 class TimeLimitExceeded(TestCaseNotPassed): __slots__ = ()
103 class CanceledByUser(TestCaseNotPassed): __slots__ = ()
104
105 class WrongAnswer(TestCaseNotPassed):
106 __slots__ = 'comment'
107 def __init__(self, comment=''):
108 self.comment = comment
109
110 class NonZeroExitCode(TestCaseNotPassed):
111 __slots__ = 'exitcode'
112 def __init__(self, exitcode):
113 self.exitcode = exitcode
114
115 class ExceptionWrapper(TestCaseNotPassed):
116 __slots__ = 'upstream'
117 def __init__(self, upstream):
118 self.upstream = upstream
119
120 class CannotStartTestee(ExceptionWrapper): __slots__ = ()
121 class CannotStartValidator(ExceptionWrapper): __slots__ = ()
122 class CannotReadOutputFile(ExceptionWrapper): __slots__ = ()
123 class CannotReadInputFile(ExceptionWrapper): __slots__ = ()
124 class CannotReadAnswerFile(ExceptionWrapper): __slots__ = ()
125
126
127
128 # Helper context managers
129
130 class CopyDeleting(object):
131 __slots__ = 'case', 'file', 'name'
132
133 def __init__(self, case, file, name):
134 self.case = case
135 self.file = file
136 self.name = name
137
138 def __enter__(self):
139 if self.name:
140 try:
141 self.file.copy(self.name)
142 except:
143 try:
144 self.__exit__(None, None, None)
145 except:
146 pass
147 raise
148
149 def __exit__(self, exc_type, exc_val, exc_tb):
150 if self.name:
151 self.case.files_to_delete.append(self.name)
152
153
154 class Copying(object):
155 __slots__ = 'file', 'name'
156
157 def __init__(self, file, name):
158 self.file = file
159 self.name = name
160
161 def __enter__(self):
162 if self.name:
163 self.file.copy(self.name)
164
165 def __exit__(self, exc_type, exc_val, exc_tb):
166 pass
167
168
169
170 # Test case types
171
172 class TestCase(object):
173 __slots__ = ('problem', 'id', 'isdummy', 'infile', 'outfile', 'points',
174 'process', 'time_started', 'time_stopped', 'time_limit_string',
175 'realinname', 'realoutname', 'maxtime', 'maxmemory',
176 'has_called_back', 'files_to_delete')
177
178 if ABCMeta:
179 __metaclass__ = ABCMeta
180
181 def __init__(case, prob, id, isdummy, points):
182 case.problem = prob
183 case.id = id
184 case.isdummy = isdummy
185 case.points = points
186 case.maxtime = case.problem.config.maxtime
187 case.maxmemory = case.problem.config.maxmemory
188 if case.maxtime:
189 case.time_limit_string = '/%.3f' % case.maxtime
190 else:
191 case.time_limit_string = ''
192 if not isdummy:
193 case.realinname = case.problem.config.testcaseinname
194 case.realoutname = case.problem.config.testcaseoutname
195 else:
196 case.realinname = case.problem.config.dummyinname
197 case.realoutname = case.problem.config.dummyoutname
198
199 @abstractmethod
200 def test(case): raise NotImplementedError
201
202 def __call__(case, callback):
203 case.has_called_back = False
204 case.files_to_delete = []
205 try:
206 return case.test(callback)
207 finally:
208 now = clock()
209 if not getattr(case, 'time_started', None):
210 case.time_started = case.time_stopped = now
211 elif not getattr(case, 'time_stopped', None):
212 case.time_stopped = now
213 if not case.has_called_back:
214 callback()
215 case.cleanup()
216
217 def cleanup(case):
218 #if getattr(case, 'infile', None):
219 # case.infile.close()
220 #if getattr(case, 'outfile', None):
221 # case.outfile.close()
222 if getattr(case, 'process', None):
223 # Try killing after three unsuccessful TERM attempts in a row
224 # (except on Windows, where TERMing is killing)
225 for i in range(3):
226 try:
227 try:
228 case.process.terminate()
229 except AttributeError:
230 # Python 2.5
231 if TerminateProcess and hasattr(proc, '_handle'):
232 # Windows API
233 TerminateProcess(proc._handle, 1)
234 else:
235 # POSIX
236 os.kill(proc.pid, SIGTERM)
237 except Exception:
238 time.sleep(0)
239 case.process.poll()
240 else:
241 case.process.wait()
242 break
243 else:
244 # If killing the process is unsuccessful three times in a row,
245 # just silently stop trying
246 for i in range(3):
247 try:
248 try:
249 case.process.kill()
250 except AttributeError:
251 # Python 2.5
252 if TerminateProcess and hasattr(proc, '_handle'):
253 # Windows API
254 TerminateProcess(proc._handle, 1)
255 else:
256 # POSIX
257 os.kill(proc.pid, SIGKILL)
258 except Exception:
259 time.sleep(0)
260 case.process.poll()
261 else:
262 case.process.wait()
263 break
264 if case.files_to_delete:
265 for name in case.files_to_delete:
266 try:
267 os.remove(name)
268 except Exception:
269 # It can't be helped
270 pass
271
272 def open_infile(case):
273 try:
274 case.infile = files.File('/'.join((case.problem.name, case.realinname.replace('$', case.id))))
275 except IOError:
276 e = sys.exc_info()[1]
277 raise CannotReadInputFile(e)
278
279 def open_outfile(case):
280 try:
281 case.outfile = files.File('/'.join((case.problem.name, case.realoutname.replace('$', case.id))))
282 except IOError:
283 e = sys.exc_info()[1]
284 raise CannotReadAnswerFile(e)
285
286
287 class ValidatedTestCase(TestCase):
288 __slots__ = 'validator'
289
290 def __init__(case, *args):
291 TestCase.__init__(case, *args)
292 if not case.problem.config.tester:
293 case.validator = None
294 else:
295 case.validator = case.problem.config.tester
296
297 def validate(case, output):
298 if not case.validator:
299 # Compare the output with the reference output
300 case.open_outfile()
301 with case.outfile.open() as refoutput:
302 for line, refline in zip_longest(output, refoutput):
303 if refline is not None and not isinstance(refline, basestring):
304 line = bytes(line, sys.getdefaultencoding())
305 if line != refline:
306 raise WrongAnswer
307 return 1
308 elif callable(case.validator):
309 return case.validator(output)
310 else:
311 # Call the validator program
312 output.close()
313 if case.problem.config.ansname:
314 case.open_outfile()
315 case.outfile.copy(case.problem.config.ansname)
316 try:
317 case.process = Popen(case.validator, stdin=devnull, stdout=PIPE, stderr=STDOUT, universal_newlines=True, bufsize=-1)
318 except OSError:
319 raise CannotStartValidator(sys.exc_info()[1])
320 comment = case.process.communicate()[0].strip()
321 match = re.match(r'(?i)(ok|(?:correct|wrong)(?:(?:\s|_)*answer)?)(?:$|\s+|[.,!:]+\s*)', comment)
322 if match:
323 comment = comment[match.end():]
324 if not case.problem.config.maxexitcode:
325 if case.process.returncode:
326 raise WrongAnswer(comment)
327 else:
328 return 1, comment
329 else:
330 return case.process.returncode / case.problem.config.maxexitcode, comment
331
332
333 class BatchTestCase(ValidatedTestCase):
334 __slots__ = ()
335
336 def test(case, callback):
337 init_canceled()
338 if sys.platform == 'win32' or not case.maxmemory:
339 preexec_fn = None
340 else:
341 def preexec_fn():
342 try:
343 import resource
344 maxmemory = int(case.maxmemory * 1048576)
345 resource.setrlimit(resource.RLIMIT_AS, (maxmemory, maxmemory))
346 # I would also set a CPU time limit but I do not want the time
347 # that passes between the calls to fork and exec to be counted in
348 except MemoryError:
349 # We do not have enough memory for ourselves;
350 # let the parent know about this
351 raise
352 except Exception:
353 # Well, at least we tried
354 pass
355 case.open_infile()
356 case.time_started = None
357 if case.problem.config.stdio:
358 if options.erase and not case.validator and case.problem.config.inname:
359 # TODO: re-use the same file name if possible
360 # FIXME: 2.5 lacks the delete parameter
361 with tempfile.NamedTemporaryFile(delete=False) as f:
362 inputdatafname = f.name
363 contextmgr = CopyDeleting(case, case.infile, inputdatafname)
364 else:
365 inputdatafname = case.problem.config.inname
366 contextmgr = Copying(case.infile, inputdatafname)
367 with contextmgr:
368 # FIXME: this U doesn't do anything good for the child process, does it?
369 with open(inputdatafname, 'rU') as infile:
370 with tempfile.TemporaryFile('w+') if options.erase and not case.validator else open(case.problem.config.outname, 'w+') as outfile:
371 try:
372 try:
373 case.process = Popen(case.problem.config.path, stdin=infile, stdout=outfile, stderr=devnull, universal_newlines=True, bufsize=-1, preexec_fn=preexec_fn)
374 except MemoryError:
375 # If there is not enough memory for the forked test.py,
376 # opt for silent dropping of the limit
377 # TODO: show a warning somewhere
378 case.process = Popen(case.problem.config.path, stdin=infile, stdout=outfile, stderr=devnull, universal_newlines=True, bufsize=-1)
379 except OSError:
380 raise CannotStartTestee(sys.exc_info()[1])
381 case.time_started = clock()
382 time_next_check = case.time_started + .15
383 if not case.maxtime:
384 while True:
385 exitcode, now = case.process.poll(), clock()
386 if exitcode is not None:
387 case.time_stopped = now
388 break
389 # For some reason (probably Microsoft's fault),
390 # msvcrt.kbhit() is slow as hell
391 else:
392 if now >= time_next_check:
393 if canceled():
394 raise CanceledByUser
395 else:
396 time_next_check = now + .15
397 time.sleep(0)
398 else:
399 time_end = case.time_started + case.maxtime
400 while True:
401 exitcode, now = case.process.poll(), clock()
402 if exitcode is not None:
403 case.time_stopped = now
404 break
405 elif now >= time_end:
406 raise TimeLimitExceeded
407 else:
408 if now >= time_next_check:
409 if canceled():
410 raise CanceledByUser
411 else:
412 time_next_check = now + .15
413 time.sleep(0)
414 if config.globalconf.force_zero_exitcode and case.process.returncode:
415 raise NonZeroExitCode(case.process.returncode)
416 callback()
417 case.has_called_back = True
418 outfile.seek(0)
419 return case.validate(outfile)
420 else:
421 case.infile.copy(case.problem.config.inname)
422 try:
423 try:
424 case.process = Popen(case.problem.config.path, stdin=devnull, stdout=devnull, stderr=STDOUT, preexec_fn=preexec_fn)
425 except MemoryError:
426 # If there is not enough memory for the forked test.py,
427 # opt for silent dropping of the limit
428 # TODO: show a warning somewhere
429 case.process = Popen(case.problem.config.path, stdin=devnull, stdout=devnull, stderr=STDOUT)
430 except OSError:
431 raise CannotStartTestee(sys.exc_info()[1])
432 case.time_started = clock()
433 time_next_check = case.time_started + .15
434 if not case.maxtime:
435 while True:
436 exitcode, now = case.process.poll(), clock()
437 if exitcode is not None:
438 case.time_stopped = now
439 break
440 else:
441 if now >= time_next_check:
442 if canceled():
443 raise CanceledByUser
444 else:
445 time_next_check = now + .15
446 time.sleep(0)
447 else:
448 time_end = case.time_started + case.maxtime
449 while True:
450 exitcode, now = case.process.poll(), clock()
451 if exitcode is not None:
452 case.time_stopped = now
453 break
454 elif now >= time_end:
455 raise TimeLimitExceeded
456 else:
457 if now >= time_next_check:
458 if canceled():
459 raise CanceledByUser
460 else:
461 time_next_check = now + .15
462 time.sleep(0)
463 if config.globalconf.force_zero_exitcode and case.process.returncode:
464 raise NonZeroExitCode(case.process.returncode)
465 callback()
466 case.has_called_back = True
467 with open(case.problem.config.outname, 'rU') as output:
468 return case.validate(output)
469
470
471 # This is the only test case type not executing any programs to be tested
472 class OutputOnlyTestCase(ValidatedTestCase):
473 __slots__ = ()
474 def cleanup(case): pass
475
476 class BestOutputTestCase(ValidatedTestCase):
477 __slots__ = ()
478
479 # This is the only test case type executing two programs simultaneously
480 class ReactiveTestCase(TestCase):
481 __slots__ = ()
482 # The basic idea is to launch the program to be tested and the grader
483 # and to pipe their standard I/O from and to each other,
484 # and then to capture the grader's exit code and use it
485 # like the exit code of an output validator is used.
486
487
488 def load_problem(prob, _types={'batch' : BatchTestCase,
489 'outonly' : OutputOnlyTestCase,
490 'bestout' : BestOutputTestCase,
491 'reactive': ReactiveTestCase}):
492 # We will need to iterate over these configuration variables twice
493 try:
494 len(prob.config.dummies)
495 except Exception:
496 prob.config.dummies = tuple(prob.config.dummies)
497 try:
498 len(prob.config.tests)
499 except Exception:
500 prob.config.tests = tuple(prob.config.tests)
501
502 if options.legacy:
503 prob.config.usegroups = False
504 prob.config.tests = list(prob.config.tests)
505 for i, name in enumerate(prob.config.tests):
506 # Same here; we'll need to iterate over them twice
507 try:
508 l = len(name)
509 except Exception:
510 try:
511 name = tuple(name)
512 except TypeError:
513 name = (name,)
514 l = len(name)
515 if len(name) > 1:
516 prob.config.usegroups = True
517 break
518 elif not len(name):
519 prob.config.tests[i] = (name,)
520
521 # First get prob.cache.padoutput right,
522 # then yield the actual test cases
523 for i in prob.config.dummies:
524 s = 'sample ' + str(i).zfill(prob.config.paddummies)
525 prob.cache.padoutput = max(prob.cache.padoutput, len(s))
526 if prob.config.usegroups:
527 for group in prob.config.tests:
528 for i in group:
529 s = str(i).zfill(prob.config.padtests)
530 prob.cache.padoutput = max(prob.cache.padoutput, len(s))
531 for i in prob.config.dummies:
532 s = str(i).zfill(prob.config.paddummies)
533 yield _types[prob.config.kind](prob, s, True, 0)
534 for group in prob.config.tests:
535 yield problem.TestGroup()
536 for i in group:
537 s = str(i).zfill(prob.config.padtests)
538 yield _types[prob.config.kind](prob, s, False, prob.config.pointmap.get(i, prob.config.pointmap.get(None, prob.config.maxexitcode if prob.config.maxexitcode else 1)))
539 yield problem.test_context_end
540 else:
541 for i in prob.config.tests:
542 s = str(i).zfill(prob.config.padtests)
543 prob.cache.padoutput = max(prob.cache.padoutput, len(s))
544 for i in prob.config.dummies:
545 s = str(i).zfill(prob.config.paddummies)
546 yield _types[prob.config.kind](prob, s, True, 0)
547 for i in prob.config.tests:
548 s = str(i).zfill(prob.config.padtests)
549 yield _types[prob.config.kind](prob, s, False, prob.config.pointmap.get(i, prob.config.pointmap.get(None, prob.config.maxexitcode if prob.config.maxexitcode else 1)))