Mercurial > ~astiob > upreckon > hgweb

comparison 2.00/testcases.py @ 22:f07b7a431ea6

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Further 2.00 work Testconfs in all supported kinds of archives should now work. Test runs are now cancelled by pressing Escape rather than Ctrl+C. Improved time control. Greatly improved temporary and helper file cleanup. The pause configuration variable can now be a callable and is now processed using subprocess rather than system().
author Oleg Oshmyan <chortos@inbox.lv>
date Wed, 22 Sep 2010 22:01:56 +0000
parents ec6f1a132109
children c1f52b5d80d6
comparison
equal deleted inserted replaced
21:ec6f1a132109 22:f07b7a431ea6
33 import ctypes 33 import ctypes
34 TerminateProcess = ctypes.windll.kernel32.TerminateProcess 34 TerminateProcess = ctypes.windll.kernel32.TerminateProcess
35 except (ImportError, AttributeError): 35 except (ImportError, AttributeError):
36 TerminateProcess = None 36 TerminateProcess = None
37 37
38
39 # Do the hacky-wacky dark magic needed to catch presses of the Escape button.
40 # If only Python supported forcible termination of threads...
41 if not sys.stdin.isatty():
42 canceled = init_canceled = lambda: False
43 pause = None
44 else:
45 try:
46 # Windows has select() too, but it is not the select() we want
47 import msvcrt
48 except ImportError:
49 try:
50 import select, termios, tty, atexit
51 except ImportError:
52 # It cannot be helped!
53 # Silently disable support for killing the program being tested
54 canceled = init_canceled = lambda: False
55 pause = None
56 else:
57 def cleanup(old=termios.tcgetattr(sys.stdin.fileno())):
58 termios.tcsetattr(sys.stdin.fileno(), termios.TCSAFLUSH, old)
59 atexit.register(cleanup)
60 del cleanup
61 tty.setcbreak(sys.stdin.fileno())
62 def canceled():
63 while select.select((sys.stdin,), (), (), 0)[0]:
64 if sys.stdin.read(1) == '\33':
65 return True
66 return False
67 def init_canceled():
68 while select.select((sys.stdin,), (), (), 0)[0]:
69 sys.stdin.read(1)
70 def pause():
71 sys.stdin.read(1)
72 else:
73 def canceled():
74 while msvcrt.kbhit():
75 c = msvcrt.getch()
76 if c == '\33':
77 return True
78 elif c == '\0':
79 # Let's hope no-one is fiddling with this
80 msvcrt.getch()
81 return False
82 def init_canceled():
83 while msvcrt.kbhit():
84 msvcrt.getch()
85 def pause():
86 msvcrt.getch()
87
88
38 __all__ = ('TestCase', 'load_problem', 'TestCaseNotPassed', 89 __all__ = ('TestCase', 'load_problem', 'TestCaseNotPassed',
39 'TimeLimitExceeded', 'WrongAnswer', 'NonZeroExitCode', 90 'TimeLimitExceeded', 'CanceledByUser', 'WrongAnswer',
40 'CannotStartTestee', 'CannotStartValidator', 91 'NonZeroExitCode', 'CannotStartTestee',
41 'CannotReadOutputFile') 92 'CannotStartValidator', 'CannotReadOutputFile',
93 'CannotReadInputFile', 'CannotReadAnswerFile')
42 94
43 95
44 96
45 # Exceptions 97 # Exceptions
46 98
47 class TestCaseNotPassed(Exception): __slots__ = () 99 class TestCaseNotPassed(Exception): __slots__ = ()
48 class TimeLimitExceeded(TestCaseNotPassed): __slots__ = () 100 class TimeLimitExceeded(TestCaseNotPassed): __slots__ = ()
101 class CanceledByUser(TestCaseNotPassed): __slots__ = ()
49 102
50 class WrongAnswer(TestCaseNotPassed): 103 class WrongAnswer(TestCaseNotPassed):
51 __slots__ = 'comment' 104 __slots__ = 'comment'
52 def __init__(self, comment=''): 105 def __init__(self, comment=''):
53 self.comment = comment 106 self.comment = comment
68 class CannotReadInputFile(ExceptionWrapper): __slots__ = () 121 class CannotReadInputFile(ExceptionWrapper): __slots__ = ()
69 class CannotReadAnswerFile(ExceptionWrapper): __slots__ = () 122 class CannotReadAnswerFile(ExceptionWrapper): __slots__ = ()
70 123
71 124
72 125
126 # Helper context managers
127
128 class CopyDeleting(object):
129 __slots__ = 'case', 'file', 'name'
130
131 def __init__(self, case, file, name):
132 self.case = case
133 self.file = file
134 self.name = name
135
136 def __enter__(self):
137 if self.name:
138 try:
139 self.file.copy(self.name)
140 except:
141 try:
142 self.__exit__(None, None, None)
143 except:
144 pass
145 raise
146
147 def __exit__(self, exc_type, exc_val, exc_tb):
148 if self.name:
149 self.case.files_to_delete.append(self.name)
150
151
152 class Copying(object):
153 __slots__ = 'file', 'name'
154
155 def __init__(self, file, name):
156 self.file = file
157 self.name = name
158
159 def __enter__(self):
160 if self.name:
161 self.file.copy(self.name)
162
163 def __exit__(self, exc_type, exc_val, exc_tb):
164 pass
165
166
167
73 # Test case types 168 # Test case types
74 169
75 class TestCase(object): 170 class TestCase(object):
76 __slots__ = ('problem', 'id', 'isdummy', 'infile', 'outfile', 'points', 171 __slots__ = ('problem', 'id', 'isdummy', 'infile', 'outfile', 'points',
77 'process', 'time_started', 'time_stopped', 'time_limit_string', 172 'process', 'time_started', 'time_stopped', 'time_limit_string',
78 'realinname', 'realoutname', 'maxtime', 'maxmemory') 173 'realinname', 'realoutname', 'maxtime', 'maxmemory',
174 'has_called_back', 'files_to_delete')
79 175
80 if ABCMeta: 176 if ABCMeta:
81 __metaclass__ = ABCMeta 177 __metaclass__ = ABCMeta
82 178
83 def __init__(case, prob, id, isdummy, points): 179 def __init__(case, prob, id, isdummy, points):
99 case.realoutname = case.problem.config.dummyoutname 195 case.realoutname = case.problem.config.dummyoutname
100 196
101 @abstractmethod 197 @abstractmethod
102 def test(case): raise NotImplementedError 198 def test(case): raise NotImplementedError
103 199
104 def __call__(case): 200 def __call__(case, callback):
201 case.has_called_back = False
202 case.files_to_delete = []
105 try: 203 try:
106 return case.test() 204 return case.test(callback)
107 finally: 205 finally:
206 now = clock()
207 if not getattr(case, 'time_started', None):
208 case.time_started = case.time_stopped = now
209 elif not getattr(case, 'time_stopped', None):
210 case.time_stopped = now
211 if not case.has_called_back:
212 callback()
108 case.cleanup() 213 case.cleanup()
109 214
110 def cleanup(case): 215 def cleanup(case):
111 if not getattr(case, 'time_started', None):
112 case.time_started = case.time_stopped = clock()
113 elif not getattr(case, 'time_stopped', None):
114 case.time_stopped = clock()
115 #if getattr(case, 'infile', None): 216 #if getattr(case, 'infile', None):
116 # case.infile.close() 217 # case.infile.close()
117 #if getattr(case, 'outfile', None): 218 #if getattr(case, 'outfile', None):
118 # case.outfile.close() 219 # case.outfile.close()
119 if getattr(case, 'process', None): 220 if getattr(case, 'process', None):
133 os.kill(proc.pid, SIGTERM) 234 os.kill(proc.pid, SIGTERM)
134 except Exception: 235 except Exception:
135 time.sleep(0) 236 time.sleep(0)
136 case.process.poll() 237 case.process.poll()
137 else: 238 else:
239 case.process.wait()
138 break 240 break
139 else: 241 else:
140 # If killing the process is unsuccessful three times in a row, 242 # If killing the process is unsuccessful three times in a row,
141 # just silently stop trying 243 # just silently stop trying
142 for i in range(3): 244 for i in range(3):
153 os.kill(proc.pid, SIGKILL) 255 os.kill(proc.pid, SIGKILL)
154 except Exception: 256 except Exception:
155 time.sleep(0) 257 time.sleep(0)
156 case.process.poll() 258 case.process.poll()
157 else: 259 else:
260 case.process.wait()
158 break 261 break
262 if case.files_to_delete:
263 for name in case.files_to_delete:
264 try:
265 os.remove(name)
266 except Exception:
267 # It can't be helped
268 pass
159 269
160 def open_infile(case): 270 def open_infile(case):
161 try: 271 try:
162 case.infile = files.File('/'.join((case.problem.name, case.realinname.replace('$', case.id)))) 272 case.infile = files.File('/'.join((case.problem.name, case.realinname.replace('$', case.id))))
163 except IOError: 273 except IOError:
190 with case.outfile.open() as refoutput: 300 with case.outfile.open() as refoutput:
191 for line, refline in zip(output, refoutput): 301 for line, refline in zip(output, refoutput):
192 if not isinstance(refline, basestring): 302 if not isinstance(refline, basestring):
193 line = bytes(line, sys.getdefaultencoding()) 303 line = bytes(line, sys.getdefaultencoding())
194 if line != refline: 304 if line != refline:
195 raise WrongAnswer() 305 raise WrongAnswer
196 try: 306 try:
197 try: 307 try:
198 next(output) 308 next(output)
199 except NameError: 309 except NameError:
200 output.next() 310 output.next()
201 except StopIteration: 311 except StopIteration:
202 pass 312 pass
203 else: 313 else:
204 raise WrongAnswer() 314 raise WrongAnswer
205 try: 315 try:
206 try: 316 try:
207 next(refoutput) 317 next(refoutput)
208 except NameError: 318 except NameError:
209 refoutput.next() 319 refoutput.next()
210 except StopIteration: 320 except StopIteration:
211 pass 321 pass
212 else: 322 else:
213 raise WrongAnswer() 323 raise WrongAnswer
214 return case.points 324 return case.points
215 elif callable(case.validator): 325 elif callable(case.validator):
216 return case.validator(output) 326 return case.validator(output)
217 else: 327 else:
218 # Call the validator program 328 # Call the validator program
219 output.close() 329 output.close()
220 case.open_outfile() 330 case.open_outfile()
221 if case.problem.config.ansname: 331 case.outfile.copy(case.problem.config.ansname)
222 case.outfile.copy(case.problem.config.ansname)
223 case.process = Popen(case.validator, stdin=devnull, stdout=PIPE, stderr=STDOUT, universal_newlines=True, bufsize=-1) 332 case.process = Popen(case.validator, stdin=devnull, stdout=PIPE, stderr=STDOUT, universal_newlines=True, bufsize=-1)
224 comment = case.process.communicate()[0].strip() 333 comment = case.process.communicate()[0].strip()
225 lower = comment.lower() 334 lower = comment.lower()
226 match = re.match(r'(ok|correct|wrong(?:(?:\s|_)*answer)?)(?:$|\s+|[.,!:]+\s*)', lower) 335 match = re.match(r'(ok|correct|wrong(?:(?:\s|_)*answer)?)(?:$|\s+|[.,!:]+\s*)', lower)
227 if match: 336 if match:
236 345
237 346
238 class BatchTestCase(ValidatedTestCase): 347 class BatchTestCase(ValidatedTestCase):
239 __slots__ = () 348 __slots__ = ()
240 349
241 def test(case): 350 def test(case, callback):
351 init_canceled()
242 if sys.platform == 'win32' or not case.maxmemory: 352 if sys.platform == 'win32' or not case.maxmemory:
243 preexec_fn = None 353 preexec_fn = None
244 else: 354 else:
245 def preexec_fn(): 355 def preexec_fn():
246 try: 356 try:
258 pass 368 pass
259 case.open_infile() 369 case.open_infile()
260 case.time_started = None 370 case.time_started = None
261 if case.problem.config.stdio: 371 if case.problem.config.stdio:
262 if options.erase and not case.validator: 372 if options.erase and not case.validator:
373 # TODO: re-use the same file name if possible
263 # FIXME: 2.5 lacks the delete parameter 374 # FIXME: 2.5 lacks the delete parameter
264 with tempfile.NamedTemporaryFile(delete=False) as f: 375 with tempfile.NamedTemporaryFile(delete=False) as f:
265 inputdatafname = f.name 376 inputdatafname = f.name
377 context = CopyDeleting(case, case.infile, inputdatafname)
266 else: 378 else:
267 inputdatafname = case.problem.config.inname 379 inputdatafname = case.problem.config.inname
268 case.infile.copy(inputdatafname) 380 context = Copying(case.infile, inputdatafname)
269 # FIXME: inputdatafname should be deleted on __exit__ 381 with context:
270 with open(inputdatafname, 'rU') as infile: 382 with open(inputdatafname, 'rU') as infile:
271 with tempfile.TemporaryFile('w+') if options.erase and not case.validator else open(case.problem.config.outname, 'w+') as outfile: 383 with tempfile.TemporaryFile('w+') if options.erase and not case.validator else open(case.problem.config.outname, 'w+') as outfile:
272 try:
273 try: 384 try:
274 case.process = Popen(case.problem.config.path, stdin=infile, stdout=outfile, stderr=devnull, universal_newlines=True, bufsize=-1, preexec_fn=preexec_fn) 385 try:
275 except MemoryError: 386 case.process = Popen(case.problem.config.path, stdin=infile, stdout=outfile, stderr=devnull, universal_newlines=True, bufsize=-1, preexec_fn=preexec_fn)
276 # If there is not enough memory for the forked test.py, 387 except MemoryError:
277 # opt for silent dropping of the limit 388 # If there is not enough memory for the forked test.py,
278 case.process = Popen(case.problem.config.path, stdin=infile, stdout=outfile, stderr=devnull, universal_newlines=True, bufsize=-1) 389 # opt for silent dropping of the limit
279 except OSError: 390 case.process = Popen(case.problem.config.path, stdin=infile, stdout=outfile, stderr=devnull, universal_newlines=True, bufsize=-1)
280 raise CannotStartTestee(sys.exc_info()[1]) 391 except OSError:
281 case.time_started = clock() 392 raise CannotStartTestee(sys.exc_info()[1])
282 # If we use a temporary file, it may not be a true file object, 393 case.time_started = clock()
283 # and if so, Popen will relay the standard output through pipes 394 if not case.maxtime:
284 if not case.maxtime: 395 while True:
285 case.process.communicate() 396 exitcode, now = case.process.poll(), clock()
286 case.time_stopped = clock() 397 if exitcode is not None:
287 else: 398 case.time_stopped = now
288 time_end = case.time_started + case.maxtime 399 break
289 # FIXME: emulate communicate() 400 elif canceled():
290 while True: 401 raise CanceledByUser
291 exitcode = case.process.poll() 402 else:
292 now = clock() 403 time_end = case.time_started + case.maxtime
293 if exitcode is not None: 404 while True:
294 case.time_stopped = now 405 exitcode, now = case.process.poll(), clock()
295 break 406 if exitcode is not None:
296 elif now >= time_end: 407 case.time_stopped = now
297 raise TimeLimitExceeded() 408 break
298 if config.globalconf.force_zero_exitcode and case.process.returncode: 409 elif now >= time_end:
299 raise NonZeroExitCode(case.process.returncode) 410 raise TimeLimitExceeded
300 outfile.seek(0) 411 elif canceled():
301 return case.validate(outfile) 412 raise CanceledByUser
413 if config.globalconf.force_zero_exitcode and case.process.returncode:
414 raise NonZeroExitCode(case.process.returncode)
415 callback()
416 case.has_called_back = True
417 outfile.seek(0)
418 return case.validate(outfile)
302 else: 419 else:
303 if case.problem.config.inname: 420 case.infile.copy(case.problem.config.inname)
304 case.infile.copy(case.problem.config.inname)
305 try: 421 try:
306 try: 422 try:
307 case.process = Popen(case.problem.config.path, stdin=devnull, stdout=devnull, stderr=STDOUT, preexec_fn=preexec_fn) 423 case.process = Popen(case.problem.config.path, stdin=devnull, stdout=devnull, stderr=STDOUT, preexec_fn=preexec_fn)
308 except MemoryError: 424 except MemoryError:
309 # If there is not enough memory for the forked test.py, 425 # If there is not enough memory for the forked test.py,
311 case.process = Popen(case.problem.config.path, stdin=devnull, stdout=devnull, stderr=STDOUT) 427 case.process = Popen(case.problem.config.path, stdin=devnull, stdout=devnull, stderr=STDOUT)
312 except OSError: 428 except OSError:
313 raise CannotStartTestee(sys.exc_info()[1]) 429 raise CannotStartTestee(sys.exc_info()[1])
314 case.time_started = clock() 430 case.time_started = clock()
315 if not case.maxtime: 431 if not case.maxtime:
316 case.process.wait() 432 while True:
317 case.time_stopped = clock() 433 exitcode, now = case.process.poll(), clock()
434 if exitcode is not None:
435 case.time_stopped = now
436 break
437 elif canceled():
438 raise CanceledByUser
318 else: 439 else:
319 time_end = case.time_started + case.maxtime 440 time_end = case.time_started + case.maxtime
320 while True: 441 while True:
321 exitcode = case.process.poll() 442 exitcode, now = case.process.poll(), clock()
322 now = clock()
323 if exitcode is not None: 443 if exitcode is not None:
324 case.time_stopped = now 444 case.time_stopped = now
325 break 445 break
326 elif now >= time_end: 446 elif now >= time_end:
327 raise TimeLimitExceeded() 447 raise TimeLimitExceeded
448 elif canceled():
449 raise CanceledByUser
328 if config.globalconf.force_zero_exitcode and case.process.returncode: 450 if config.globalconf.force_zero_exitcode and case.process.returncode:
329 raise NonZeroExitCode(case.process.returncode) 451 raise NonZeroExitCode(case.process.returncode)
452 callback()
453 case.has_called_back = True
330 with open(case.problem.config.outname, 'rU') as output: 454 with open(case.problem.config.outname, 'rU') as output:
331 return case.validate(output) 455 return case.validate(output)
332 456
333 457
334 # This is the only test case type not executing any programs to be tested 458 # This is the only test case type not executing any programs to be tested
351 def load_problem(prob, _types={'batch' : BatchTestCase, 475 def load_problem(prob, _types={'batch' : BatchTestCase,
352 'outonly' : OutputOnlyTestCase, 476 'outonly' : OutputOnlyTestCase,
353 'bestout' : BestOutputTestCase, 477 'bestout' : BestOutputTestCase,
354 'reactive': ReactiveTestCase}): 478 'reactive': ReactiveTestCase}):
355 if prob.config.usegroups: 479 if prob.config.usegroups:
480 # FIXME: test groups should again be supported!
356 pass 481 pass
357 else: 482 else:
358 # We will need to iterate over these configuration variables twice 483 # We will need to iterate over these configuration variables twice
359 try: 484 try:
360 len(prob.config.dummies) 485 len(prob.config.dummies)