Mercurial > ~astiob > upreckon > hgweb
annotate testcases.py @ 76:0e5ae28e0b2b
Points are now weighted on a test context basis
In particular, this has allowed for simple extensions to the format
of testconf to award points to whole test groups without at the same time
compromising the future ability of giving partial score for correct
but slow solutions. Specifically, the groupweight configuration variable
has been added and normally has the format {groupindex: points} where
groupindex is the group's index in the tests configuration variable.
The backwards incompatible change is that test contexts are no longer
guaranteed to learn the score awarded or the maximum possible score
for every test case and may instead be notified about them in batches.
In other news, the pointmap and groupweight configuration variables can
(now) be given as sequences in addition to mappings. (Technically,
the distinction currently made is dict versus everything else.) Items
of a sequence pointmap/groupweight correspond directly to the test cases/
groups defined in the tests configuration variable; in particular,
when groups are used, tests=[1],[2,3];pointmap={1:1,2:2,3:3} can now be
written as pointmap=tests=[1],[2,3]. Missing items are handled in the same
way in which they are handled when the variable is a mapping. Note
that the items of groupweight correspond to whole test groups rather
than individual test cases.
In other news again, the wording of problem total lines has been changed
from '<unweighted> points; weighted score: <weighted>' to '<weighted>
points (<unweighted> before weighting)', and group total lines now
properly report fractional numbers of points (this is a bug fix).
author | Oleg Oshmyan <chortos@inbox.lv> |
---|---|
date | Sat, 08 Jan 2011 16:03:35 +0200 |
parents | 7520b6bb6636 |
children | 69eadc60f4e2 |
rev | line source |
---|---|
21 | 1 #! /usr/bin/env python |
16 | 2 # Copyright (c) 2010 Chortos-2 <chortos@inbox.lv> |
3 | |
43 | 4 # TODO: copy the ansfile if not options.erase even if no validator is used |
5 | |
21 | 6 from __future__ import division, with_statement |
7 | |
8 try: | |
9 from compat import * | |
10 import files, problem, config | |
11 except ImportError: | |
12 import __main__ | |
13 __main__.import_error(sys.exc_info()[1]) | |
14 else: | |
15 from __main__ import clock, options | |
16 | |
17 import glob, re, sys, tempfile, time | |
18 from subprocess import Popen, PIPE, STDOUT | |
19 | |
20 import os | |
21 devnull = open(os.path.devnull, 'w+') | |
22 | |
23 try: | |
24 from signal import SIGTERM, SIGKILL | |
25 except ImportError: | |
26 SIGTERM = 15 | |
27 SIGKILL = 9 | |
28 | |
16 | 29 try: |
21 | 30 from _subprocess import TerminateProcess |
31 except ImportError: | |
32 # CPython 2.5 does define _subprocess.TerminateProcess even though it is | |
33 # not used in the subprocess module, but maybe something else does not | |
34 try: | |
35 import ctypes | |
36 TerminateProcess = ctypes.windll.kernel32.TerminateProcess | |
37 except (ImportError, AttributeError): | |
38 TerminateProcess = None | |
39 | |
22 | 40 |
72
7520b6bb6636
Windows Error Reporting is now suppressed (at least the dialogs)
Oleg Oshmyan <chortos@inbox.lv>
parents:
71
diff
changeset
|
41 # Do not show error messages due to errors in the program being tested |
7520b6bb6636
Windows Error Reporting is now suppressed (at least the dialogs)
Oleg Oshmyan <chortos@inbox.lv>
parents:
71
diff
changeset
|
42 try: |
7520b6bb6636
Windows Error Reporting is now suppressed (at least the dialogs)
Oleg Oshmyan <chortos@inbox.lv>
parents:
71
diff
changeset
|
43 import ctypes |
7520b6bb6636
Windows Error Reporting is now suppressed (at least the dialogs)
Oleg Oshmyan <chortos@inbox.lv>
parents:
71
diff
changeset
|
44 try: |
7520b6bb6636
Windows Error Reporting is now suppressed (at least the dialogs)
Oleg Oshmyan <chortos@inbox.lv>
parents:
71
diff
changeset
|
45 errmode = ctypes.windll.kernel32.GetErrorMode() |
7520b6bb6636
Windows Error Reporting is now suppressed (at least the dialogs)
Oleg Oshmyan <chortos@inbox.lv>
parents:
71
diff
changeset
|
46 except AttributeError: |
7520b6bb6636
Windows Error Reporting is now suppressed (at least the dialogs)
Oleg Oshmyan <chortos@inbox.lv>
parents:
71
diff
changeset
|
47 errmode = ctypes.windll.kernel32.SetErrorMode(0) |
7520b6bb6636
Windows Error Reporting is now suppressed (at least the dialogs)
Oleg Oshmyan <chortos@inbox.lv>
parents:
71
diff
changeset
|
48 errmode |= 0x8003 |
7520b6bb6636
Windows Error Reporting is now suppressed (at least the dialogs)
Oleg Oshmyan <chortos@inbox.lv>
parents:
71
diff
changeset
|
49 ctypes.windll.kernel32.SetErrorMode(errmode) |
7520b6bb6636
Windows Error Reporting is now suppressed (at least the dialogs)
Oleg Oshmyan <chortos@inbox.lv>
parents:
71
diff
changeset
|
50 except Exception: |
7520b6bb6636
Windows Error Reporting is now suppressed (at least the dialogs)
Oleg Oshmyan <chortos@inbox.lv>
parents:
71
diff
changeset
|
51 pass |
7520b6bb6636
Windows Error Reporting is now suppressed (at least the dialogs)
Oleg Oshmyan <chortos@inbox.lv>
parents:
71
diff
changeset
|
52 |
7520b6bb6636
Windows Error Reporting is now suppressed (at least the dialogs)
Oleg Oshmyan <chortos@inbox.lv>
parents:
71
diff
changeset
|
53 |
22 | 54 # Do the hacky-wacky dark magic needed to catch presses of the Escape button. |
55 # If only Python supported forcible termination of threads... | |
56 if not sys.stdin.isatty(): | |
57 canceled = init_canceled = lambda: False | |
58 pause = None | |
59 else: | |
60 try: | |
61 # Windows has select() too, but it is not the select() we want | |
62 import msvcrt | |
63 except ImportError: | |
64 try: | |
56
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
65 from select import select |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
66 import termios, tty, atexit |
22 | 67 except ImportError: |
68 # It cannot be helped! | |
69 # Silently disable support for killing the program being tested | |
70 canceled = init_canceled = lambda: False | |
71 pause = None | |
72 else: | |
73 def cleanup(old=termios.tcgetattr(sys.stdin.fileno())): | |
74 termios.tcsetattr(sys.stdin.fileno(), termios.TCSAFLUSH, old) | |
75 atexit.register(cleanup) | |
76 del cleanup | |
77 tty.setcbreak(sys.stdin.fileno()) | |
56
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
78 def canceled(select=select, stdin=sys.stdin, read=sys.stdin.read): |
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
79 while select((stdin,), (), (), 0)[0]: |
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
80 if read(1) == '\33': |
22 | 81 return True |
82 return False | |
83 def init_canceled(): | |
56
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
84 while select((sys.stdin,), (), (), 0)[0]: |
22 | 85 sys.stdin.read(1) |
86 def pause(): | |
87 sys.stdin.read(1) | |
88 else: | |
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
89 def canceled(kbhit=msvcrt.kbhit, getch=msvcrt.getch): |
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
90 while kbhit(): |
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
91 c = getch() |
22 | 92 if c == '\33': |
93 return True | |
94 elif c == '\0': | |
95 # Let's hope no-one is fiddling with this | |
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
96 getch() |
22 | 97 return False |
98 def init_canceled(): | |
99 while msvcrt.kbhit(): | |
100 msvcrt.getch() | |
101 def pause(): | |
102 msvcrt.getch() | |
103 | |
56
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
104 try: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
105 from signal import SIGCHLD, signal, SIG_DFL |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
106 from select import select, error as select_error |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
107 from errno import EINTR |
65
fcb5ab97f08e
Improved run-time reporting and fixed a potential hang on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
63
diff
changeset
|
108 import fcntl |
fcb5ab97f08e
Improved run-time reporting and fixed a potential hang on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
63
diff
changeset
|
109 try: |
fcb5ab97f08e
Improved run-time reporting and fixed a potential hang on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
63
diff
changeset
|
110 import cPickle as pickle |
fcb5ab97f08e
Improved run-time reporting and fixed a potential hang on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
63
diff
changeset
|
111 except ImportError: |
fcb5ab97f08e
Improved run-time reporting and fixed a potential hang on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
63
diff
changeset
|
112 import pickle |
56
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
113 except ImportError: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
114 try: |
61
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
115 from _subprocess import WAIT_OBJECT_0, STD_INPUT_HANDLE, INFINITE |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
116 except ImportError: |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
117 WAIT_OBJECT_0 = 0 |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
118 STD_INPUT_HANDLE = -10 |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
119 INFINITE = -1 |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
120 try: |
56
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
121 import ctypes |
61
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
122 SetConsoleMode = ctypes.windll.kernel32.SetConsoleMode |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
123 FlushConsoleInputBuffer = ctypes.windll.kernel32.FlushConsoleInputBuffer |
56
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
124 WaitForMultipleObjects = ctypes.windll.kernel32.WaitForMultipleObjects |
61
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
125 ReadConsoleInputA = ctypes.windll.kernel32.ReadConsoleInputA |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
126 try: |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
127 from _subprocess import GetStdHandle |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
128 except ImportError: |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
129 GetStdHandle = ctypes.windll.kernel32.GetStdHandle |
56
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
130 except (ImportError, AttributeError): |
61
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
131 console_input = False |
56
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
132 else: |
61
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
133 hStdin = GetStdHandle(STD_INPUT_HANDLE) |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
134 console_input = bool(SetConsoleMode(hStdin, 1)) |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
135 if console_input: |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
136 FlushConsoleInputBuffer(hStdin) |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
137 class KEY_EVENT_RECORD(ctypes.Structure): |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
138 _fields_ = (("bKeyDown", ctypes.c_int), |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
139 ("wRepeatCount", ctypes.c_ushort), |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
140 ("wVirtualKeyCode", ctypes.c_ushort), |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
141 ("wVirtualScanCode", ctypes.c_ushort), |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
142 ("UnicodeChar", ctypes.c_wchar), |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
143 ("dwControlKeyState", ctypes.c_uint)) |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
144 class INPUT_RECORD(ctypes.Structure): |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
145 _fields_ = (("EventType", ctypes.c_int), |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
146 ("KeyEvent", KEY_EVENT_RECORD)) |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
147 # Memory limits (currently) are not supported |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
148 def call(*args, **kwargs): |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
149 case = kwargs.pop('case') |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
150 try: |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
151 case.process = Popen(*args, **kwargs) |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
152 except OSError: |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
153 raise CannotStartTestee(sys.exc_info()[1]) |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
154 case.time_started = clock() |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
155 if not console_input: |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
156 if case.maxtime: |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
157 if WaitForSingleObject(case.process._handle, int(case.maxtime * 1000)) != WAIT_OBJECT_0: |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
158 raise TimeLimitExceeded |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
159 else: |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
160 case.process.wait() |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
161 else: |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
162 ir = INPUT_RECORD() |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
163 n = ctypes.c_int() |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
164 lpHandles = (ctypes.c_int * 2)(hStdin, case.process._handle) |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
165 if case.maxtime: |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
166 time_end = clock() + case.maxtime |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
167 while case.process.poll() is None: |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
168 remaining = time_end - clock() |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
169 if remaining > 0: |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
170 if WaitForMultipleObjects(2, lpHandles, False, int(remaining * 1000)) == WAIT_OBJECT_0: |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
171 ReadConsoleInputA(hStdin, ctypes.byref(ir), 1, ctypes.byref(n)) |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
172 if ir.EventType == 1 and ir.KeyEvent.bKeyDown and ir.KeyEvent.wVirtualKeyCode == 27: |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
173 raise CanceledByUser |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
174 else: |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
175 raise TimeLimitExceeded |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
176 else: |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
177 while case.process.poll() is None: |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
178 if WaitForMultipleObjects(2, lpHandles, False, INFINITE) == WAIT_OBJECT_0: |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
179 ReadConsoleInputA(hStdin, ctypes.byref(ir), 1, ctypes.byref(n)) |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
180 if ir.EventType == 1 and ir.KeyEvent.bKeyDown and ir.KeyEvent.wVirtualKeyCode == 27: |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
181 raise CanceledByUser |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
182 case.time_stopped = clock() |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
183 if not console_input: |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
184 try: |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
185 try: |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
186 from _subprocess import WaitForSingleObject |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
187 except ImportError: |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
188 import ctypes |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
189 WaitForSingleObject = ctypes.windll.kernel32.WaitForSingleObject |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
190 except (ImportError, AttributeError): |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
191 # TODO: move the default implementation here |
24f144e11b5e
Accurate run-time reporting on Win32
Oleg Oshmyan <chortos@inbox.lv>
parents:
58
diff
changeset
|
192 call = None |
56
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
193 else: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
194 # Make SIGCHLD interrupt sleep() and select() |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
195 def bury_child(signum, frame): |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
196 try: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
197 bury_child.case.time_stopped = clock() |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
198 except Exception: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
199 pass |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
200 signal(SIGCHLD, bury_child) |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
201 |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
202 # If you want this to work, don't set any stdio argument to PIPE |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
203 def call_real(*args, **kwargs): |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
204 bury_child.case = case = kwargs.pop('case') |
63
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
205 preexec_fn_ = kwargs.get('preexec_fn', None) |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
206 read, write = os.pipe() |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
207 def preexec_fn(): |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
208 os.close(read) |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
209 if preexec_fn_: |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
210 preexec_fn_() |
65
fcb5ab97f08e
Improved run-time reporting and fixed a potential hang on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
63
diff
changeset
|
211 fcntl.fcntl(write, fcntl.F_SETFD, fcntl.fcntl(write, fcntl.F_GETFD) | getattr(fcntl, 'FD_CLOEXEC', 1)) |
69
c0f1b87013ad
Fixed a crash on Python 3 on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
65
diff
changeset
|
212 fwrite = os.fdopen(write, 'wb') |
65
fcb5ab97f08e
Improved run-time reporting and fixed a potential hang on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
63
diff
changeset
|
213 pickle.dump(clock(), fwrite, 1) |
63
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
214 kwargs['preexec_fn'] = preexec_fn |
56
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
215 try: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
216 case.process = Popen(*args, **kwargs) |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
217 except OSError: |
63
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
218 os.close(read) |
56
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
219 raise CannotStartTestee(sys.exc_info()[1]) |
63
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
220 finally: |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
221 os.close(write) |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
222 try: |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
223 if pause is None: |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
224 if case.maxtime: |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
225 time.sleep(case.maxtime) |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
226 if case.process.poll() is None: |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
227 raise TimeLimitExceeded |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
228 else: |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
229 case.process.wait() |
56
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
230 else: |
63
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
231 if not case.maxtime: |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
232 try: |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
233 while case.process.poll() is None: |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
234 if select((sys.stdin,), (), ())[0]: |
56
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
235 if sys.stdin.read(1) == '\33': |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
236 raise CanceledByUser |
63
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
237 except select_error: |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
238 if sys.exc_info()[1].args[0] != EINTR: |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
239 raise |
56
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
240 else: |
63
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
241 case.process.poll() |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
242 else: |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
243 time_end = clock() + case.maxtime |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
244 try: |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
245 while case.process.poll() is None: |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
246 remaining = time_end - clock() |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
247 if remaining > 0: |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
248 if select((sys.stdin,), (), (), remaining)[0]: |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
249 if sys.stdin.read(1) == '\33': |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
250 raise CanceledByUser |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
251 else: |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
252 raise TimeLimitExceeded |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
253 except select_error: |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
254 if sys.exc_info()[1].args[0] != EINTR: |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
255 raise |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
256 else: |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
257 case.process.poll() |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
258 finally: |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
259 case.time_started = pickle.loads(os.read(read, 512)) |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
260 os.close(read) |
fb9d0223a871
Fixed negative run times reported on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
62
diff
changeset
|
261 del bury_child.case |
56
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
262 def call(*args, **kwargs): |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
263 if 'preexec_fn' in kwargs: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
264 try: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
265 return call_real(*args, **kwargs) |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
266 except MemoryError: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
267 # If there is not enough memory for the forked test.py, |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
268 # opt for silent dropping of the limit |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
269 # TODO: show a warning somewhere |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
270 del kwargs['preexec_fn'] |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
271 return call_real(*args, **kwargs) |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
272 else: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
273 return call_real(*args, **kwargs) |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
274 |
22 | 275 |
21 | 276 __all__ = ('TestCase', 'load_problem', 'TestCaseNotPassed', |
22 | 277 'TimeLimitExceeded', 'CanceledByUser', 'WrongAnswer', |
278 'NonZeroExitCode', 'CannotStartTestee', | |
279 'CannotStartValidator', 'CannotReadOutputFile', | |
280 'CannotReadInputFile', 'CannotReadAnswerFile') | |
21 | 281 |
282 | |
283 | |
284 # Exceptions | |
285 | |
286 class TestCaseNotPassed(Exception): __slots__ = () | |
287 class TimeLimitExceeded(TestCaseNotPassed): __slots__ = () | |
22 | 288 class CanceledByUser(TestCaseNotPassed): __slots__ = () |
21 | 289 |
290 class WrongAnswer(TestCaseNotPassed): | |
291 __slots__ = 'comment' | |
292 def __init__(self, comment=''): | |
293 self.comment = comment | |
294 | |
295 class NonZeroExitCode(TestCaseNotPassed): | |
296 __slots__ = 'exitcode' | |
297 def __init__(self, exitcode): | |
298 self.exitcode = exitcode | |
299 | |
300 class ExceptionWrapper(TestCaseNotPassed): | |
301 __slots__ = 'upstream' | |
302 def __init__(self, upstream): | |
303 self.upstream = upstream | |
304 | |
305 class CannotStartTestee(ExceptionWrapper): __slots__ = () | |
306 class CannotStartValidator(ExceptionWrapper): __slots__ = () | |
307 class CannotReadOutputFile(ExceptionWrapper): __slots__ = () | |
308 class CannotReadInputFile(ExceptionWrapper): __slots__ = () | |
309 class CannotReadAnswerFile(ExceptionWrapper): __slots__ = () | |
310 | |
311 | |
312 | |
22 | 313 # Helper context managers |
314 | |
315 class CopyDeleting(object): | |
316 __slots__ = 'case', 'file', 'name' | |
317 | |
318 def __init__(self, case, file, name): | |
319 self.case = case | |
320 self.file = file | |
321 self.name = name | |
322 | |
323 def __enter__(self): | |
324 if self.name: | |
325 try: | |
326 self.file.copy(self.name) | |
327 except: | |
328 try: | |
329 self.__exit__(None, None, None) | |
330 except: | |
331 pass | |
332 raise | |
333 | |
334 def __exit__(self, exc_type, exc_val, exc_tb): | |
335 if self.name: | |
336 self.case.files_to_delete.append(self.name) | |
337 | |
338 | |
339 class Copying(object): | |
340 __slots__ = 'file', 'name' | |
341 | |
342 def __init__(self, file, name): | |
343 self.file = file | |
344 self.name = name | |
345 | |
346 def __enter__(self): | |
347 if self.name: | |
348 self.file.copy(self.name) | |
349 | |
350 def __exit__(self, exc_type, exc_val, exc_tb): | |
351 pass | |
352 | |
353 | |
354 | |
21 | 355 # Test case types |
16 | 356 |
357 class TestCase(object): | |
21 | 358 __slots__ = ('problem', 'id', 'isdummy', 'infile', 'outfile', 'points', |
359 'process', 'time_started', 'time_stopped', 'time_limit_string', | |
22 | 360 'realinname', 'realoutname', 'maxtime', 'maxmemory', |
361 'has_called_back', 'files_to_delete') | |
21 | 362 |
363 if ABCMeta: | |
364 __metaclass__ = ABCMeta | |
16 | 365 |
21 | 366 def __init__(case, prob, id, isdummy, points): |
16 | 367 case.problem = prob |
21 | 368 case.id = id |
369 case.isdummy = isdummy | |
370 case.points = points | |
371 case.maxtime = case.problem.config.maxtime | |
372 case.maxmemory = case.problem.config.maxmemory | |
373 if case.maxtime: | |
374 case.time_limit_string = '/%.3f' % case.maxtime | |
375 else: | |
376 case.time_limit_string = '' | |
377 if not isdummy: | |
378 case.realinname = case.problem.config.testcaseinname | |
379 case.realoutname = case.problem.config.testcaseoutname | |
380 else: | |
381 case.realinname = case.problem.config.dummyinname | |
382 case.realoutname = case.problem.config.dummyoutname | |
383 | |
384 @abstractmethod | |
385 def test(case): raise NotImplementedError | |
16 | 386 |
22 | 387 def __call__(case, callback): |
388 case.has_called_back = False | |
389 case.files_to_delete = [] | |
21 | 390 try: |
22 | 391 return case.test(callback) |
21 | 392 finally: |
22 | 393 now = clock() |
394 if not getattr(case, 'time_started', None): | |
395 case.time_started = case.time_stopped = now | |
396 elif not getattr(case, 'time_stopped', None): | |
397 case.time_stopped = now | |
398 if not case.has_called_back: | |
399 callback() | |
21 | 400 case.cleanup() |
401 | |
402 def cleanup(case): | |
403 #if getattr(case, 'infile', None): | |
404 # case.infile.close() | |
405 #if getattr(case, 'outfile', None): | |
406 # case.outfile.close() | |
407 if getattr(case, 'process', None): | |
408 # Try killing after three unsuccessful TERM attempts in a row | |
409 # (except on Windows, where TERMing is killing) | |
410 for i in range(3): | |
411 try: | |
412 try: | |
413 case.process.terminate() | |
414 except AttributeError: | |
415 # Python 2.5 | |
416 if TerminateProcess and hasattr(proc, '_handle'): | |
417 # Windows API | |
418 TerminateProcess(proc._handle, 1) | |
419 else: | |
420 # POSIX | |
421 os.kill(proc.pid, SIGTERM) | |
422 except Exception: | |
423 time.sleep(0) | |
424 case.process.poll() | |
425 else: | |
22 | 426 case.process.wait() |
21 | 427 break |
428 else: | |
429 # If killing the process is unsuccessful three times in a row, | |
430 # just silently stop trying | |
431 for i in range(3): | |
432 try: | |
433 try: | |
434 case.process.kill() | |
435 except AttributeError: | |
436 # Python 2.5 | |
437 if TerminateProcess and hasattr(proc, '_handle'): | |
438 # Windows API | |
439 TerminateProcess(proc._handle, 1) | |
440 else: | |
441 # POSIX | |
442 os.kill(proc.pid, SIGKILL) | |
443 except Exception: | |
444 time.sleep(0) | |
445 case.process.poll() | |
446 else: | |
22 | 447 case.process.wait() |
21 | 448 break |
22 | 449 if case.files_to_delete: |
450 for name in case.files_to_delete: | |
451 try: | |
452 os.remove(name) | |
453 except Exception: | |
454 # It can't be helped | |
455 pass | |
21 | 456 |
457 def open_infile(case): | |
458 try: | |
459 case.infile = files.File('/'.join((case.problem.name, case.realinname.replace('$', case.id)))) | |
460 except IOError: | |
461 e = sys.exc_info()[1] | |
462 raise CannotReadInputFile(e) | |
463 | |
464 def open_outfile(case): | |
465 try: | |
466 case.outfile = files.File('/'.join((case.problem.name, case.realoutname.replace('$', case.id)))) | |
467 except IOError: | |
468 e = sys.exc_info()[1] | |
469 raise CannotReadAnswerFile(e) | |
470 | |
16 | 471 |
21 | 472 class ValidatedTestCase(TestCase): |
473 __slots__ = 'validator' | |
474 | |
475 def __init__(case, *args): | |
476 TestCase.__init__(case, *args) | |
477 if not case.problem.config.tester: | |
478 case.validator = None | |
479 else: | |
480 case.validator = case.problem.config.tester | |
481 | |
482 def validate(case, output): | |
483 if not case.validator: | |
484 # Compare the output with the reference output | |
485 case.open_outfile() | |
486 with case.outfile.open() as refoutput: | |
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
487 for line, refline in zip_longest(output, refoutput): |
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
488 if refline is not None and not isinstance(refline, basestring): |
21 | 489 line = bytes(line, sys.getdefaultencoding()) |
490 if line != refline: | |
22 | 491 raise WrongAnswer |
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
492 return 1 |
21 | 493 elif callable(case.validator): |
494 return case.validator(output) | |
495 else: | |
496 # Call the validator program | |
497 output.close() | |
23 | 498 if case.problem.config.ansname: |
499 case.open_outfile() | |
500 case.outfile.copy(case.problem.config.ansname) | |
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
501 try: |
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
502 case.process = Popen(case.validator, stdin=devnull, stdout=PIPE, stderr=STDOUT, universal_newlines=True, bufsize=-1) |
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
503 except OSError: |
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
504 raise CannotStartValidator(sys.exc_info()[1]) |
21 | 505 comment = case.process.communicate()[0].strip() |
26 | 506 match = re.match(r'(?i)(ok|(?:correct|wrong)(?:(?:\s|_)*answer)?)(?:$|\s+|[.,!:]+\s*)', comment) |
21 | 507 if match: |
508 comment = comment[match.end():] | |
509 if not case.problem.config.maxexitcode: | |
510 if case.process.returncode: | |
511 raise WrongAnswer(comment) | |
512 else: | |
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
513 return 1, comment |
21 | 514 else: |
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
515 return case.process.returncode / case.problem.config.maxexitcode, comment |
21 | 516 |
517 | |
518 class BatchTestCase(ValidatedTestCase): | |
519 __slots__ = () | |
520 | |
22 | 521 def test(case, callback): |
522 init_canceled() | |
21 | 523 if sys.platform == 'win32' or not case.maxmemory: |
524 preexec_fn = None | |
525 else: | |
526 def preexec_fn(): | |
527 try: | |
528 import resource | |
529 maxmemory = int(case.maxmemory * 1048576) | |
530 resource.setrlimit(resource.RLIMIT_AS, (maxmemory, maxmemory)) | |
531 # I would also set a CPU time limit but I do not want the time | |
532 # that passes between the calls to fork and exec to be counted in | |
533 except MemoryError: | |
534 # We do not have enough memory for ourselves; | |
535 # let the parent know about this | |
536 raise | |
537 except Exception: | |
538 # Well, at least we tried | |
539 pass | |
540 case.open_infile() | |
541 case.time_started = None | |
542 if case.problem.config.stdio: | |
54 | 543 if options.erase and not case.validator or not case.problem.config.inname: |
22 | 544 # TODO: re-use the same file name if possible |
21 | 545 # FIXME: 2.5 lacks the delete parameter |
546 with tempfile.NamedTemporaryFile(delete=False) as f: | |
22 | 547 inputdatafname = f.name |
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
548 contextmgr = CopyDeleting(case, case.infile, inputdatafname) |
21 | 549 else: |
550 inputdatafname = case.problem.config.inname | |
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
551 contextmgr = Copying(case.infile, inputdatafname) |
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
552 with contextmgr: |
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
553 # FIXME: this U doesn't do anything good for the child process, does it? |
22 | 554 with open(inputdatafname, 'rU') as infile: |
555 with tempfile.TemporaryFile('w+') if options.erase and not case.validator else open(case.problem.config.outname, 'w+') as outfile: | |
57
855bdfeb32a6
NameErrors within call() are now reported
Oleg Oshmyan <chortos@inbox.lv>
parents:
56
diff
changeset
|
556 if call is not None: |
56
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
557 call(case.problem.config.path, case=case, stdin=infile, stdout=outfile, stderr=devnull, universal_newlines=True, bufsize=-1, preexec_fn=preexec_fn) |
57
855bdfeb32a6
NameErrors within call() are now reported
Oleg Oshmyan <chortos@inbox.lv>
parents:
56
diff
changeset
|
558 else: |
22 | 559 try: |
56
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
560 try: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
561 case.process = Popen(case.problem.config.path, stdin=infile, stdout=outfile, stderr=devnull, universal_newlines=True, bufsize=-1, preexec_fn=preexec_fn) |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
562 except MemoryError: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
563 # If there is not enough memory for the forked test.py, |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
564 # opt for silent dropping of the limit |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
565 # TODO: show a warning somewhere |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
566 case.process = Popen(case.problem.config.path, stdin=infile, stdout=outfile, stderr=devnull, universal_newlines=True, bufsize=-1) |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
567 except OSError: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
568 raise CannotStartTestee(sys.exc_info()[1]) |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
569 case.time_started = clock() |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
570 time_next_check = case.time_started + .15 |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
571 if not case.maxtime: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
572 while True: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
573 exitcode, now = case.process.poll(), clock() |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
574 if exitcode is not None: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
575 case.time_stopped = now |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
576 break |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
577 # For some reason (probably Microsoft's fault), |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
578 # msvcrt.kbhit() is slow as hell |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
579 else: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
580 if now >= time_next_check: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
581 if canceled(): |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
582 raise CanceledByUser |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
583 else: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
584 time_next_check = now + .15 |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
585 time.sleep(.001) |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
586 else: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
587 time_end = case.time_started + case.maxtime |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
588 while True: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
589 exitcode, now = case.process.poll(), clock() |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
590 if exitcode is not None: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
591 case.time_stopped = now |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
592 break |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
593 elif now >= time_end: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
594 raise TimeLimitExceeded |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
595 else: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
596 if now >= time_next_check: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
597 if canceled(): |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
598 raise CanceledByUser |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
599 else: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
600 time_next_check = now + .15 |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
601 time.sleep(.001) |
62
593ad09cd69b
Multiple exit code handling fixes
Oleg Oshmyan <chortos@inbox.lv>
parents:
61
diff
changeset
|
602 if config.globalconf.force_zero_exitcode and case.process.returncode or case.process.returncode < 0: |
22 | 603 raise NonZeroExitCode(case.process.returncode) |
604 callback() | |
605 case.has_called_back = True | |
606 outfile.seek(0) | |
607 return case.validate(outfile) | |
21 | 608 else: |
22 | 609 case.infile.copy(case.problem.config.inname) |
57
855bdfeb32a6
NameErrors within call() are now reported
Oleg Oshmyan <chortos@inbox.lv>
parents:
56
diff
changeset
|
610 if call is not None: |
56
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
611 call(case.problem.config.path, case=case, stdin=devnull, stdout=devnull, stderr=STDOUT, preexec_fn=preexec_fn) |
57
855bdfeb32a6
NameErrors within call() are now reported
Oleg Oshmyan <chortos@inbox.lv>
parents:
56
diff
changeset
|
612 else: |
21 | 613 try: |
56
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
614 try: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
615 case.process = Popen(case.problem.config.path, stdin=devnull, stdout=devnull, stderr=STDOUT, preexec_fn=preexec_fn) |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
616 except MemoryError: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
617 # If there is not enough memory for the forked test.py, |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
618 # opt for silent dropping of the limit |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
619 # TODO: show a warning somewhere |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
620 case.process = Popen(case.problem.config.path, stdin=devnull, stdout=devnull, stderr=STDOUT) |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
621 except OSError: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
622 raise CannotStartTestee(sys.exc_info()[1]) |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
623 case.time_started = clock() |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
624 time_next_check = case.time_started + .15 |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
625 if not case.maxtime: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
626 while True: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
627 exitcode, now = case.process.poll(), clock() |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
628 if exitcode is not None: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
629 case.time_stopped = now |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
630 break |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
631 else: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
632 if now >= time_next_check: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
633 if canceled(): |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
634 raise CanceledByUser |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
635 else: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
636 time_next_check = now + .15 |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
637 time.sleep(.001) |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
638 else: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
639 time_end = case.time_started + case.maxtime |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
640 while True: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
641 exitcode, now = case.process.poll(), clock() |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
642 if exitcode is not None: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
643 case.time_stopped = now |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
644 break |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
645 elif now >= time_end: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
646 raise TimeLimitExceeded |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
647 else: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
648 if now >= time_next_check: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
649 if canceled(): |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
650 raise CanceledByUser |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
651 else: |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
652 time_next_check = now + .15 |
693a938bdeee
Accurate run-time reporting on POSIX
Oleg Oshmyan <chortos@inbox.lv>
parents:
55
diff
changeset
|
653 time.sleep(.001) |
62
593ad09cd69b
Multiple exit code handling fixes
Oleg Oshmyan <chortos@inbox.lv>
parents:
61
diff
changeset
|
654 if config.globalconf.force_zero_exitcode and case.process.returncode or case.process.returncode < 0: |
21 | 655 raise NonZeroExitCode(case.process.returncode) |
22 | 656 callback() |
657 case.has_called_back = True | |
21 | 658 with open(case.problem.config.outname, 'rU') as output: |
659 return case.validate(output) | |
660 | |
661 | |
662 # This is the only test case type not executing any programs to be tested | |
663 class OutputOnlyTestCase(ValidatedTestCase): | |
664 __slots__ = () | |
665 def cleanup(case): pass | |
666 | |
667 class BestOutputTestCase(ValidatedTestCase): | |
668 __slots__ = () | |
669 | |
670 # This is the only test case type executing two programs simultaneously | |
671 class ReactiveTestCase(TestCase): | |
672 __slots__ = () | |
673 # The basic idea is to launch the program to be tested and the grader | |
674 # and to pipe their standard I/O from and to each other, | |
675 # and then to capture the grader's exit code and use it | |
26 | 676 # like the exit code of an output validator is used. |
21 | 677 |
678 | |
71
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
679 class DummyTestContext(problem.TestGroup): |
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
680 __slots__ = () |
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
681 def end(self): |
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
682 say('Sample total: %d/%d tests' % (self.ncorrect, self.ntotal)) |
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
683 return 0, 0, self.log |
71
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
684 |
21 | 685 def load_problem(prob, _types={'batch' : BatchTestCase, |
686 'outonly' : OutputOnlyTestCase, | |
687 'bestout' : BestOutputTestCase, | |
688 'reactive': ReactiveTestCase}): | |
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
689 # We will need to iterate over these configuration variables twice |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
690 try: |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
691 len(prob.config.dummies) |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
692 except Exception: |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
693 prob.config.dummies = tuple(prob.config.dummies) |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
694 try: |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
695 len(prob.config.tests) |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
696 except Exception: |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
697 prob.config.tests = tuple(prob.config.tests) |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
698 |
23 | 699 if options.legacy: |
700 prob.config.usegroups = False | |
58 | 701 newtests = [] |
23 | 702 for i, name in enumerate(prob.config.tests): |
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
703 # Same here; we'll need to iterate over them twice |
23 | 704 try: |
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
705 l = len(name) |
23 | 706 except Exception: |
707 try: | |
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
708 name = tuple(name) |
23 | 709 except TypeError: |
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
710 name = (name,) |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
711 l = len(name) |
58 | 712 if l > 1: |
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
713 prob.config.usegroups = True |
58 | 714 newtests.append(name) |
715 if prob.config.usegroups: | |
716 prob.config.tests = newtests | |
717 del newtests | |
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
718 |
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
719 # Even if they have duplicate test identifiers, we must honour sequence pointmaps |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
720 if isinstance(prob.config.pointmap, dict): |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
721 def getpoints(i, j, k=None): |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
722 try: |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
723 return prob.config.pointmap[i] |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
724 except KeyError: |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
725 try: |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
726 return prob.config.pointmap[None] |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
727 except KeyError: |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
728 return prob.config.maxexitcode or 1 |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
729 elif prob.config.usegroups: |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
730 def getpoints(i, j, k): |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
731 try: |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
732 return prob.config.pointmap[k][j] |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
733 except LookupError: |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
734 return prob.config.maxexitcode or 1 |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
735 else: |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
736 def getpoints(i, j): |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
737 try: |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
738 return prob.config.pointmap[j] |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
739 except LookupError: |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
740 return prob.config.maxexitcode or 1 |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
741 |
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
742 # First get prob.cache.padoutput right, |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
743 # then yield the actual test cases |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
744 for i in prob.config.dummies: |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
745 s = 'sample ' + str(i).zfill(prob.config.paddummies) |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
746 prob.cache.padoutput = max(prob.cache.padoutput, len(s)) |
16 | 747 if prob.config.usegroups: |
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
748 if not isinstance(prob.config.groupweight, dict): |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
749 prob.config.groupweight = dict(enumerate(prob.config.groupweight)) |
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
750 for group in prob.config.tests: |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
751 for i in group: |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
752 s = str(i).zfill(prob.config.padtests) |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
753 prob.cache.padoutput = max(prob.cache.padoutput, len(s)) |
71
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
754 yield DummyTestContext() |
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
755 for i in prob.config.dummies: |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
756 s = str(i).zfill(prob.config.paddummies) |
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
757 yield _types[prob.config.kind](prob, s, True, 0) |
71
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
758 yield problem.test_context_end |
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
759 for k, group in enumerate(prob.config.tests): |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
760 if not group: |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
761 continue |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
762 yield problem.TestGroup(prob.config.groupweight.get(k, prob.config.groupweight.get(None))) |
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
763 for j, i in enumerate(group): |
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
764 s = str(i).zfill(prob.config.padtests) |
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
765 yield _types[prob.config.kind](prob, s, False, getpoints(i, j, k)) |
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
766 yield problem.test_context_end |
16 | 767 else: |
768 for i in prob.config.tests: | |
21 | 769 s = str(i).zfill(prob.config.padtests) |
770 prob.cache.padoutput = max(prob.cache.padoutput, len(s)) | |
771 for i in prob.config.dummies: | |
772 s = str(i).zfill(prob.config.paddummies) | |
773 yield _types[prob.config.kind](prob, s, True, 0) | |
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
774 for j, i in enumerate(prob.config.tests): |
21 | 775 s = str(i).zfill(prob.config.padtests) |
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
776 yield _types[prob.config.kind](prob, s, False, getpoints(i, j)) |