Mercurial > ~astiob > upreckon > hgweb
annotate testcases.py @ 89:3ae6cb69e4ef
Sample total is no longer printed if there are no samples
| author | Oleg Oshmyan <chortos@inbox.lv> |
|---|---|
| date | Mon, 28 Feb 2011 15:10:40 +0000 |
| parents | cd347cfca272 |
| children | 1fb319ec33af |
| rev | line source |
|---|---|
|
77
69eadc60f4e2
Memory limit is now applied to the RSS when os.wait4 is available
Oleg Oshmyan <chortos@inbox.lv>
parents:
76
diff
changeset
|
1 # Copyright (c) 2010-2011 Chortos-2 <chortos@inbox.lv> |
| 16 | 2 |
| 43 | 3 # TODO: copy the ansfile if not options.erase even if no validator is used |
| 4 | |
| 21 | 5 from __future__ import division, with_statement |
| 6 | |
| 7 try: | |
| 8 from compat import * | |
| 9 import files, problem, config | |
| 10 except ImportError: | |
| 11 import __main__ | |
| 12 __main__.import_error(sys.exc_info()[1]) | |
| 13 else: | |
|
85
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
14 from __main__ import options |
| 21 | 15 |
| 16 import glob, re, sys, tempfile, time | |
| 17 from subprocess import Popen, PIPE, STDOUT | |
| 18 | |
| 19 import os | |
| 20 devnull = open(os.path.devnull, 'w+') | |
| 21 | |
|
85
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
22 if options.autotime: |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
23 # This is really a dirty hack that assumes that sleep() does not spend |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
24 # the CPU time of the current process and that if clock() measures |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
25 # wall-clock time, then it is more precise than time() is. Both these |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
26 # assumptions are true on all platforms I have tested this on so far, |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
27 # but I am not aware of any guarantee that they will both be true |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
28 # on every other platform. |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
29 c = time.clock() |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
30 time.sleep(1) |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
31 c = time.clock() - c |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
32 if int(c + .5) == 1: |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
33 clock = time.clock |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
34 else: |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
35 clock = time.time |
|
741ae3391b61
Moved clock/time detection into platform-specific modules and testcases
Oleg Oshmyan <chortos@inbox.lv>
parents:
83
diff
changeset
|
36 |
| 21 | 37 try: |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
38 from win32 import * |
|
72
7520b6bb6636
Windows Error Reporting is now suppressed (at least the dialogs)
Oleg Oshmyan <chortos@inbox.lv>
parents:
71
diff
changeset
|
39 except Exception: |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
40 from unix import * |
| 22 | 41 |
| 21 | 42 __all__ = ('TestCase', 'load_problem', 'TestCaseNotPassed', |
| 22 | 43 'TimeLimitExceeded', 'CanceledByUser', 'WrongAnswer', |
| 44 'NonZeroExitCode', 'CannotStartTestee', | |
| 45 'CannotStartValidator', 'CannotReadOutputFile', | |
|
81
24752db487c5
Fixed errors in the win32 module
Oleg Oshmyan <chortos@inbox.lv>
parents:
79
diff
changeset
|
46 'CannotReadInputFile', 'CannotReadAnswerFile', |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
47 'MemoryLimitExceeded', 'CPUTimeLimitExceeded', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
48 'WallTimeLimitExceeded') |
| 21 | 49 |
| 50 | |
| 51 | |
| 52 # Exceptions | |
| 53 | |
| 54 class TestCaseNotPassed(Exception): __slots__ = () | |
| 55 class TimeLimitExceeded(TestCaseNotPassed): __slots__ = () | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
56 class CPUTimeLimitExceeded(TimeLimitExceeded): __slots__ = () |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
57 class WallTimeLimitExceeded(TimeLimitExceeded): __slots__ = () |
|
77
69eadc60f4e2
Memory limit is now applied to the RSS when os.wait4 is available
Oleg Oshmyan <chortos@inbox.lv>
parents:
76
diff
changeset
|
58 class MemoryLimitExceeded(TestCaseNotPassed): __slots__ = () |
| 22 | 59 class CanceledByUser(TestCaseNotPassed): __slots__ = () |
| 21 | 60 |
| 61 class WrongAnswer(TestCaseNotPassed): | |
| 62 __slots__ = 'comment' | |
| 63 def __init__(self, comment=''): | |
| 64 self.comment = comment | |
| 65 | |
| 66 class NonZeroExitCode(TestCaseNotPassed): | |
| 67 __slots__ = 'exitcode' | |
| 68 def __init__(self, exitcode): | |
| 69 self.exitcode = exitcode | |
| 70 | |
| 71 class ExceptionWrapper(TestCaseNotPassed): | |
| 72 __slots__ = 'upstream' | |
| 73 def __init__(self, upstream): | |
| 74 self.upstream = upstream | |
| 75 | |
| 76 class CannotStartTestee(ExceptionWrapper): __slots__ = () | |
| 77 class CannotStartValidator(ExceptionWrapper): __slots__ = () | |
| 78 class CannotReadOutputFile(ExceptionWrapper): __slots__ = () | |
| 79 class CannotReadInputFile(ExceptionWrapper): __slots__ = () | |
| 80 class CannotReadAnswerFile(ExceptionWrapper): __slots__ = () | |
| 81 | |
| 82 | |
| 83 | |
| 22 | 84 # Helper context managers |
| 85 | |
| 86 class CopyDeleting(object): | |
| 87 __slots__ = 'case', 'file', 'name' | |
| 88 | |
| 89 def __init__(self, case, file, name): | |
| 90 self.case = case | |
| 91 self.file = file | |
| 92 self.name = name | |
| 93 | |
| 94 def __enter__(self): | |
| 95 if self.name: | |
| 96 try: | |
| 97 self.file.copy(self.name) | |
| 98 except: | |
| 99 try: | |
| 100 self.__exit__(None, None, None) | |
| 101 except: | |
| 102 pass | |
| 103 raise | |
| 104 | |
| 105 def __exit__(self, exc_type, exc_val, exc_tb): | |
| 106 if self.name: | |
| 107 self.case.files_to_delete.append(self.name) | |
| 108 | |
| 109 | |
| 110 class Copying(object): | |
| 111 __slots__ = 'file', 'name' | |
| 112 | |
| 113 def __init__(self, file, name): | |
| 114 self.file = file | |
| 115 self.name = name | |
| 116 | |
| 117 def __enter__(self): | |
| 118 if self.name: | |
| 119 self.file.copy(self.name) | |
| 120 | |
| 121 def __exit__(self, exc_type, exc_val, exc_tb): | |
| 122 pass | |
| 123 | |
| 124 | |
| 125 | |
| 21 | 126 # Test case types |
| 16 | 127 |
| 128 class TestCase(object): | |
| 21 | 129 __slots__ = ('problem', 'id', 'isdummy', 'infile', 'outfile', 'points', |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
130 'process', 'time_started', 'time_stopped', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
131 'realinname', 'realoutname', 'maxcputime', 'maxwalltime', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
132 'maxmemory', 'has_called_back', 'files_to_delete', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
133 'cpu_time_limit_string', 'wall_time_limit_string', |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
134 'time_limit_string') |
| 21 | 135 |
| 136 if ABCMeta: | |
| 137 __metaclass__ = ABCMeta | |
| 16 | 138 |
| 21 | 139 def __init__(case, prob, id, isdummy, points): |
| 16 | 140 case.problem = prob |
| 21 | 141 case.id = id |
| 142 case.isdummy = isdummy | |
| 143 case.points = points | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
144 case.maxcputime = case.problem.config.maxcputime |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
145 case.maxwalltime = case.problem.config.maxwalltime |
| 21 | 146 case.maxmemory = case.problem.config.maxmemory |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
147 if case.maxcputime: |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
148 case.cpu_time_limit_string = '/%.3f' % case.maxcputime |
| 21 | 149 else: |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
150 case.cpu_time_limit_string = '' |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
151 if case.maxwalltime: |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
152 case.wall_time_limit_string = '/%.3f' % case.maxwalltime |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
153 else: |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
154 case.wall_time_limit_string = '' |
| 21 | 155 if not isdummy: |
| 156 case.realinname = case.problem.config.testcaseinname | |
| 157 case.realoutname = case.problem.config.testcaseoutname | |
| 158 else: | |
| 159 case.realinname = case.problem.config.dummyinname | |
| 160 case.realoutname = case.problem.config.dummyoutname | |
| 161 | |
| 162 @abstractmethod | |
| 163 def test(case): raise NotImplementedError | |
| 16 | 164 |
| 22 | 165 def __call__(case, callback): |
| 166 case.has_called_back = False | |
| 167 case.files_to_delete = [] | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
168 case.time_limit_string = case.wall_time_limit_string |
| 21 | 169 try: |
| 22 | 170 return case.test(callback) |
| 21 | 171 finally: |
| 22 | 172 now = clock() |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
173 if getattr(case, 'time_started', None) is None: |
| 22 | 174 case.time_started = case.time_stopped = now |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
175 elif getattr(case, 'time_stopped', None) is None: |
| 22 | 176 case.time_stopped = now |
| 177 if not case.has_called_back: | |
| 178 callback() | |
| 21 | 179 case.cleanup() |
| 180 | |
| 181 def cleanup(case): | |
| 182 #if getattr(case, 'infile', None): | |
| 183 # case.infile.close() | |
| 184 #if getattr(case, 'outfile', None): | |
| 185 # case.outfile.close() | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
186 if getattr(case, 'process', None) and case.process.returncode is None: |
|
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
187 # Try KILLing after three unsuccessful TERM attempts in a row |
| 21 | 188 for i in range(3): |
| 189 try: | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
190 terminate(case.process) |
| 21 | 191 except Exception: |
| 192 time.sleep(0) | |
| 193 case.process.poll() | |
| 194 else: | |
| 22 | 195 case.process.wait() |
| 21 | 196 break |
| 197 else: | |
| 198 # If killing the process is unsuccessful three times in a row, | |
| 199 # just silently stop trying | |
| 200 for i in range(3): | |
| 201 try: | |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
202 kill(case.process) |
| 21 | 203 except Exception: |
| 204 time.sleep(0) | |
| 205 case.process.poll() | |
| 206 else: | |
| 22 | 207 case.process.wait() |
| 21 | 208 break |
| 22 | 209 if case.files_to_delete: |
| 210 for name in case.files_to_delete: | |
| 211 try: | |
| 212 os.remove(name) | |
| 213 except Exception: | |
| 214 # It can't be helped | |
| 215 pass | |
| 21 | 216 |
| 217 def open_infile(case): | |
| 218 try: | |
| 219 case.infile = files.File('/'.join((case.problem.name, case.realinname.replace('$', case.id)))) | |
| 220 except IOError: | |
| 221 e = sys.exc_info()[1] | |
| 222 raise CannotReadInputFile(e) | |
| 223 | |
| 224 def open_outfile(case): | |
| 225 try: | |
| 226 case.outfile = files.File('/'.join((case.problem.name, case.realoutname.replace('$', case.id)))) | |
| 227 except IOError: | |
| 228 e = sys.exc_info()[1] | |
| 229 raise CannotReadAnswerFile(e) | |
| 230 | |
| 16 | 231 |
| 21 | 232 class ValidatedTestCase(TestCase): |
| 233 __slots__ = 'validator' | |
| 234 | |
| 235 def __init__(case, *args): | |
| 236 TestCase.__init__(case, *args) | |
| 237 if not case.problem.config.tester: | |
| 238 case.validator = None | |
| 239 else: | |
| 240 case.validator = case.problem.config.tester | |
| 241 | |
| 242 def validate(case, output): | |
| 243 if not case.validator: | |
| 244 # Compare the output with the reference output | |
| 245 case.open_outfile() | |
| 246 with case.outfile.open() as refoutput: | |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
247 for line, refline in zip_longest(output, refoutput): |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
248 if refline is not None and not isinstance(refline, basestring): |
| 21 | 249 line = bytes(line, sys.getdefaultencoding()) |
| 250 if line != refline: | |
| 22 | 251 raise WrongAnswer |
|
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
252 return 1 |
| 21 | 253 elif callable(case.validator): |
| 254 return case.validator(output) | |
| 255 else: | |
| 256 # Call the validator program | |
| 257 output.close() | |
| 23 | 258 if case.problem.config.ansname: |
| 259 case.open_outfile() | |
| 260 case.outfile.copy(case.problem.config.ansname) | |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
261 try: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
262 case.process = Popen(case.validator, stdin=devnull, stdout=PIPE, stderr=STDOUT, universal_newlines=True, bufsize=-1) |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
263 except OSError: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
264 raise CannotStartValidator(sys.exc_info()[1]) |
| 21 | 265 comment = case.process.communicate()[0].strip() |
| 26 | 266 match = re.match(r'(?i)(ok|(?:correct|wrong)(?:(?:\s|_)*answer)?)(?:$|\s+|[.,!:]+\s*)', comment) |
| 21 | 267 if match: |
| 268 comment = comment[match.end():] | |
| 269 if not case.problem.config.maxexitcode: | |
| 270 if case.process.returncode: | |
| 271 raise WrongAnswer(comment) | |
| 272 else: | |
|
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
273 return 1, comment |
| 21 | 274 else: |
|
24
c23d81f4a1a3
Score returned by TestCase.__call__() is now normalized to 0..1
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
275 return case.process.returncode / case.problem.config.maxexitcode, comment |
| 21 | 276 |
| 277 | |
| 278 class BatchTestCase(ValidatedTestCase): | |
| 279 __slots__ = () | |
| 280 | |
| 22 | 281 def test(case, callback): |
| 21 | 282 case.open_infile() |
| 283 case.time_started = None | |
| 284 if case.problem.config.stdio: | |
| 54 | 285 if options.erase and not case.validator or not case.problem.config.inname: |
| 22 | 286 # TODO: re-use the same file name if possible |
| 21 | 287 # FIXME: 2.5 lacks the delete parameter |
| 288 with tempfile.NamedTemporaryFile(delete=False) as f: | |
| 22 | 289 inputdatafname = f.name |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
290 contextmgr = CopyDeleting(case, case.infile, inputdatafname) |
| 21 | 291 else: |
| 292 inputdatafname = case.problem.config.inname | |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
293 contextmgr = Copying(case.infile, inputdatafname) |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
24
diff
changeset
|
294 with contextmgr: |
|
79
ee8a99dcaaed
Renamed configuration variable tasknames to problems
Oleg Oshmyan <chortos@inbox.lv>
parents:
77
diff
changeset
|
295 with open(inputdatafname) as infile: |
| 83 | 296 with tempfile.TemporaryFile('w+') if options.erase and (not case.validator or callable(case.validator)) else open(case.problem.config.outname, 'w+') as outfile: |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
297 call(case.problem.config.path, case=case, stdin=infile, stdout=outfile, stderr=devnull, universal_newlines=True, bufsize=-1) |
|
62
593ad09cd69b
Multiple exit code handling fixes
Oleg Oshmyan <chortos@inbox.lv>
parents:
61
diff
changeset
|
298 if config.globalconf.force_zero_exitcode and case.process.returncode or case.process.returncode < 0: |
| 22 | 299 raise NonZeroExitCode(case.process.returncode) |
| 300 callback() | |
| 301 case.has_called_back = True | |
| 302 outfile.seek(0) | |
| 303 return case.validate(outfile) | |
| 21 | 304 else: |
| 22 | 305 case.infile.copy(case.problem.config.inname) |
|
82
06356af50bf9
Finished testcases reorganization and CPU time limit implementation
Oleg Oshmyan <chortos@inbox.lv>
parents:
81
diff
changeset
|
306 call(case.problem.config.path, case=case, stdin=devnull, stdout=devnull, stderr=STDOUT) |
|
62
593ad09cd69b
Multiple exit code handling fixes
Oleg Oshmyan <chortos@inbox.lv>
parents:
61
diff
changeset
|
307 if config.globalconf.force_zero_exitcode and case.process.returncode or case.process.returncode < 0: |
| 21 | 308 raise NonZeroExitCode(case.process.returncode) |
| 22 | 309 callback() |
| 310 case.has_called_back = True | |
| 21 | 311 with open(case.problem.config.outname, 'rU') as output: |
| 312 return case.validate(output) | |
| 313 | |
| 314 | |
| 315 # This is the only test case type not executing any programs to be tested | |
| 316 class OutputOnlyTestCase(ValidatedTestCase): | |
| 317 __slots__ = () | |
| 318 def cleanup(case): pass | |
| 319 | |
| 320 class BestOutputTestCase(ValidatedTestCase): | |
| 321 __slots__ = () | |
| 322 | |
| 323 # This is the only test case type executing two programs simultaneously | |
| 324 class ReactiveTestCase(TestCase): | |
| 325 __slots__ = () | |
| 326 # The basic idea is to launch the program to be tested and the grader | |
| 327 # and to pipe their standard I/O from and to each other, | |
| 328 # and then to capture the grader's exit code and use it | |
| 26 | 329 # like the exit code of an output validator is used. |
| 21 | 330 |
| 331 | |
|
71
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
332 class DummyTestContext(problem.TestGroup): |
|
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
333 __slots__ = () |
|
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
334 def end(self): |
|
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
335 say('Sample total: %d/%d tests' % (self.ncorrect, self.ntotal)) |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
336 return 0, 0, self.log |
|
71
1bee3a0beeb5
Added a 'Sample total' line when using test groups
Oleg Oshmyan <chortos@inbox.lv>
parents:
69
diff
changeset
|
337 |
| 21 | 338 def load_problem(prob, _types={'batch' : BatchTestCase, |
| 339 'outonly' : OutputOnlyTestCase, | |
| 340 'bestout' : BestOutputTestCase, | |
| 341 'reactive': ReactiveTestCase}): | |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
342 # We will need to iterate over these configuration variables twice |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
343 try: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
344 len(prob.config.dummies) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
345 except Exception: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
346 prob.config.dummies = tuple(prob.config.dummies) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
347 try: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
348 len(prob.config.tests) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
349 except Exception: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
350 prob.config.tests = tuple(prob.config.tests) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
351 |
| 23 | 352 if options.legacy: |
| 353 prob.config.usegroups = False | |
| 58 | 354 newtests = [] |
| 23 | 355 for i, name in enumerate(prob.config.tests): |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
356 # Same here; we'll need to iterate over them twice |
| 23 | 357 try: |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
358 l = len(name) |
| 23 | 359 except Exception: |
| 360 try: | |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
361 name = tuple(name) |
| 23 | 362 except TypeError: |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
363 name = (name,) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
364 l = len(name) |
| 58 | 365 if l > 1: |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
366 prob.config.usegroups = True |
| 58 | 367 newtests.append(name) |
| 368 if prob.config.usegroups: | |
| 369 prob.config.tests = newtests | |
| 370 del newtests | |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
371 |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
372 # Even if they have duplicate test identifiers, we must honour sequence pointmaps |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
373 if isinstance(prob.config.pointmap, dict): |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
374 def getpoints(i, j, k=None): |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
375 try: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
376 return prob.config.pointmap[i] |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
377 except KeyError: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
378 try: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
379 return prob.config.pointmap[None] |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
380 except KeyError: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
381 return prob.config.maxexitcode or 1 |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
382 elif prob.config.usegroups: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
383 def getpoints(i, j, k): |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
384 try: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
385 return prob.config.pointmap[k][j] |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
386 except LookupError: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
387 return prob.config.maxexitcode or 1 |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
388 else: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
389 def getpoints(i, j): |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
390 try: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
391 return prob.config.pointmap[j] |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
392 except LookupError: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
393 return prob.config.maxexitcode or 1 |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
394 |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
395 # First get prob.cache.padoutput right, |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
396 # then yield the actual test cases |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
397 for i in prob.config.dummies: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
398 s = 'sample ' + str(i).zfill(prob.config.paddummies) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
399 prob.cache.padoutput = max(prob.cache.padoutput, len(s)) |
| 16 | 400 if prob.config.usegroups: |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
401 if not isinstance(prob.config.groupweight, dict): |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
402 prob.config.groupweight = dict(enumerate(prob.config.groupweight)) |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
403 for group in prob.config.tests: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
404 for i in group: |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
405 s = str(i).zfill(prob.config.padtests) |
|
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
406 prob.cache.padoutput = max(prob.cache.padoutput, len(s)) |
|
89
3ae6cb69e4ef
Sample total is no longer printed if there are no samples
Oleg Oshmyan <chortos@inbox.lv>
parents:
88
diff
changeset
|
407 if prob.config.dummies: |
|
3ae6cb69e4ef
Sample total is no longer printed if there are no samples
Oleg Oshmyan <chortos@inbox.lv>
parents:
88
diff
changeset
|
408 yield DummyTestContext() |
|
3ae6cb69e4ef
Sample total is no longer printed if there are no samples
Oleg Oshmyan <chortos@inbox.lv>
parents:
88
diff
changeset
|
409 for i in prob.config.dummies: |
|
3ae6cb69e4ef
Sample total is no longer printed if there are no samples
Oleg Oshmyan <chortos@inbox.lv>
parents:
88
diff
changeset
|
410 s = str(i).zfill(prob.config.paddummies) |
|
3ae6cb69e4ef
Sample total is no longer printed if there are no samples
Oleg Oshmyan <chortos@inbox.lv>
parents:
88
diff
changeset
|
411 yield _types[prob.config.kind](prob, s, True, 0) |
|
3ae6cb69e4ef
Sample total is no longer printed if there are no samples
Oleg Oshmyan <chortos@inbox.lv>
parents:
88
diff
changeset
|
412 yield problem.test_context_end |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
413 for k, group in enumerate(prob.config.tests): |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
414 if not group: |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
415 continue |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
416 yield problem.TestGroup(prob.config.groupweight.get(k, prob.config.groupweight.get(None))) |
|
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
417 for j, i in enumerate(group): |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
418 s = str(i).zfill(prob.config.padtests) |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
419 yield _types[prob.config.kind](prob, s, False, getpoints(i, j, k)) |
|
39
2b459f9743b4
Test groups are now supported
Oleg Oshmyan <chortos@inbox.lv>
parents:
27
diff
changeset
|
420 yield problem.test_context_end |
| 16 | 421 else: |
| 422 for i in prob.config.tests: | |
| 21 | 423 s = str(i).zfill(prob.config.padtests) |
| 424 prob.cache.padoutput = max(prob.cache.padoutput, len(s)) | |
| 425 for i in prob.config.dummies: | |
| 426 s = str(i).zfill(prob.config.paddummies) | |
| 427 yield _types[prob.config.kind](prob, s, True, 0) | |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
428 for j, i in enumerate(prob.config.tests): |
| 21 | 429 s = str(i).zfill(prob.config.padtests) |
|
76
0e5ae28e0b2b
Points are now weighted on a test context basis
Oleg Oshmyan <chortos@inbox.lv>
parents:
72
diff
changeset
|
430 yield _types[prob.config.kind](prob, s, False, getpoints(i, j)) |