Mercurial > ~astiob > upreckon > hgweb
annotate test.py @ 5:eb15a5a9b026
Output validators can now award partial scores
Added the maxexitcode configuration option.
Output validators can now cause a fraction of the points allocated for the test case to be awarded (exitcode/maxexitcode is the fraction, ranging from 0 to 1).
| author | Oleg Oshmyan <chortos@inbox.lv> |
|---|---|
| date | Sun, 14 Feb 2010 00:08:16 +0000 |
| parents | 06eef313aeaa |
| children | b0034b18f942 |
| rev | line source |
|---|---|
| 0 | 1 #! /usr/bin/python |
| 3 | 2 # Copyright (c) 2009-2010 Chortos-2 <chortos@inbox.lv> |
| 0 | 3 |
| 4 import os, sys, shutil, time, subprocess, filecmp, optparse, signal, tempfile, tarfile, zipfile | |
| 5 | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
6 parser = optparse.OptionParser(version='test.py 1.21.0 (SVN)', usage='usage: %prog [options] [problem names] [[path' + os.path.sep + 'to' + os.path.sep + ']solution-app] [test case numbers]\n\nTest case numbers can be specified in plain text or as a Python expression\nif there is only one positional argument.\n\nOnly problem names listed in testconf.py are recognized.') |
| 0 | 7 parser.add_option('-e', '--exclude', dest='exclude', action='append', help='test case number(s) to exclude, as a Python expression; multiple -e options can be supplied') |
| 8 parser.add_option('-c', '--cleanup', dest='clean', action='store_true', default=False, help='delete the copies of input/output files and exit') | |
| 9 parser.add_option('-s', '--save-io', dest='erase', action='store_false', default=True, help='do not delete the copies of input/output files after the last test case; create copies of input files and store output in files even if the solution uses standard I/O; delete the stored input/output files if the solution uses standard I/O and the -c/--cleanup option is specified') | |
| 4 | 10 parser.add_option('-m', '--copy-io', dest='copyonly', action='store_true', default=False, help='only create a copy of the input/output files of the last test case for manual testing; to delete them, use options -cs or -cm') |
| 0 | 11 parser.add_option('-x', '--auto-exit', dest='pause', action='store_false', default=True, help='do not wait for a key to be pressed when finished testing') |
| 12 parser.add_option('-p', '--python', action='store_true', default=False, help='always parse all positional arguments as a single Python expression (including the first argument even if it names an executable file)') | |
| 13 parser.add_option('-t', '--detect-time', dest='autotime', action='store_true', default=False, help='spend a second detecting the most precise time measurement function') | |
| 14 | |
| 15 options, args = parser.parse_args() | |
| 16 parser.destroy() | |
| 17 del parser | |
| 18 | |
| 19 globals1 = set(globals()) | |
| 20 | |
| 21 # Initialize some configuration variables with default values | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
22 tasknames = (os.path.curdir,) |
| 0 | 23 maxtime = 0 |
| 24 tests = () | |
| 25 dummies = () | |
| 26 testsexcluded = () | |
| 27 padwithzeroestolength = 0 | |
| 28 taskweight = 100 | |
| 29 pointmap = {} | |
| 30 stdio = False | |
| 31 dummyinname = '' | |
| 32 dummyoutname = '' | |
| 33 tester = '' | |
|
5
eb15a5a9b026
Output validators can now award partial scores
Oleg Oshmyan <chortos@inbox.lv>
parents:
4
diff
changeset
|
34 maxexitcode = 0 |
| 0 | 35 |
| 36 def exectestconf_helper(name): | |
| 37 if os.path.isfile('tests.tar'): | |
| 38 f = tarfile.open('tests.tar') | |
| 39 try: | |
| 40 exec f.extractfile(name).read() in globals() | |
| 41 f.close() | |
| 42 return True | |
| 43 except KeyError: | |
| 44 f.close() | |
| 45 if os.path.isfile('tests.zip'): | |
| 46 f = zipfile.ZipFile('tests.zip') | |
| 47 try: | |
| 48 exec f.open(name, 'rU').read() in globals() | |
| 49 f.close() | |
| 50 return True | |
| 51 except KeyError: | |
| 52 f.close() | |
| 53 if os.path.isfile('tests.tgz'): | |
| 54 f = tarfile.open('tests.tgz') | |
| 55 try: | |
| 56 exec f.extractfile(name).read() in globals() | |
| 57 f.close() | |
| 58 return True | |
| 59 except KeyError: | |
| 60 f.close() | |
| 61 if os.path.isfile('tests.tar.gz'): | |
| 62 f = tarfile.open('tests.tar.gz') | |
| 63 try: | |
| 64 exec f.extractfile(name).read() in globals() | |
| 65 f.close() | |
| 66 return True | |
| 67 except KeyError: | |
| 68 f.close() | |
| 69 if os.path.isfile('tests.tbz2'): | |
| 70 f = tarfile.open('tests.tbz2') | |
| 71 try: | |
| 72 exec f.extractfile(name).read() in globals() | |
| 73 f.close() | |
| 74 return True | |
| 75 except KeyError: | |
| 76 f.close() | |
| 77 if os.path.isfile('tests.tar.bz2'): | |
| 78 f = tarfile.open('tests.tar.bz2') | |
| 79 try: | |
| 80 exec f.extractfile(name).read() in globals() | |
| 81 f.close() | |
| 82 return True | |
| 83 except KeyError: | |
| 84 f.close() | |
| 85 return False | |
| 86 | |
| 87 try: | |
| 88 execfile('testconf.py') | |
| 89 except IOError, error: | |
| 90 exc_info = sys.exc_info()[2] | |
| 91 try: | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
92 execfile(os.path.join('tests', 'testconf.py')) |
| 0 | 93 except IOError: |
| 94 if not exectestconf_helper('testconf.py'): | |
| 95 raise IOError, (error.errno, 'The configuration file is missing', error.filename), exc_info | |
| 96 del exc_info | |
| 97 | |
| 98 globals2 = set(globals()) | |
| 99 globals2.remove('globals1') | |
| 100 globals2 -= globals1 | |
| 101 del globals1 | |
| 102 | |
| 103 shared = {} | |
| 104 g = globals() | |
| 105 for k in globals2: | |
| 106 shared[k] = g[k] | |
| 107 | |
| 108 newtasknames = [] | |
| 109 while len(args) and args[0] in tasknames: | |
| 110 newtasknames.append(args[0]) | |
| 111 del args[0] | |
| 112 if len(newtasknames): | |
| 113 tasknames = newtasknames | |
| 114 | |
| 115 scoresumoveralltasks = 0 | |
| 116 scoremaxoveralltasks = 0 | |
| 117 ntasks = 0 | |
| 118 nfulltasks = 0 | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
119 cwd = '' # At any time this is either '' or taskname |
| 0 | 120 |
| 121 if options.autotime: | |
| 122 c = time.clock() | |
| 123 time.sleep(1) | |
| 124 c = time.clock() - c | |
| 125 if int(c + .99999) == 1: | |
| 126 clock = time.clock | |
| 127 else: | |
| 128 clock = time.time | |
| 129 elif os.name == 'nt': | |
| 130 clock = time.clock | |
| 131 else: | |
| 132 clock = time.time | |
| 133 | |
| 134 if options.copyonly: | |
| 135 options.erase = False | |
| 136 | |
| 137 def existstestcase_helper(name): | |
| 138 if os.path.isfile('tests.tar'): | |
| 139 f = tarfile.open('tests.tar') | |
| 140 try: | |
| 141 f.getmember(name) | |
| 142 f.close() | |
| 143 return True | |
| 144 except KeyError: | |
| 145 f.close() | |
| 146 if os.path.isfile('tests.zip'): | |
| 147 f = zipfile.ZipFile('tests.zip') | |
| 148 try: | |
| 149 f.getinfo(name) | |
| 150 f.close() | |
| 151 return True | |
| 152 except KeyError: | |
| 153 f.close() | |
| 154 if os.path.isfile('tests.tgz'): | |
| 155 f = tarfile.open('tests.tgz') | |
| 156 try: | |
| 157 f.getmember(name) | |
| 158 f.close() | |
| 159 return True | |
| 160 except KeyError: | |
| 161 f.close() | |
| 162 if os.path.isfile('tests.tar.gz'): | |
| 163 f = tarfile.open('tests.tar.gz') | |
| 164 try: | |
| 165 f.getmember(name) | |
| 166 f.close() | |
| 167 return True | |
| 168 except KeyError: | |
| 169 f.close() | |
| 170 if os.path.isfile('tests.tbz2'): | |
| 171 f = tarfile.open('tests.tbz2') | |
| 172 try: | |
| 173 f.getmember(name) | |
| 174 f.close() | |
| 175 return True | |
| 176 except KeyError: | |
| 177 f.close() | |
| 178 if os.path.isfile('tests.tar.bz2'): | |
| 179 f = tarfile.open('tests.tar.bz2') | |
| 180 try: | |
| 181 f.getmember(name) | |
| 182 f.close() | |
| 183 return True | |
| 184 except KeyError: | |
| 185 f.close() | |
| 186 return False | |
| 187 | |
| 188 def existstestcase(name): | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
189 if os.path.isfile(os.path.join('tests', taskname, name)) or os.path.isfile(os.path.join('tests', name)): |
| 0 | 190 return True |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
191 if cwd and (os.path.isfile(os.path.join(oldcwd, 'tests', cwd, name)) or os.path.isfile(os.path.join(oldcwd, 'tests', name))): |
| 0 | 192 return True |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
193 if existstestcase_helper(os.path.join(taskname, name)) or existstestcase_helper(name): |
| 0 | 194 return True |
| 195 if cwd: | |
| 196 os.chdir(oldcwd) | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
197 if existstestcase_helper(os.path.join(cwd, name)) or existstestcase_helper(name): |
| 0 | 198 os.chdir(cwd) |
| 199 return True | |
| 200 os.chdir(cwd) | |
| 201 return False | |
| 202 | |
| 203 def opentestcase_helper(name): | |
| 204 if os.path.isfile('tests.tar'): | |
| 205 f = tarfile.open('tests.tar') | |
| 206 try: | |
| 207 c = f.extractfile(name) | |
| 208 return c | |
| 209 except KeyError: | |
| 210 f.close() | |
| 211 if os.path.isfile('tests.zip'): | |
| 212 f = zipfile.ZipFile('tests.zip') | |
| 213 try: | |
| 214 c = f.open(name, 'rU') | |
| 215 f.close() | |
| 216 return c | |
| 217 except KeyError: | |
| 218 f.close() | |
| 219 if os.path.isfile('tests.tgz'): | |
| 220 f = tarfile.open('tests.tgz') | |
| 221 try: | |
| 222 c = f.extractfile(name) | |
| 223 return c | |
| 224 except KeyError: | |
| 225 f.close() | |
| 226 if os.path.isfile('tests.tar.gz'): | |
| 227 f = tarfile.open('tests.tar.gz') | |
| 228 try: | |
| 229 c = f.extractfile(name) | |
| 230 return c | |
| 231 except KeyError: | |
| 232 f.close() | |
| 233 if os.path.isfile('tests.tbz2'): | |
| 234 f = tarfile.open('tests.tbz2') | |
| 235 try: | |
| 236 c = f.extractfile(name) | |
| 237 return c | |
| 238 except KeyError: | |
| 239 f.close() | |
| 240 if os.path.isfile('tests.tar.bz2'): | |
| 241 f = tarfile.open('tests.tar.bz2') | |
| 242 try: | |
| 243 c = f.extractfile(name) | |
| 244 return c | |
| 245 except KeyError: | |
| 246 f.close() | |
| 247 return None | |
| 248 | |
| 249 def opentestcase(name): | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
250 if os.path.isfile(os.path.join('tests', taskname, name)): |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
251 return open(os.path.join('tests', taskname, name), 'rU') |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
252 elif os.path.isfile(os.path.join('tests', name)): |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
253 return open(os.path.join('tests', name), 'rU') |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
254 f = opentestcase_helper(os.path.join(taskname, name)) |
| 0 | 255 if not f: |
| 256 f = opentestcase_helper(name) | |
| 257 if f: | |
| 258 return f | |
| 259 if cwd: | |
| 3 | 260 if os.path.isfile(os.path.join(oldcwd, 'tests', cwd, name)): |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
261 return open(os.path.join(oldcwd, 'tests', cwd, name), 'rU') |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
262 elif os.path.isfile(os.path.join(oldcwd, 'tests', name)): |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
263 return open(os.path.join(oldcwd, 'tests', name), 'rU') |
| 0 | 264 os.chdir(oldcwd) |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
265 f = opentestcase_helper(os.path.join(cwd, name)) |
| 0 | 266 if not f: |
| 267 f = opentestcase_helper(name) | |
| 268 os.chdir(cwd) | |
| 269 if f: | |
| 270 return f | |
| 271 raise KeyError, 'The test-case-defining file \'' + name + '\' cannot be found' | |
| 272 | |
| 273 def copytestcase_helper(name, target): | |
| 274 if os.path.isfile('tests.tar'): | |
| 275 f = tarfile.open('tests.tar') | |
| 276 try: | |
| 277 m = f.getmember(name) | |
| 278 m.name = target | |
| 279 f.extract(m) | |
| 280 f.close() | |
| 281 return True | |
| 282 except KeyError: | |
| 283 f.close() | |
| 284 if os.path.isfile('tests.zip'): | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
285 if not os.path.isabs(target): |
| 0 | 286 f = zipfile.ZipFile('tests.zip') |
| 287 m = f.getinfo(name) | |
| 288 try: | |
| 289 m.filename = target | |
| 290 f.extract(m) | |
| 291 f.close() | |
| 292 return True | |
| 293 except KeyError: | |
| 294 f.close() | |
| 295 else: | |
| 296 oldcwd = os.getcwdu() | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
297 os.chdir('/') # FIXME: portability? |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
298 f = zipfile.ZipFile(os.path.join(oldcwd, 'tests.zip')) |
| 0 | 299 try: |
| 300 m = f.getinfo(name) | |
| 301 m.filename = target[1:] | |
| 302 f.extract(m) | |
| 303 f.close() | |
| 304 os.chdir(oldcwd) | |
| 305 return True | |
| 306 except KeyError: | |
| 307 f.close() | |
| 308 os.chdir(oldwcd) | |
| 309 if os.path.isfile('tests.tgz'): | |
| 310 f = tarfile.open('tests.tgz') | |
| 311 try: | |
| 312 m = f.getmember(name) | |
| 313 m.name = target | |
| 314 f.extract(m) | |
| 315 f.close() | |
| 316 return True | |
| 317 except KeyError: | |
| 318 f.close() | |
| 319 if os.path.isfile('tests.tar.gz'): | |
| 320 f = tarfile.open('tests.tar.gz') | |
| 321 try: | |
| 322 m = f.getmember(name) | |
| 323 m.name = target | |
| 324 f.extract(m) | |
| 325 f.close() | |
| 326 return True | |
| 327 except KeyError: | |
| 328 f.close() | |
| 329 if os.path.isfile('tests.tbz2'): | |
| 330 f = tarfile.open('tests.tbz2') | |
| 331 try: | |
| 332 m = f.getmember(name) | |
| 333 m.name = target | |
| 334 f.extract(m) | |
| 335 f.close() | |
| 336 return True | |
| 337 except KeyError: | |
| 338 f.close() | |
| 339 if os.path.isfile('tests.tar.bz2'): | |
| 340 f = tarfile.open('tests.tar.bz2') | |
| 341 try: | |
| 342 m = f.getmember(name) | |
| 343 m.name = target | |
| 344 f.extract(m) | |
| 345 f.close() | |
| 346 return True | |
| 347 except KeyError: | |
| 348 f.close() | |
| 349 return False | |
| 350 | |
| 351 def copytestcase(name, target): | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
352 if os.path.isfile(os.path.join('tests', taskname, name)): |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
353 shutil.copyfile(os.path.join('tests', taskname, name), target) |
| 0 | 354 return |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
355 elif os.path.isfile(os.path.join('tests', name)): |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
356 shutil.copyfile(os.path.join('tests', name), target) |
| 0 | 357 return |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
358 if copytestcase_helper(os.path.join(taskname, name), target) or copytestcase_helper(name, target): |
| 0 | 359 return |
| 360 if cwd: | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
361 if os.path.isfile(os.path.join(oldcwd, 'tests', cwd, name)): |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
362 shutil.copyfile(os.path.join(oldcwd, 'tests', cwd, name), target) |
| 0 | 363 return |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
364 elif os.path.isfile(os.path.join(oldcwd, 'tests', name)): |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
365 shutil.copyfile(os.path.join(oldcwd, 'tests', name), target) |
| 0 | 366 return |
| 367 os.chdir(oldcwd) | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
368 if copytestcase_helper(os.path.join(cwd, name), target) or copytestcase_helper(name, target): |
| 0 | 369 os.chdir(cwd) |
| 370 return | |
| 371 os.chdir(cwd) | |
| 372 raise KeyError, 'The test-case-defining file \'' + name + '\' cannot be found' | |
| 373 | |
| 374 # Always chdir if the directory exists but use any existing config | |
| 375 def chdir_and_exec_testconf(): | |
| 376 global cwd | |
| 377 cwd = '' | |
| 378 if os.path.isdir(taskname): | |
| 379 os.chdir(taskname) | |
| 3 | 380 if taskname != os.path.curdir: |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
381 cwd = taskname |
| 0 | 382 try: |
| 383 execfile('testconf.py', globals()) | |
| 384 return | |
| 385 except IOError: | |
| 386 pass | |
| 387 if not cwd: | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
388 if os.path.isfile(os.path.join('tests', taskname, 'testconf.py')): |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
389 execfile(os.path.join('tests', taskname, 'testconf.py'), globals()) |
| 0 | 390 return |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
391 if os.path.isfile(os.path.join('tests', 'testconf.py')): |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
392 execfile(os.path.join('tests', 'testconf.py'), globals()) |
| 0 | 393 return |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
394 if exectestconf_helper(os.path.join(taskname, 'testconf.py')) or exectestconf_helper('testconf.py'): |
| 0 | 395 return |
| 396 if cwd: | |
| 397 os.chdir(oldcwd) | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
398 if os.path.isfile(os.path.join('tests', cwd, 'testconf.py')): |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
399 execfile(os.path.join('tests', cwd, 'testconf.py'), globals()) |
| 0 | 400 os.chdir(cwd) |
| 401 return | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
402 if os.path.isfile(os.path.join('tests', 'testconf.py')): |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
403 execfile(os.path.join('tests', 'testconf.py'), globals()) |
| 0 | 404 os.chdir(cwd) |
| 405 return | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
406 if exectestconf_helper(os.path.join(cwd, 'testconf.py')) or exectestconf_helper('testconf.py'): |
| 0 | 407 os.chdir(cwd) |
| 408 return | |
| 409 if os.path.isfile('testconf.py'): | |
| 410 execfile('testconf.py', globals()) | |
| 411 os.chdir(cwd) | |
| 412 return | |
| 413 os.chdir(cwd) | |
| 414 elif os.path.isfile('testconf.py'): | |
| 415 execfile('testconf.py', globals()) | |
| 416 return | |
| 417 raise KeyError, 'The configuration file for task ' + taskname + ' is missing' | |
| 418 | |
| 419 try: | |
| 420 name | |
| 421 namedefined = True | |
| 422 except Exception: | |
| 423 namedefined = False | |
| 424 | |
| 425 for taskname in tasknames: | |
| 426 if ntasks: | |
| 427 print | |
| 428 | |
| 429 try: | |
| 430 if len(tasknames) > 1: | |
| 431 print taskname | |
| 432 except Exception: | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
433 if taskname != os.path.curdir or ntasks: |
| 0 | 434 print taskname |
| 435 | |
| 436 try: del inname | |
| 437 except NameError: pass | |
| 438 try: del outname | |
| 439 except NameError: pass | |
| 440 try: del ansname | |
| 441 except NameError: pass | |
| 442 | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
443 if not namedefined and taskname != os.path.curdir: |
|
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
444 name = os.path.join(os.path.curdir, taskname) |
| 0 | 445 for k in shared: |
| 446 g[k] = shared[k] | |
| 447 | |
| 448 oldcwd = os.getcwdu() | |
| 449 chdir_and_exec_testconf() | |
| 450 | |
| 451 if options.clean: | |
| 452 try: | |
| 453 if not stdio or tester: | |
| 454 if not tester: | |
| 455 inname | |
| 456 outname | |
| 457 if tester: | |
| 458 ansname | |
| 459 except NameError, error: | |
| 460 raise NameError, 'configuration ' + str(error).replace('name ', 'variable ', 1), sys.exc_info()[2] | |
| 461 if not options.erase: | |
| 462 try: | |
| 463 inname = inname.replace('%', taskname) | |
| 464 except NameError: | |
| 465 inname = taskname + '.in' | |
| 466 try: | |
| 467 outname = outname.replace('%', taskname) | |
| 468 except NameError: | |
| 469 outname = taskname + '.out' | |
| 470 try: | |
| 471 ansname = ansname.replace('%', taskname) | |
| 472 except NameError: | |
| 473 ansname = taskname + '.ans' | |
| 4 | 474 elif not stdio or tester or not options.erase: |
| 0 | 475 inname = inname.replace('%', taskname) |
| 476 outname = outname.replace('%', taskname) | |
| 477 if tester: | |
| 478 ansname = ansname.replace('%', taskname) | |
| 479 if not stdio or tester or not options.erase: | |
| 480 if os.path.exists(inname): os.remove(inname) | |
| 481 if os.path.exists(outname): os.remove(outname) | |
| 482 if (tester or not options.erase) and ansname: | |
| 483 if os.path.exists(ansname): os.remove(ansname) | |
| 484 continue | |
| 485 | |
| 486 try: | |
| 487 name | |
| 488 except NameError, error: | |
| 489 if str(error).count('name') == 1: | |
| 490 raise NameError, 'configuration ' + str(error), sys.exc_info()[2] | |
| 491 else: | |
| 492 raise NameError, 'configuration ' + str(error).replace('name ', 'variable ', 1), sys.exc_info()[2] | |
| 493 | |
| 494 try: | |
| 495 if not stdio: | |
| 496 inname | |
| 497 outname | |
| 498 testcaseinname | |
| 499 if tester: | |
| 500 outname | |
| 501 if ansname: | |
| 502 testcaseoutname | |
| 503 else: | |
| 504 testcaseoutname | |
| 505 except NameError, error: | |
| 506 raise NameError, 'configuration ' + str(error).replace('name ', 'variable ', 1), sys.exc_info()[2] | |
| 507 | |
| 508 if not options.erase: | |
| 509 try: | |
| 510 inname | |
| 511 except NameError: | |
| 512 inname = taskname + '.in' | |
| 513 try: | |
| 514 outname | |
| 515 except NameError: | |
| 516 outname = taskname + '.out' | |
| 517 try: | |
| 518 ansname | |
| 519 except NameError: | |
| 520 ansname = taskname + '.ans' | |
| 521 | |
| 522 if options.pause: | |
| 523 try: | |
| 524 pause | |
| 525 except NameError, error: | |
| 526 if os.name == 'posix': | |
| 527 pause = 'read -s -n 1' | |
| 3 | 528 print 'Configuration ' + str(error).replace('name ', 'variable ') + '; it was devised automatically but the choice might be incorrect, so test.py might exit immediately after the testing is completed.' |
| 0 | 529 elif os.name == 'nt': |
| 530 pause = 'pause' | |
| 531 else: | |
| 532 raise NameError, 'configuration ' + str(error).replace('name ', 'variable ') + ' and cannot be devised automatically', sys.exc_info()[2] | |
| 533 | |
| 534 if not dummyinname: | |
| 535 dummyinname = testcaseinname | |
| 536 if not dummyoutname and (not tester or ansname): | |
| 537 dummyoutname = testcaseoutname | |
| 538 | |
| 539 dummyinname = dummyinname.replace('%', taskname) | |
| 540 dummyoutname = dummyoutname.replace('%', taskname) | |
| 541 testcaseinname = testcaseinname.replace('%', taskname) | |
| 542 if not stdio or not options.erase: | |
| 543 inname = inname.replace('%', taskname) | |
| 544 outname = outname.replace('%', taskname) | |
| 545 try: | |
| 546 ansname = ansname.replace('%', taskname) | |
| 547 except NameError: | |
| 548 pass | |
| 549 if tester: | |
| 550 try: inname = inname.replace('%', taskname) | |
| 551 except NameError: pass | |
| 552 outname = outname.replace('%', taskname) | |
| 553 if ansname: | |
| 554 ansname = ansname.replace('%', taskname) | |
| 555 testcaseoutname = testcaseoutname.replace('%', taskname) | |
| 556 else: | |
| 557 testcaseoutname = testcaseoutname.replace('%', taskname) | |
| 558 | |
| 559 if isinstance(padwithzeroestolength, tuple): | |
| 560 padwithzeroestolength, paddummieswithzeroestolength = padwithzeroestolength | |
| 561 else: | |
| 562 paddummieswithzeroestolength = padwithzeroestolength | |
| 563 | |
| 564 if options.python: | |
| 565 dummies = () | |
| 566 s = ' '.join(args) | |
| 567 tests = eval(s) | |
| 568 try: | |
| 569 tests.__iter__ | |
| 570 except AttributeError: | |
| 571 tests = (tests,) | |
| 572 elif len(args): | |
| 573 if os.path.exists(args[0]): | |
| 574 name = args[0] | |
| 575 del args[0] | |
| 576 if len(args) > 1: | |
| 577 dummies = () | |
| 578 tests = args | |
| 579 elif len(args): | |
| 580 dummies = () | |
| 581 s = args[0] | |
| 582 if len(s) < padwithzeroestolength: | |
| 583 s = s.zfill(padwithzeroestolength) | |
| 584 if existstestcase(testcaseinname.replace('$', s)): | |
| 585 tests = (s,) | |
| 586 else: | |
| 587 try: | |
| 588 tests = eval(args[0]) | |
| 589 try: | |
| 590 tests.__iter__ | |
| 591 except AttributeError: | |
| 592 tests = (tests,) | |
| 593 except Exception: | |
| 594 tests = (s,) | |
| 595 | |
| 596 if options.exclude: | |
| 597 testsexcluded = [] | |
| 598 for i in options.exclude: | |
| 599 v = eval(i) | |
| 600 try: | |
| 601 testsexcluded.extend(v) | |
| 602 except TypeError: | |
| 603 testsexcluded.append(v) | |
| 604 | |
| 605 # Windows doesn't like paths beginning with .\ and not ending with an extension | |
| 606 name = os.path.normcase(name) | |
|
2
bddcc05aba59
Finished path portability improvements
Oleg Oshmyan <chortos@inbox.lv>
parents:
1
diff
changeset
|
607 if os.name == 'nt' and name.startswith('.\\'): |
| 0 | 608 name = name[2:] |
| 609 | |
| 610 newpointmap = {} | |
| 611 | |
| 612 for i in pointmap: | |
| 613 try: | |
| 614 for j in i: | |
| 615 newpointmap[j] = pointmap[i] | |
| 616 except TypeError: | |
| 617 newpointmap[i] = pointmap[i] | |
| 618 | |
| 619 pointmap = newpointmap | |
| 620 | |
| 621 if maxtime > 0: | |
| 622 strmaxtime = '/%.3f' % maxtime | |
| 623 else: | |
| 624 strmaxtime = '' | |
| 625 | |
| 626 padoutputtolength = 0 | |
| 627 ntests = [] | |
| 628 | |
| 629 for j in dummies: | |
| 630 try: | |
| 631 j.__iter__ | |
| 632 except AttributeError: | |
| 633 j = (j,) | |
| 634 ntests.append((j, True)) | |
| 635 for i in j: | |
| 636 s = str(i) | |
| 637 if len(s) < paddummieswithzeroestolength: | |
| 638 s = s.zfill(paddummieswithzeroestolength) | |
| 639 s = 'sample ' + s | |
| 640 if padoutputtolength < len(s): | |
| 641 padoutputtolength = len(s) | |
| 642 | |
| 643 for j in tests: | |
| 644 try: | |
| 645 j.__iter__ | |
| 646 except AttributeError: | |
| 647 j = (j,) | |
| 648 ntests.append((j, False)) | |
| 649 for i in j: | |
| 650 s = str(i) | |
| 651 if len(s) < padwithzeroestolength: | |
| 652 s = s.zfill(padwithzeroestolength) | |
| 653 if padoutputtolength < len(s): | |
| 654 padoutputtolength = len(s) | |
| 655 | |
| 656 tests = ntests | |
| 657 score = maxpoints = ncorrect = ntotal = ncorrectvalued = nvalued = 0 | |
| 658 | |
| 659 if options.copyonly: | |
| 660 j, isdummy = tests[-1] | |
| 661 if isdummy: | |
| 662 realinname = dummyinname | |
| 663 realoutname = dummyoutname | |
| 664 else: | |
| 665 realinname = testcaseinname | |
| 666 realoutname = testcaseoutname | |
| 667 for i in j: | |
| 668 if i in testsexcluded and not isdummy: | |
| 669 continue | |
| 670 s = str(i) | |
| 671 if isdummy: | |
| 672 if len(s) < paddummieswithzeroestolength: | |
| 673 s = s.zfill(paddummieswithzeroestolength) | |
| 674 else: | |
| 675 if len(s) < padwithzeroestolength: | |
| 676 s = s.zfill(padwithzeroestolength) | |
| 677 copytestcase(realinname.replace('$', s), inname) | |
| 678 if ansname: | |
| 679 copytestcase(realoutname.replace('$', s), ansname) | |
| 680 continue | |
| 681 | |
| 682 for j, isdummy in tests: | |
| 683 ncorrectgrp = 0 | |
| 684 ntotalgrp = 0 | |
| 685 scoregrp = 0 | |
| 686 maxpointsgrp = 0 | |
| 687 if isdummy: | |
| 688 realinname = dummyinname | |
| 689 realoutname = dummyoutname | |
| 690 else: | |
| 691 realinname = testcaseinname | |
| 692 realoutname = testcaseoutname | |
| 693 for i in j: | |
| 694 if i in testsexcluded and not isdummy: | |
| 695 continue | |
| 696 ntotalgrp += 1 | |
| 697 s = str(i) | |
| 698 if isdummy: | |
| 699 npoints = 0 | |
| 700 if len(s) < paddummieswithzeroestolength: | |
| 701 s = s.zfill(paddummieswithzeroestolength) | |
| 702 spref = 'sample ' | |
| 703 else: | |
| 704 npoints = pointmap.get(None, 1) | |
| 705 npoints = pointmap.get(i, npoints) | |
| 706 maxpointsgrp += npoints | |
| 707 if npoints: | |
| 708 nvalued += 1 | |
| 709 if len(s) < padwithzeroestolength: | |
| 710 s = s.zfill(padwithzeroestolength) | |
| 711 spref = '' | |
| 712 print ' ' * (padoutputtolength - len(spref + s)) + spref + s + ':', | |
| 713 sys.stdout.flush() | |
| 714 outputdata = open(os.devnull, 'w') | |
| 715 if stdio: | |
| 716 f = tempfile.NamedTemporaryFile(delete=False) | |
| 717 inputdatafname = f.name | |
| 718 f.close() | |
| 719 copytestcase(realinname.replace('$', s), inputdatafname) | |
| 720 inputdata = open(inputdatafname, 'rU') | |
| 721 if options.erase: | |
| 722 tempoutput = tempfile.TemporaryFile('w+') | |
| 723 else: | |
| 724 tempoutput = open(outname, 'w+') | |
| 725 try: | |
| 726 proc = subprocess.Popen(name, stdin=inputdata, stdout=tempoutput, stderr=outputdata, universal_newlines=True) | |
| 727 except OSError, error: | |
| 728 raise OSError, 'The program to be tested cannot be launched: ' + str(error), sys.exc_info()[2] | |
| 729 else: | |
| 730 if os.path.exists(outname): | |
| 731 os.remove(outname) | |
| 732 copytestcase(realinname.replace('$', s), inname) | |
| 733 try: | |
| 734 proc = subprocess.Popen(name, stdin=outputdata, stdout=outputdata, stderr=outputdata, universal_newlines=True) | |
| 735 except OSError, error: | |
| 736 raise OSError, 'The program to be tested cannot be launched: ' + str(error), sys.exc_info()[2] | |
| 737 cl = clock() | |
| 738 if maxtime > 0: | |
| 739 while 1: | |
| 740 proc.poll() | |
| 741 elapsed = clock() - cl | |
| 742 if proc.returncode == None: | |
| 743 if elapsed >= maxtime: | |
| 744 print '%.3f%s s, 0/%d, time limit exceeded' % (elapsed, strmaxtime, npoints) | |
| 745 sys.stdout.flush() | |
| 746 while proc.returncode == None: | |
| 747 try: | |
| 748 proc.terminate() | |
| 749 except OSError: | |
| 750 pass | |
| 751 except AttributeError: | |
| 752 try: | |
| 753 os.kill(proc.pid, signal.SIGTERM) | |
| 754 except Exception: | |
| 755 pass | |
| 756 proc.poll() | |
| 757 outputdata.close() | |
| 758 if stdio: | |
| 759 tempoutput.close() | |
| 760 break | |
| 761 else: | |
| 762 print '%.3f%s s,' % (elapsed, strmaxtime), | |
| 763 sys.stdout.flush() | |
| 764 elapsed = 0 | |
| 765 if stdio: | |
| 766 tempoutput.seek(0) | |
| 767 lines = tempoutput.readlines() | |
| 768 tempoutput.close() | |
| 769 break | |
| 770 if elapsed >= maxtime: | |
| 771 continue | |
| 772 else: | |
| 773 data = proc.communicate() | |
| 774 elapsed = clock() - cl | |
| 775 print '%.3f%s s,' % (elapsed, strmaxtime), | |
| 776 sys.stdout.flush() | |
| 777 if stdio: | |
| 778 tempoutput.seek(0) | |
| 779 lines = tempoutput.readlines() | |
| 780 tempoutput.close() | |
| 781 outputdata.close() | |
| 782 if stdio: | |
| 783 inputdata.close() | |
| 784 try: | |
| 785 os.unlink(inputdatafname) | |
| 786 except Exception: | |
| 787 pass | |
| 788 if proc.returncode > 0: | |
| 789 print '0/%d, non-zero return code %d' % (npoints, proc.returncode) | |
| 790 sys.stdout.flush() | |
| 791 elif proc.returncode < 0: | |
| 792 print '0/%d, terminated by signal %d' % (npoints, -proc.returncode) | |
| 793 sys.stdout.flush() | |
| 794 else: | |
| 795 if not tester: | |
| 796 if stdio: | |
| 797 outputdata = opentestcase(realoutname.replace('$', s)) | |
| 798 r = 0 | |
| 799 data = outputdata.read().splitlines(True) | |
| 800 if len(lines) != len(data): | |
| 801 r = 1 | |
| 802 else: | |
| 803 for i in zip(lines, data): | |
| 804 if i[0] != i[1]: | |
| 805 r = 1 | |
| 806 break | |
| 807 outputdata.close() | |
| 808 else: | |
| 809 try: | |
| 810 inputdata = open(outname, 'rU') | |
| 811 except IOError: | |
| 812 print '0/%g, output file not created or not readable' % npoints | |
| 813 sys.stdout.flush() | |
| 814 r = None | |
| 815 else: | |
| 816 outputdata = opentestcase(realoutname.replace('$', s)) | |
| 817 r = 0 | |
| 818 lines = inputdata.readlines() | |
| 819 data = outputdata.read().splitlines(True) | |
| 820 if len(lines) != len(data): | |
| 821 r = 1 | |
| 822 else: | |
| 823 for i in zip(lines, data): | |
| 824 if i[0] != i[1]: | |
| 825 r = 1 | |
| 826 break | |
| 827 inputdata.close() | |
| 828 outputdata.close() | |
| 829 else: | |
| 830 if ansname: | |
| 831 copytestcase(realoutname.replace('$', s), ansname) | |
| 832 if stdio: | |
| 833 try: copytestcase(realinname.replace('$', s), inname) | |
| 834 except NameError: pass | |
| 835 outputdata = open(outname, 'w') | |
| 836 outputdata.writelines(lines) | |
| 837 outputdata.close() | |
| 838 try: | |
| 839 proc = subprocess.Popen(tester, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) | |
| 840 except OSError, error: | |
| 841 raise OSError, 'The tester application cannot be launched: ' + str(error), sys.exc_info()[2] | |
| 842 data = proc.communicate() | |
| 843 r = proc.returncode | |
| 844 if tester and data[0]: | |
| 845 data = ''.join((' (', data[0].strip(), ')')) | |
| 846 else: | |
| 847 data = '' | |
|
5
eb15a5a9b026
Output validators can now award partial scores
Oleg Oshmyan <chortos@inbox.lv>
parents:
4
diff
changeset
|
848 if not maxexitcode and r or maxexitcode and not r: |
| 0 | 849 print '0/%g, wrong answer%s' % (npoints, data) |
| 850 sys.stdout.flush() | |
|
5
eb15a5a9b026
Output validators can now award partial scores
Oleg Oshmyan <chortos@inbox.lv>
parents:
4
diff
changeset
|
851 elif not maxexitcode and r == 0 or maxexitcode and r >= maxexitcode: |
| 0 | 852 print '%g/%g, OK%s' % (npoints, npoints, data) |
| 853 sys.stdout.flush() | |
| 854 scoregrp += npoints | |
| 855 ncorrectgrp += 1 | |
| 856 if npoints: | |
| 857 ncorrectvalued += 1 | |
|
5
eb15a5a9b026
Output validators can now award partial scores
Oleg Oshmyan <chortos@inbox.lv>
parents:
4
diff
changeset
|
858 elif maxexitcode and r != None: |
|
eb15a5a9b026
Output validators can now award partial scores
Oleg Oshmyan <chortos@inbox.lv>
parents:
4
diff
changeset
|
859 actualpoints = npoints*r/maxexitcode if not npoints*r%maxexitcode else float(npoints*r)/maxexitcode |
|
eb15a5a9b026
Output validators can now award partial scores
Oleg Oshmyan <chortos@inbox.lv>
parents:
4
diff
changeset
|
860 print '%g/%g, partly OK%s' % (actualpoints, npoints, data) |
|
eb15a5a9b026
Output validators can now award partial scores
Oleg Oshmyan <chortos@inbox.lv>
parents:
4
diff
changeset
|
861 sys.stdout.flush() |
|
eb15a5a9b026
Output validators can now award partial scores
Oleg Oshmyan <chortos@inbox.lv>
parents:
4
diff
changeset
|
862 scoregrp += actualpoints |
|
eb15a5a9b026
Output validators can now award partial scores
Oleg Oshmyan <chortos@inbox.lv>
parents:
4
diff
changeset
|
863 ncorrectgrp += 1 |
|
eb15a5a9b026
Output validators can now award partial scores
Oleg Oshmyan <chortos@inbox.lv>
parents:
4
diff
changeset
|
864 if npoints: |
|
eb15a5a9b026
Output validators can now award partial scores
Oleg Oshmyan <chortos@inbox.lv>
parents:
4
diff
changeset
|
865 ncorrectvalued += 1 |
| 0 | 866 if ntotalgrp: |
| 867 if scoregrp < maxpointsgrp: | |
| 868 scoregrp = 0 | |
| 869 if ntotalgrp > 1: | |
| 870 print 'Group total: %d/%d tests; %g/%g points' % (ncorrectgrp, ntotalgrp, scoregrp, maxpointsgrp) | |
| 871 sys.stdout.flush() | |
| 872 ncorrect += ncorrectgrp | |
| 873 ntotal += ntotalgrp | |
| 874 score += scoregrp | |
| 875 maxpoints += maxpointsgrp | |
| 876 | |
| 877 if options.erase: | |
| 878 if not stdio or tester: | |
| 879 if os.path.exists(inname): os.remove(inname) | |
| 880 if os.path.exists(outname): os.remove(outname) | |
| 881 if tester and ansname: | |
| 882 if os.path.exists(ansname): os.remove(ansname) | |
| 883 elif stdio: | |
| 884 copytestcase(realinname.replace('$', s), inname) | |
| 885 copytestcase(realoutname.replace('$', s), ansname) | |
|
5
eb15a5a9b026
Output validators can now award partial scores
Oleg Oshmyan <chortos@inbox.lv>
parents:
4
diff
changeset
|
886 actualpoints = (score*taskweight/maxpoints if not score*taskweight%maxpoints else float(score*taskweight)/maxpoints) if maxpoints else 0 |
| 0 | 887 if nvalued != ntotal: |
|
5
eb15a5a9b026
Output validators can now award partial scores
Oleg Oshmyan <chortos@inbox.lv>
parents:
4
diff
changeset
|
888 print 'Grand total: %d/%d tests (%d/%d valued); %g/%g points; weighted score: %g/%g' % (ncorrect, ntotal, ncorrectvalued, nvalued, score, maxpoints, actualpoints, taskweight) |
| 0 | 889 else: |
|
5
eb15a5a9b026
Output validators can now award partial scores
Oleg Oshmyan <chortos@inbox.lv>
parents:
4
diff
changeset
|
890 print 'Grand total: %d/%d tests; %g/%g points; weighted score: %g/%g' % (ncorrect, ntotal, score, maxpoints, actualpoints, taskweight) |
| 0 | 891 |
|
5
eb15a5a9b026
Output validators can now award partial scores
Oleg Oshmyan <chortos@inbox.lv>
parents:
4
diff
changeset
|
892 scoresumoveralltasks += actualpoints |
| 0 | 893 scoremaxoveralltasks += taskweight |
| 894 ntasks += 1 | |
| 895 nfulltasks += int((score == maxpoints) if maxpoints else (taskweight == 0)) | |
| 896 | |
| 897 os.chdir(oldcwd) | |
| 898 | |
| 899 if options.clean or options.copyonly: | |
| 900 sys.exit() | |
| 901 | |
| 902 if ntasks != 1: | |
| 903 print | |
| 904 print 'Grand grand total: %g/%g weighted points; %d/%d problems solved fully' % (scoresumoveralltasks, scoremaxoveralltasks, nfulltasks, ntasks) | |
| 905 | |
| 906 if options.pause: | |
| 907 print 'Press any key to exit... ', | |
| 908 sys.stdout.flush() | |
| 909 os.system(pause + ' >' + os.devnull) |