Mercurial > ~astiob > upreckon > hgweb
annotate upreckon-vcs @ 76:0e5ae28e0b2b
Points are now weighted on a test context basis
In particular, this has allowed for simple extensions to the format
of testconf to award points to whole test groups without at the same time
compromising the future ability of giving partial score for correct
but slow solutions. Specifically, the groupweight configuration variable
has been added and normally has the format {groupindex: points} where
groupindex is the group's index in the tests configuration variable.
The backwards incompatible change is that test contexts are no longer
guaranteed to learn the score awarded or the maximum possible score
for every test case and may instead be notified about them in batches.
In other news, the pointmap and groupweight configuration variables can
(now) be given as sequences in addition to mappings. (Technically,
the distinction currently made is dict versus everything else.) Items
of a sequence pointmap/groupweight correspond directly to the test cases/
groups defined in the tests configuration variable; in particular,
when groups are used, tests=[1],[2,3];pointmap={1:1,2:2,3:3} can now be
written as pointmap=tests=[1],[2,3]. Missing items are handled in the same
way in which they are handled when the variable is a mapping. Note
that the items of groupweight correspond to whole test groups rather
than individual test cases.
In other news again, the wording of problem total lines has been changed
from '<unweighted> points; weighted score: <weighted>' to '<weighted>
points (<unweighted> before weighting)', and group total lines now
properly report fractional numbers of points (this is a bug fix).
| author | Oleg Oshmyan <chortos@inbox.lv> |
|---|---|
| date | Sat, 08 Jan 2011 16:03:35 +0200 |
| parents | b9d5857f7b9a |
| children | d46bd7ee3e69 |
| rev | line source |
|---|---|
| 21 | 1 #! /usr/bin/env python |
| 16 | 2 # Copyright (c) 2009-2010 Chortos-2 <chortos@inbox.lv> |
| 3 | |
| 4 from __future__ import division, with_statement | |
| 5 import optparse, sys, compat | |
| 21 | 6 |
| 7 def import_error(e): | |
| 45 | 8 say('Error: your installation of Upreckon is incomplete;', str(e).lower() + '.', file=sys.stderr) |
| 21 | 9 sys.exit(3) |
| 10 | |
| 11 from compat import * | |
| 16 | 12 |
|
33
f90bd2d1a12b
Converted revision reporting from SVN to hg.
Oleg Oshmyan <chortos@inbox.lv>
parents:
26
diff
changeset
|
13 version = '2.00.0 ($$REV$$)' |
| 45 | 14 parser = optparse.OptionParser(version='Upreckon '+version, epilog='Python 2.5 or newer is required.') |
| 23 | 15 parser.add_option('-1', dest='legacy', action='store_true', default=False, help='handle configuration files in a way more compatible with test.py 1.x') |
| 45 | 16 parser.add_option('-u', '--update', dest='update', action='store_true', default=False, help='update the installed Upreckon to the latest publicly available version') |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
17 parser.add_option('-p', '--problem', dest='problems', metavar='PROBLEM', action='append', help='test only the PROBLEM (this option can be specified more than once with different problem names, all of which will be tested)') |
| 16 | 18 parser.add_option('-m', '--copy-io', dest='copyonly', action='store_true', default=False, help='create a copy of the input/output files of the last test case for manual testing and exit') |
| 19 parser.add_option('-x', '--auto-exit', dest='pause', action='store_false', default=True, help='do not wait for a key to be pressed after finishing testing') | |
| 21 | 20 parser.add_option('-s', '--save-io', dest='erase', action='store_false', default=True, help='do not delete the copies of input/output files after the last test case; create copies of input files and store output in files even if the solution uses standard I/O; delete the stored input/output files if the solution uses standard I/O and the -c/--cleanup option is specified') |
| 21 parser.add_option('-t', '--detect-time', dest='autotime', action='store_true', default=False, help='spend a second detecting the most precise time measurement function') | |
| 22 parser.add_option('--no-time-limits', dest='no_maxtime', action='store_true', default=False, help='disable all time limits') | |
| 16 | 23 |
| 24 options, args = parser.parse_args() | |
| 25 parser.destroy() | |
| 26 del parser | |
| 27 | |
| 28 if options.update: | |
| 29 try: | |
| 30 urllib, urlread = compat.import_urllib() | |
| 31 except ImportError: | |
| 32 sys.exit('Error: the urllib Python module is missing. Without it, an automatic update is impossible.') | |
| 33 | |
| 34 latesttext = urlread('http://chortos.selfip.net/~astiob/test.py/version.txt') | |
| 35 latest = latesttext.split('.') | |
| 36 installed = version.split('.') | |
| 37 update = None | |
| 38 | |
| 39 if latest[0] > installed[0]: | |
| 40 update = 'major' | |
| 41 elif latest[0] == installed[0]: | |
| 42 if latest[1] > installed[1]: | |
| 43 update = 'feature' | |
| 44 elif latest[1] == installed[1]: | |
| 45 if latest[2] > installed[2]: | |
| 46 update = 'bug-fixing' | |
| 47 elif latest[2] == installed[2]: | |
| 45 | 48 say('You are using the latest publicly available version of Upreckon.') |
| 16 | 49 sys.exit() |
| 50 | |
| 51 if not update: | |
| 45 | 52 say('Your copy of Upreckon is newer than the publicly available version.') |
| 16 | 53 sys.exit() |
| 54 | |
| 45 | 55 say('A ' + update + ' update to Upreckon is available. Downloading...') |
| 16 | 56 sys.stdout.flush() |
| 45 | 57 # FIXME: need to update all files! |
| 16 | 58 urllib.urlretrieve('http://chortos.selfip.net/~astiob/test.py/test.py', sys.argv[0]) |
| 45 | 59 say('Downloaded and installed. Now you are using Upreckon ' + latesttext + '.') |
| 16 | 60 sys.exit() |
| 61 | |
| 22 | 62 import config, itertools, os, subprocess, sys, time |
| 16 | 63 |
|
70
b9d5857f7b9a
Better emulation of built-ins for testconf
Oleg Oshmyan <chortos@inbox.lv>
parents:
66
diff
changeset
|
64 if options.legacy: |
|
b9d5857f7b9a
Better emulation of built-ins for testconf
Oleg Oshmyan <chortos@inbox.lv>
parents:
66
diff
changeset
|
65 compat.pseudobuiltins += 'xrange', |
|
b9d5857f7b9a
Better emulation of built-ins for testconf
Oleg Oshmyan <chortos@inbox.lv>
parents:
66
diff
changeset
|
66 |
| 21 | 67 if options.autotime: |
| 22 | 68 # This is really a dirty hack that assumes that sleep() does not spend |
| 69 # the CPU time of the current process and that if clock() measures | |
| 70 # wall-clock time, then it is more precise than time() is. Both these | |
| 71 # assumptions are true on all platforms I have tested this on so far, | |
| 72 # but I am not aware of any guarantee that they will both be true | |
| 73 # on every other platform. | |
| 21 | 74 c = time.clock() |
| 75 time.sleep(1) | |
| 76 c = time.clock() - c | |
| 77 if int(c + .5) == 1: | |
| 78 clock = time.clock | |
| 79 else: | |
| 80 clock = time.time | |
| 81 elif sys.platform == 'win32': | |
| 82 clock = time.clock | |
| 83 else: | |
| 84 clock = time.time | |
| 16 | 85 |
| 86 try: | |
|
66
34ba0b353fc6
Fixed an issue when import errors would be ignored
Oleg Oshmyan <chortos@inbox.lv>
parents:
64
diff
changeset
|
87 import testcases |
|
34ba0b353fc6
Fixed an issue when import errors would be ignored
Oleg Oshmyan <chortos@inbox.lv>
parents:
64
diff
changeset
|
88 except ImportError: |
|
34ba0b353fc6
Fixed an issue when import errors would be ignored
Oleg Oshmyan <chortos@inbox.lv>
parents:
64
diff
changeset
|
89 import_error(sys.exc_info()[1]) |
|
34ba0b353fc6
Fixed an issue when import errors would be ignored
Oleg Oshmyan <chortos@inbox.lv>
parents:
64
diff
changeset
|
90 |
|
34ba0b353fc6
Fixed an issue when import errors would be ignored
Oleg Oshmyan <chortos@inbox.lv>
parents:
64
diff
changeset
|
91 try: |
| 22 | 92 from testcases import pause |
| 93 except ImportError: | |
| 94 pause = None | |
| 95 | |
| 96 try: | |
| 21 | 97 globalconf = config.load_global() |
| 16 | 98 |
| 21 | 99 # Do this check here so that if we have to warn them, we do it as early as possible |
| 22 | 100 if options.pause and not pause and not hasattr(globalconf, 'pause'): |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
101 if os.name == 'posix': |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
102 globalconf.pause = 'read -s -n 1' |
| 45 | 103 say('Warning: configuration variable pause is not defined; it was devised automatically but the choice might be incorrect, so Upreckon might exit immediately after the testing is completed.', file=sys.stderr) |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
104 sys.stderr.flush() |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
105 elif os.name == 'nt': |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
106 globalconf.pause = 'pause' |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
107 else: |
|
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
108 sys.exit('Error: configuration variable pause is not defined and cannot be devised automatically.') |
| 16 | 109 |
| 21 | 110 try: |
| 111 from problem import * | |
| 112 except ImportError: | |
| 113 import_error(sys.exc_info()[1]) | |
| 16 | 114 |
| 21 | 115 # Support single-problem configurations |
| 116 if globalconf.tasknames is None: | |
| 117 shouldprintnames = False | |
| 118 globalconf.multiproblem = False | |
| 119 globalconf.tasknames = os.path.curdir, | |
| 16 | 120 else: |
| 21 | 121 globalconf.multiproblem = True |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
122 shouldprintnames = True |
| 16 | 123 |
| 21 | 124 ntasks = 0 |
| 125 nfulltasks = 0 | |
| 126 maxscore = 0 | |
| 127 realscore = 0 | |
| 16 | 128 |
|
25
b500e117080e
Bug fixes and overhead reduction
Oleg Oshmyan <chortos@inbox.lv>
parents:
23
diff
changeset
|
129 for taskname in (globalconf.tasknames if not options.problems else options.problems): |
| 21 | 130 problem = Problem(taskname) |
| 131 | |
| 22 | 132 if ntasks and not options.copyonly: say() |
| 21 | 133 if shouldprintnames: say(taskname) |
| 134 | |
| 135 if options.copyonly: | |
| 136 problem.copytestdata() | |
| 137 else: | |
| 138 real, max = problem.test() | |
| 139 | |
| 140 ntasks += 1 | |
| 22 | 141 nfulltasks += real == max |
| 21 | 142 realscore += real |
| 143 maxscore += max | |
| 144 | |
| 145 if options.copyonly: | |
| 146 sys.exit() | |
| 147 | |
| 148 if ntasks != 1: | |
| 149 say() | |
| 64 | 150 say('Grand total: %g/%g weighted points; %d/%d problems solved fully' % (realscore, maxscore, nfulltasks, ntasks)) |
| 21 | 151 except KeyboardInterrupt: |
| 152 sys.exit('Exiting due to a keyboard interrupt.') | |
| 16 | 153 |
| 154 if options.pause: | |
| 21 | 155 say('Press any key to exit...') |
| 16 | 156 sys.stdout.flush() |
| 157 | |
| 22 | 158 if pause: |
| 159 pause() | |
| 160 elif callable(globalconf.pause): | |
| 161 globalconf.pause() | |
| 162 else: | |
| 163 with open(os.devnull, 'w') as devnull: | |
| 164 subprocess.call(globalconf.pause, stdout=devnull, stderr=subprocess.STDOUT) |