Mercurial > ~astiob > upreckon > hgweb
view problem.py @ 76:0e5ae28e0b2b
Points are now weighted on a test context basis
In particular, this has allowed for simple extensions to the format
of testconf to award points to whole test groups without at the same time
compromising the future ability of giving partial score for correct
but slow solutions. Specifically, the groupweight configuration variable
has been added and normally has the format {groupindex: points} where
groupindex is the group's index in the tests configuration variable.
The backwards incompatible change is that test contexts are no longer
guaranteed to learn the score awarded or the maximum possible score
for every test case and may instead be notified about them in batches.
In other news, the pointmap and groupweight configuration variables can
(now) be given as sequences in addition to mappings. (Technically,
the distinction currently made is dict versus everything else.) Items
of a sequence pointmap/groupweight correspond directly to the test cases/
groups defined in the tests configuration variable; in particular,
when groups are used, tests=[1],[2,3];pointmap={1:1,2:2,3:3} can now be
written as pointmap=tests=[1],[2,3]. Missing items are handled in the same
way in which they are handled when the variable is a mapping. Note
that the items of groupweight correspond to whole test groups rather
than individual test cases.
In other news again, the wording of problem total lines has been changed
from '<unweighted> points; weighted score: <weighted>' to '<weighted>
points (<unweighted> before weighting)', and group total lines now
properly report fractional numbers of points (this is a bug fix).
author | Oleg Oshmyan <chortos@inbox.lv> |
---|---|
date | Sat, 08 Jan 2011 16:03:35 +0200 |
parents | 007f7eb6fb2b |
children | 69eadc60f4e2 |
line wrap: on
line source
#! /usr/bin/env python # Copyright (c) 2010 Chortos-2 <chortos@inbox.lv> from __future__ import division, with_statement try: from compat import * import config, testcases except ImportError: import __main__ __main__.import_error(sys.exc_info()[1]) else: from __main__ import clock, options import os, re, sys try: from collections import deque except ImportError: deque = list try: import signal except ImportError: signalnames = () else: # Construct a cache of all signal names available on the current # platform. Prefer names from the UNIX standards over other versions. unixnames = frozenset(('HUP', 'INT', 'QUIT', 'ILL', 'ABRT', 'FPE', 'KILL', 'SEGV', 'PIPE', 'ALRM', 'TERM', 'USR1', 'USR2', 'CHLD', 'CONT', 'STOP', 'TSTP', 'TTIN', 'TTOU', 'BUS', 'POLL', 'PROF', 'SYS', 'TRAP', 'URG', 'VTALRM', 'XCPU', 'XFSZ')) signalnames = {} for name in dir(signal): if re.match('SIG[A-Z]+$', name): value = signal.__dict__[name] if isinstance(value, int) and (value not in signalnames or name[3:] in unixnames): signalnames[value] = name del unixnames __all__ = 'Problem', 'TestContext', 'test_context_end', 'TestGroup' def strerror(e): s = getattr(e, 'strerror') if not s: s = str(e) return ' (%s%s)' % (s[0].lower(), s[1:]) if s else '' class Cache(object): def __init__(self, mydict): self.__dict__ = mydict class TestContext(object): pass test_context_end = object() class TestGroup(TestContext): __slots__ = 'points', 'case', 'log', 'correct', 'allcorrect', 'real', 'max', 'ntotal', 'nvalued', 'ncorrect', 'ncorrectvalued' def __init__(self, points=None): self.points = points self.real = self.max = self.ntotal = self.nvalued = self.ncorrect = self.ncorrectvalued = 0 self.allcorrect = True self.log = [] def case_start(self, case): self.case = case self.correct = False self.ntotal += 1 if case.points: self.nvalued += 1 def case_correct(self): self.correct = True self.ncorrect += 1 if self.case.points: self.ncorrectvalued += 1 def case_end(self): self.log.append((self.case, self.correct)) del self.case if not self.correct: self.allcorrect = False def score(self, real, max): self.real += real self.max += max def end(self): if not self.allcorrect: self.real = 0 if self.points is not None and self.points != self.max: max, weighted = self.points, self.real * self.points / self.max if self.max else 0 before_weighting = ' (%g/%g before weighting)' % (self.real, self.max) else: max, weighted = self.max, self.real before_weighting = '' say('Group total: %d/%d tests, %g/%g points%s' % (self.ncorrect, self.ntotal, weighted, max, before_weighting)) # No real need to flush stdout, as it will anyway be flushed in a moment, # when either the problem total or the next test case's ID is printed return weighted, max, self.log class Problem(object): __slots__ = 'name', 'config', 'cache', 'testcases' def __init__(prob, name): if not isinstance(name, basestring): # This shouldn't happen, of course raise TypeError('Problem() argument 1 must be string, not ' + type(name).__name__) prob.name = name prob.config = config.load_problem(name) prob.cache = Cache({'padoutput': 0}) prob.testcases = testcases.load_problem(prob) # TODO def build(prob): raise NotImplementedError def test(prob): case = None try: contexts = deque((TestGroup(),)) for case in prob.testcases: if case is test_context_end: real, max, log = contexts.pop().end() for case, correct in log: contexts[-1].case_start(case) if correct: contexts[-1].case_correct() contexts[-1].case_end() contexts[-1].score(real, max) continue elif isinstance(case, TestContext): contexts.append(case) continue contexts[-1].case_start(case) granted = 0 id = str(case.id) if case.isdummy: id = 'sample ' + id say('%*s: ' % (prob.cache.padoutput, id), end='') sys.stdout.flush() try: granted = case(lambda: (say('%7.3f%s s, ' % (case.time_stopped - case.time_started, case.time_limit_string), end=''), sys.stdout.flush())) except testcases.CanceledByUser: verdict = 'canceled by the user' except testcases.TimeLimitExceeded: verdict = 'time limit exceeded' except testcases.WrongAnswer: e = sys.exc_info()[1] if e.comment: verdict = 'wrong answer (%s)' % e.comment else: verdict = 'wrong answer' except testcases.NonZeroExitCode: e = sys.exc_info()[1] if e.exitcode < 0: if sys.platform == 'win32': verdict = 'terminated with error 0x%X' % (e.exitcode + 0x100000000) elif -e.exitcode in signalnames: verdict = 'terminated by signal %d (%s)' % (-e.exitcode, signalnames[-e.exitcode]) else: verdict = 'terminated by signal %d' % -e.exitcode else: verdict = 'non-zero return code %d' % e.exitcode except testcases.CannotStartTestee: verdict = 'cannot launch the program to test%s' % strerror(sys.exc_info()[1].upstream) except testcases.CannotStartValidator: verdict = 'cannot launch the validator%s' % strerror(sys.exc_info()[1].upstream) except testcases.CannotReadOutputFile: verdict = 'cannot read the output file%s' % strerror(sys.exc_info()[1].upstream) except testcases.CannotReadInputFile: verdict = 'cannot read the input file%s' % strerror(sys.exc_info()[1].upstream) except testcases.CannotReadAnswerFile: verdict = 'cannot read the reference output file%s' % strerror(sys.exc_info()[1].upstream) except testcases.TestCaseNotPassed: verdict = 'unspecified reason [this may be a bug in test.py]%s' % strerror(sys.exc_info()[1]) #except Exception: # verdict = 'unknown error [this may be a bug in test.py]%s' % strerror(sys.exc_info()[1]) else: try: granted, comment = granted except TypeError: comment = '' else: if comment: comment = ' (%s)' % comment if granted >= 1: contexts[-1].case_correct() verdict = 'OK' + comment elif not granted: verdict = 'wrong answer' + comment else: verdict = 'partly correct' + comment granted *= case.points say('%g/%g, %s' % (granted, case.points, verdict)) contexts[-1].case_end() contexts[-1].score(granted, case.points) weighted = contexts[0].real * prob.config.taskweight / contexts[0].max if contexts[0].max else 0 before_weighting = valued = '' if prob.config.taskweight != contexts[0].max: before_weighting = ' (%g/%g before weighting)' % (contexts[0].real, contexts[0].max) if contexts[0].nvalued != contexts[0].ntotal: valued = ' (%d/%d valued)' % (contexts[0].ncorrectvalued, contexts[0].nvalued) say('Problem total: %d/%d tests%s, %g/%g points%s' % (contexts[0].ncorrect, contexts[0].ntotal, valued, weighted, prob.config.taskweight, before_weighting)) sys.stdout.flush() return weighted, prob.config.taskweight finally: if options.erase and (not prob.config.stdio or case and case.validator): for var in 'in', 'out': name = getattr(prob.config, var + 'name') if name: try: os.remove(name) except Exception: pass if case.validator and not callable(case.validator): if prob.config.ansname: try: os.remove(prob.config.ansname) except Exception: pass