Mercurial > ~astiob > upreckon > hgweb

comparison testcases.py @ 76:0e5ae28e0b2b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Points are now weighted on a test context basis In particular, this has allowed for simple extensions to the format of testconf to award points to whole test groups without at the same time compromising the future ability of giving partial score for correct but slow solutions. Specifically, the groupweight configuration variable has been added and normally has the format {groupindex: points} where groupindex is the group's index in the tests configuration variable. The backwards incompatible change is that test contexts are no longer guaranteed to learn the score awarded or the maximum possible score for every test case and may instead be notified about them in batches. In other news, the pointmap and groupweight configuration variables can (now) be given as sequences in addition to mappings. (Technically, the distinction currently made is dict versus everything else.) Items of a sequence pointmap/groupweight correspond directly to the test cases/ groups defined in the tests configuration variable; in particular, when groups are used, tests=[1],[2,3];pointmap={1:1,2:2,3:3} can now be written as pointmap=tests=[1],[2,3]. Missing items are handled in the same way in which they are handled when the variable is a mapping. Note that the items of groupweight correspond to whole test groups rather than individual test cases. In other news again, the wording of problem total lines has been changed from '<unweighted> points; weighted score: <weighted>' to '<weighted> points (<unweighted> before weighting)', and group total lines now properly report fractional numbers of points (this is a bug fix).
author Oleg Oshmyan <chortos@inbox.lv>
date Sat, 08 Jan 2011 16:03:35 +0200
parents 7520b6bb6636
children 69eadc60f4e2
comparison
equal deleted inserted replaced
75:007f7eb6fb2b 76:0e5ae28e0b2b
678 678
679 class DummyTestContext(problem.TestGroup): 679 class DummyTestContext(problem.TestGroup):
680 __slots__ = () 680 __slots__ = ()
681 def end(self): 681 def end(self):
682 say('Sample total: %d/%d tests' % (self.ncorrect, self.ntotal)) 682 say('Sample total: %d/%d tests' % (self.ncorrect, self.ntotal))
683 return self.log 683 return 0, 0, self.log
684
685 684
686 def load_problem(prob, _types={'batch' : BatchTestCase, 685 def load_problem(prob, _types={'batch' : BatchTestCase,
687 'outonly' : OutputOnlyTestCase, 686 'outonly' : OutputOnlyTestCase,
688 'bestout' : BestOutputTestCase, 687 'bestout' : BestOutputTestCase,
689 'reactive': ReactiveTestCase}): 688 'reactive': ReactiveTestCase}):
715 newtests.append(name) 714 newtests.append(name)
716 if prob.config.usegroups: 715 if prob.config.usegroups:
717 prob.config.tests = newtests 716 prob.config.tests = newtests
718 del newtests 717 del newtests
719 718
719 # Even if they have duplicate test identifiers, we must honour sequence pointmaps
720 if isinstance(prob.config.pointmap, dict):
721 def getpoints(i, j, k=None):
722 try:
723 return prob.config.pointmap[i]
724 except KeyError:
725 try:
726 return prob.config.pointmap[None]
727 except KeyError:
728 return prob.config.maxexitcode or 1
729 elif prob.config.usegroups:
730 def getpoints(i, j, k):
731 try:
732 return prob.config.pointmap[k][j]
733 except LookupError:
734 return prob.config.maxexitcode or 1
735 else:
736 def getpoints(i, j):
737 try:
738 return prob.config.pointmap[j]
739 except LookupError:
740 return prob.config.maxexitcode or 1
741
720 # First get prob.cache.padoutput right, 742 # First get prob.cache.padoutput right,
721 # then yield the actual test cases 743 # then yield the actual test cases
722 for i in prob.config.dummies: 744 for i in prob.config.dummies:
723 s = 'sample ' + str(i).zfill(prob.config.paddummies) 745 s = 'sample ' + str(i).zfill(prob.config.paddummies)
724 prob.cache.padoutput = max(prob.cache.padoutput, len(s)) 746 prob.cache.padoutput = max(prob.cache.padoutput, len(s))
725 if prob.config.usegroups: 747 if prob.config.usegroups:
748 if not isinstance(prob.config.groupweight, dict):
749 prob.config.groupweight = dict(enumerate(prob.config.groupweight))
726 for group in prob.config.tests: 750 for group in prob.config.tests:
727 for i in group: 751 for i in group:
728 s = str(i).zfill(prob.config.padtests) 752 s = str(i).zfill(prob.config.padtests)
729 prob.cache.padoutput = max(prob.cache.padoutput, len(s)) 753 prob.cache.padoutput = max(prob.cache.padoutput, len(s))
730 yield DummyTestContext() 754 yield DummyTestContext()
731 for i in prob.config.dummies: 755 for i in prob.config.dummies:
732 s = str(i).zfill(prob.config.paddummies) 756 s = str(i).zfill(prob.config.paddummies)
733 yield _types[prob.config.kind](prob, s, True, 0) 757 yield _types[prob.config.kind](prob, s, True, 0)
734 yield problem.test_context_end 758 yield problem.test_context_end
735 for group in prob.config.tests: 759 for k, group in enumerate(prob.config.tests):
736 yield problem.TestGroup() 760 if not group:
737 for i in group: 761 continue
762 yield problem.TestGroup(prob.config.groupweight.get(k, prob.config.groupweight.get(None)))
763 for j, i in enumerate(group):
738 s = str(i).zfill(prob.config.padtests) 764 s = str(i).zfill(prob.config.padtests)
739 yield _types[prob.config.kind](prob, s, False, prob.config.pointmap.get(i, prob.config.pointmap.get(None, prob.config.maxexitcode if prob.config.maxexitcode else 1))) 765 yield _types[prob.config.kind](prob, s, False, getpoints(i, j, k))
740 yield problem.test_context_end 766 yield problem.test_context_end
741 else: 767 else:
742 for i in prob.config.tests: 768 for i in prob.config.tests:
743 s = str(i).zfill(prob.config.padtests) 769 s = str(i).zfill(prob.config.padtests)
744 prob.cache.padoutput = max(prob.cache.padoutput, len(s)) 770 prob.cache.padoutput = max(prob.cache.padoutput, len(s))
745 for i in prob.config.dummies: 771 for i in prob.config.dummies:
746 s = str(i).zfill(prob.config.paddummies) 772 s = str(i).zfill(prob.config.paddummies)
747 yield _types[prob.config.kind](prob, s, True, 0) 773 yield _types[prob.config.kind](prob, s, True, 0)
748 for i in prob.config.tests: 774 for j, i in enumerate(prob.config.tests):
749 s = str(i).zfill(prob.config.padtests) 775 s = str(i).zfill(prob.config.padtests)
750 yield _types[prob.config.kind](prob, s, False, prob.config.pointmap.get(i, prob.config.pointmap.get(None, prob.config.maxexitcode if prob.config.maxexitcode else 1))) 776 yield _types[prob.config.kind](prob, s, False, getpoints(i, j))