Mercurial > ~astiob > upreckon > hgweb

comparison test.py @ 5:eb15a5a9b026

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Output validators can now award partial scores Added the maxexitcode configuration option. Output validators can now cause a fraction of the points allocated for the test case to be awarded (exitcode/maxexitcode is the fraction, ranging from 0 to 1).
author Oleg Oshmyan <chortos@inbox.lv>
date Sun, 14 Feb 2010 00:08:16 +0000
parents 06eef313aeaa
children b0034b18f942
comparison
equal deleted inserted replaced
4:06eef313aeaa 5:eb15a5a9b026
29 pointmap = {} 29 pointmap = {}
30 stdio = False 30 stdio = False
31 dummyinname = '' 31 dummyinname = ''
32 dummyoutname = '' 32 dummyoutname = ''
33 tester = '' 33 tester = ''
34 maxexitcode = 0
34 35
35 def exectestconf_helper(name): 36 def exectestconf_helper(name):
36 if os.path.isfile('tests.tar'): 37 if os.path.isfile('tests.tar'):
37 f = tarfile.open('tests.tar') 38 f = tarfile.open('tests.tar')
38 try: 39 try:
842 r = proc.returncode 843 r = proc.returncode
843 if tester and data[0]: 844 if tester and data[0]:
844 data = ''.join((' (', data[0].strip(), ')')) 845 data = ''.join((' (', data[0].strip(), ')'))
845 else: 846 else:
846 data = '' 847 data = ''
847 if r: 848 if not maxexitcode and r or maxexitcode and not r:
848 print '0/%g, wrong answer%s' % (npoints, data) 849 print '0/%g, wrong answer%s' % (npoints, data)
849 sys.stdout.flush() 850 sys.stdout.flush()
850 elif r == 0: 851 elif not maxexitcode and r == 0 or maxexitcode and r >= maxexitcode:
851 print '%g/%g, OK%s' % (npoints, npoints, data) 852 print '%g/%g, OK%s' % (npoints, npoints, data)
852 sys.stdout.flush() 853 sys.stdout.flush()
853 scoregrp += npoints 854 scoregrp += npoints
855 ncorrectgrp += 1
856 if npoints:
857 ncorrectvalued += 1
858 elif maxexitcode and r != None:
859 actualpoints = npoints*r/maxexitcode if not npoints*r%maxexitcode else float(npoints*r)/maxexitcode
860 print '%g/%g, partly OK%s' % (actualpoints, npoints, data)
861 sys.stdout.flush()
862 scoregrp += actualpoints
854 ncorrectgrp += 1 863 ncorrectgrp += 1
855 if npoints: 864 if npoints:
856 ncorrectvalued += 1 865 ncorrectvalued += 1
857 if ntotalgrp: 866 if ntotalgrp:
858 if scoregrp < maxpointsgrp: 867 if scoregrp < maxpointsgrp:
872 if tester and ansname: 881 if tester and ansname:
873 if os.path.exists(ansname): os.remove(ansname) 882 if os.path.exists(ansname): os.remove(ansname)
874 elif stdio: 883 elif stdio:
875 copytestcase(realinname.replace('$', s), inname) 884 copytestcase(realinname.replace('$', s), inname)
876 copytestcase(realoutname.replace('$', s), ansname) 885 copytestcase(realoutname.replace('$', s), ansname)
886 actualpoints = (score*taskweight/maxpoints if not score*taskweight%maxpoints else float(score*taskweight)/maxpoints) if maxpoints else 0
877 if nvalued != ntotal: 887 if nvalued != ntotal:
878 print 'Grand total: %d/%d tests (%d/%d valued); %g/%g points; weighted score: %g/%g' % (ncorrect, ntotal, ncorrectvalued, nvalued, score, maxpoints, (score*taskweight/maxpoints if not score*taskweight%maxpoints else float(score*taskweight)/maxpoints) if maxpoints else 0, taskweight) 888 print 'Grand total: %d/%d tests (%d/%d valued); %g/%g points; weighted score: %g/%g' % (ncorrect, ntotal, ncorrectvalued, nvalued, score, maxpoints, actualpoints, taskweight)
879 else: 889 else:
880 print 'Grand total: %d/%d tests; %g/%g points; weighted score: %g/%g' % (ncorrect, ntotal, score, maxpoints, (score*taskweight/maxpoints if not score*taskweight%maxpoints else float(score*taskweight)/maxpoints) if maxpoints else 0, taskweight) 890 print 'Grand total: %d/%d tests; %g/%g points; weighted score: %g/%g' % (ncorrect, ntotal, score, maxpoints, actualpoints, taskweight)
881 891
882 scoresumoveralltasks += score*taskweight//maxpoints if maxpoints else 0 892 scoresumoveralltasks += actualpoints
883 scoremaxoveralltasks += taskweight 893 scoremaxoveralltasks += taskweight
884 ntasks += 1 894 ntasks += 1
885 nfulltasks += int((score == maxpoints) if maxpoints else (taskweight == 0)) 895 nfulltasks += int((score == maxpoints) if maxpoints else (taskweight == 0))
886 896
887 os.chdir(oldcwd) 897 os.chdir(oldcwd)