view 1.20/test.py @ 9:ed90b375d197

Award maxexitcode points by default The default number of points allocated per test case is now maxexitcode if it is set and is an integer (so that all given points are integers as well).
author Oleg Oshmyan <chortos@inbox.lv>
date Sun, 14 Feb 2010 00:51:27 +0000
parents ddc9aa02007b
children d4fc9341664e
line wrap: on
line source

#! /usr/bin/python
# Copyright (c) 2009, 2010 Chortos-2 <chortos@inbox.lv>

import os, sys, shutil, time, subprocess, filecmp, optparse, signal, tempfile, tarfile, zipfile

parser = optparse.OptionParser(version='test.py 1.20.1', usage='usage: %prog [options] [problem names] [[path/to/]solution-app] [test case numbers]\n\nTest case numbers can be specified in plain text or as a Python expression\nif there is only one positional argument.\n\nOnly problem names listed in testconf.py are recognized.')
parser.add_option('-e', '--exclude', dest='exclude', action='append', help='test case number(s) to exclude, as a Python expression; multiple -e options can be supplied')
parser.add_option('-c', '--cleanup', dest='clean', action='store_true', default=False, help='delete the copies of input/output files and exit')
parser.add_option('-s', '--save-io', dest='erase', action='store_false', default=True, help='do not delete the copies of input/output files after the last test case; create copies of input files and store output in files even if the solution uses standard I/O; delete the stored input/output files if the solution uses standard I/O and the -c/--cleanup option is specified')
parser.add_option('-m', '--copy-io', dest='copyonly', action='store_true', default=False, help='only create a copy of the input/output files of the last test case for manual testing; to delete them, use options -cs')
parser.add_option('-x', '--auto-exit', dest='pause', action='store_false', default=True, help='do not wait for a key to be pressed when finished testing')
parser.add_option('-p', '--python', action='store_true', default=False, help='always parse all positional arguments as a single Python expression (including the first argument even if it names an executable file)')
parser.add_option('-t', '--detect-time', dest='autotime', action='store_true', default=False, help='spend a second detecting the most precise time measurement function')

options, args = parser.parse_args()
parser.destroy()
del parser

globals1 = set(globals())

# Initialize some configuration variables with default values
tasknames = ('.',)
maxtime = 0
tests = ()
dummies = ()
testsexcluded = ()
padwithzeroestolength = 0
taskweight = 100
pointmap = {}
stdio = False
dummyinname = ''
dummyoutname = ''
tester = ''

def exectestconf_helper(name):
	if os.path.isfile('tests.tar'):
		f = tarfile.open('tests.tar')
		try:
			exec f.extractfile(name).read() in globals()
			f.close()
			return True
		except KeyError:
			f.close()
	if os.path.isfile('tests.zip'):
		f = zipfile.ZipFile('tests.zip')
		try:
			exec f.open(name, 'rU').read() in globals()
			f.close()
			return True
		except KeyError:
			f.close()
	if os.path.isfile('tests.tgz'):
		f = tarfile.open('tests.tgz')
		try:
			exec f.extractfile(name).read() in globals()
			f.close()
			return True
		except KeyError:
			f.close()
	if os.path.isfile('tests.tar.gz'):
		f = tarfile.open('tests.tar.gz')
		try:
			exec f.extractfile(name).read() in globals()
			f.close()
			return True
		except KeyError:
			f.close()
	if os.path.isfile('tests.tbz2'):
		f = tarfile.open('tests.tbz2')
		try:
			exec f.extractfile(name).read() in globals()
			f.close()
			return True
		except KeyError:
			f.close()
	if os.path.isfile('tests.tar.bz2'):
		f = tarfile.open('tests.tar.bz2')
		try:
			exec f.extractfile(name).read() in globals()
			f.close()
			return True
		except KeyError:
			f.close()
	return False

try:
	execfile('testconf.py')
except IOError, error:
	exc_info = sys.exc_info()[2]
	try:
		execfile('tests/testconf.py')
	except IOError:
		if not exectestconf_helper('testconf.py'):
			raise IOError, (error.errno, 'The configuration file is missing', error.filename), exc_info
	del exc_info

globals2 = set(globals())
globals2.remove('globals1')
globals2 -= globals1
del globals1

shared = {}
g = globals()
for k in globals2:
	shared[k] = g[k]

newtasknames = []
while len(args) and args[0] in tasknames:
	newtasknames.append(args[0])
	del args[0]
if len(newtasknames):
	tasknames = newtasknames

scoresumoveralltasks = 0
scoremaxoveralltasks = 0
ntasks = 0
nfulltasks = 0
cwd = '' # At any time this is either '' or taskname + '/'

if options.autotime:
	c = time.clock()
	time.sleep(1)
	c = time.clock() - c
	if int(c + .99999) == 1:
		clock = time.clock
	else:
		clock = time.time
elif os.name == 'nt':
	clock = time.clock
else:
	clock = time.time

if options.copyonly:
	options.erase = False

def existstestcase_helper(name):
	if os.path.isfile('tests.tar'):
		f = tarfile.open('tests.tar')
		try:
			f.getmember(name)
			f.close()
			return True
		except KeyError:
			f.close()
	if os.path.isfile('tests.zip'):
		f = zipfile.ZipFile('tests.zip')
		try:
			f.getinfo(name)
			f.close()
			return True
		except KeyError:
			f.close()
	if os.path.isfile('tests.tgz'):
		f = tarfile.open('tests.tgz')
		try:
			f.getmember(name)
			f.close()
			return True
		except KeyError:
			f.close()
	if os.path.isfile('tests.tar.gz'):
		f = tarfile.open('tests.tar.gz')
		try:
			f.getmember(name)
			f.close()
			return True
		except KeyError:
			f.close()
	if os.path.isfile('tests.tbz2'):
		f = tarfile.open('tests.tbz2')
		try:
			f.getmember(name)
			f.close()
			return True
		except KeyError:
			f.close()
	if os.path.isfile('tests.tar.bz2'):
		f = tarfile.open('tests.tar.bz2')
		try:
			f.getmember(name)
			f.close()
			return True
		except KeyError:
			f.close()
	return False

def existstestcase(name):
	if os.path.isfile('tests/' + taskname + '/' + name) or os.path.isfile('tests/' + name):
		return True
	if cwd and (os.path.isfile(oldcwd + '/tests/' + cwd + name) or os.path.isfile(oldcwd + '/tests/' + name)):
		return True
	if existstestcase_helper(taskname + '/' + name) or existstestcase_helper(name):
		return True
	if cwd:
		os.chdir(oldcwd)
		if existstestcase_helper(cwd + name) or existstestcase_helper(name):
			os.chdir(cwd)
			return True
		os.chdir(cwd)
	return False

def opentestcase_helper(name):
	if os.path.isfile('tests.tar'):
		f = tarfile.open('tests.tar')
		try:
			c = f.extractfile(name)
			return c
		except KeyError:
			f.close()
	if os.path.isfile('tests.zip'):
		f = zipfile.ZipFile('tests.zip')
		try:
			c = f.open(name, 'rU')
			f.close()
			return c
		except KeyError:
			f.close()
	if os.path.isfile('tests.tgz'):
		f = tarfile.open('tests.tgz')
		try:
			c = f.extractfile(name)
			return c
		except KeyError:
			f.close()
	if os.path.isfile('tests.tar.gz'):
		f = tarfile.open('tests.tar.gz')
		try:
			c = f.extractfile(name)
			return c
		except KeyError:
			f.close()
	if os.path.isfile('tests.tbz2'):
		f = tarfile.open('tests.tbz2')
		try:
			c = f.extractfile(name)
			return c
		except KeyError:
			f.close()
	if os.path.isfile('tests.tar.bz2'):
		f = tarfile.open('tests.tar.bz2')
		try:
			c = f.extractfile(name)
			return c
		except KeyError:
			f.close()
	return None

def opentestcase(name):
	if os.path.isfile('tests/' + taskname + '/' + name):
		return open('tests/' + taskname + '/' + name, 'rU')
	elif os.path.isfile('tests/' + name):
		return open('tests/' + name, 'rU')
	f = opentestcase_helper(taskname + '/' + name)
	if not f:
		f = opentestcase_helper(name)
	if f:
		return f
	if cwd:
		if os.path.isfile(oldcwd + '/tests/' + cwd + name):
			return open(oldcwd + '/tests/' + cwd + name, 'rU')
		elif os.path.isfile(oldcwd + '/tests/' + name):
			return open(oldcwd + '/tests/' + name, 'rU')
		os.chdir(oldcwd)
		f = opentestcase_helper(cwd + name)
		if not f:
			f = opentestcase_helper(name)
		os.chdir(cwd)
		if f:
			return f
	raise KeyError, 'The test-case-defining file \'' + name + '\' cannot be found'

def copytestcase_helper(name, target):
	if os.path.isfile('tests.tar'):
		f = tarfile.open('tests.tar')
		try:
			m = f.getmember(name)
			m.name = target
			f.extract(m)
			f.close()
			return True
		except KeyError:
			f.close()
	if os.path.isfile('tests.zip'):
		if not target.startswith('/'):
			f = zipfile.ZipFile('tests.zip')
			m = f.getinfo(name)
			try:
				m.filename = target
				f.extract(m)
				f.close()
				return True
			except KeyError:
				f.close()
		else:
			oldcwd = os.getcwdu()
			os.chdir('/')
			f = zipfile.ZipFile(oldcwd + '/tests.zip')
			try:
				m = f.getinfo(name)
				m.filename = target[1:]
				f.extract(m)
				f.close()
				os.chdir(oldcwd)
				return True
			except KeyError:
				f.close()
				os.chdir(oldwcd)
	if os.path.isfile('tests.tgz'):
		f = tarfile.open('tests.tgz')
		try:
			m = f.getmember(name)
			m.name = target
			f.extract(m)
			f.close()
			return True
		except KeyError:
			f.close()
	if os.path.isfile('tests.tar.gz'):
		f = tarfile.open('tests.tar.gz')
		try:
			m = f.getmember(name)
			m.name = target
			f.extract(m)
			f.close()
			return True
		except KeyError:
			f.close()
	if os.path.isfile('tests.tbz2'):
		f = tarfile.open('tests.tbz2')
		try:
			m = f.getmember(name)
			m.name = target
			f.extract(m)
			f.close()
			return True
		except KeyError:
			f.close()
	if os.path.isfile('tests.tar.bz2'):
		f = tarfile.open('tests.tar.bz2')
		try:
			m = f.getmember(name)
			m.name = target
			f.extract(m)
			f.close()
			return True
		except KeyError:
			f.close()
	return False

def copytestcase(name, target):
	if os.path.isfile('tests/' + taskname + '/' + name):
		shutil.copyfile('tests/' + taskname + '/' + name, target)
		return
	elif os.path.isfile('tests/' + name):
		shutil.copyfile('tests/' + name, target)
		return
	if copytestcase_helper(taskname + '/' + name, target) or copytestcase_helper(name, target):
		return
	if cwd:
		if os.path.isfile(oldcwd + '/tests/' + cwd + name):
			shutil.copyfile(oldcwd + '/tests/' + cwd + name, target)
			return
		elif os.path.isfile(oldcwd + '/tests/' + name):
			shutil.copyfile(oldcwd + '/tests/' + name, target)
			return
		os.chdir(oldcwd)
		if copytestcase_helper(cwd + name, target) or copytestcase_helper(name, target):
			os.chdir(cwd)
			return
		os.chdir(cwd)
	raise KeyError, 'The test-case-defining file \'' + name + '\' cannot be found'

# Always chdir if the directory exists but use any existing config
def chdir_and_exec_testconf():
	global cwd
	cwd = ''
	if os.path.isdir(taskname):
		os.chdir(taskname)
		if taskname != '.':
			cwd = taskname + '/'
		try:
			execfile('testconf.py', globals())
			return
		except IOError:
			pass
	if not cwd:
		if os.path.isfile('tests/' + taskname + '/testconf.py'):
			execfile('tests/' + taskname + '/testconf.py', globals())
			return
		if os.path.isfile('tests/testconf.py'):
			execfile('tests/testconf.py', globals())
			return
	if exectestconf_helper(taskname + '/testconf.py') or exectestconf_helper('testconf.py'):
		return
	if cwd:
		os.chdir(oldcwd)
		if os.path.isfile('tests/' + cwd + 'testconf.py'):
			execfile('tests/' + cwd + 'testconf.py', globals())
			os.chdir(cwd)
			return
		if os.path.isfile('tests/testconf.py'):
			execfile('tests/testconf.py', globals())
			os.chdir(cwd)
			return
		if exectestconf_helper(cwd + 'testconf.py') or exectestconf_helper('testconf.py'):
			os.chdir(cwd)
			return
		if os.path.isfile('testconf.py'):
			execfile('testconf.py', globals())
			os.chdir(cwd)
			return
		os.chdir(cwd)
	elif os.path.isfile('testconf.py'):
		execfile('testconf.py', globals())
		return
	raise KeyError, 'The configuration file for task ' + taskname + ' is missing'

try:
	name
	namedefined = True
except Exception:
	namedefined = False

for taskname in tasknames:
	if ntasks:
		print
	
	try:
		if len(tasknames) > 1:
			print taskname
	except Exception:
		if taskname != '.' or ntasks:
			print taskname
	
	try: del inname
	except NameError: pass
	try: del outname
	except NameError: pass
	try: del ansname
	except NameError: pass
	
	if not namedefined and taskname != '.':
		name = './' + taskname
	for k in shared:
		g[k] = shared[k]
	
	oldcwd = os.getcwdu()
	chdir_and_exec_testconf()
	
	if options.clean:
		try:
			if not stdio or tester:
				if not tester:
					inname
				outname
			if tester:
				ansname
		except NameError, error:
			raise NameError, 'configuration ' + str(error).replace('name ', 'variable ', 1), sys.exc_info()[2]
		if not options.erase:
			try:
				inname = inname.replace('%', taskname)
			except NameError:
				inname = taskname + '.in'
			try:
				outname = outname.replace('%', taskname)
			except NameError:
				outname = taskname + '.out'
			try:
				ansname = ansname.replace('%', taskname)
			except NameError:
				ansname = taskname + '.ans'
		else:
			inname = inname.replace('%', taskname)
			outname = outname.replace('%', taskname)
			if tester:
				ansname = ansname.replace('%', taskname)
		if not stdio or tester or not options.erase:
			if os.path.exists(inname): os.remove(inname)
			if os.path.exists(outname): os.remove(outname)
		if (tester or not options.erase) and ansname:
			if os.path.exists(ansname): os.remove(ansname)
		continue
	
	try:
		name
	except NameError, error:
		if str(error).count('name') == 1:
			raise NameError, 'configuration ' + str(error), sys.exc_info()[2]
		else:
			raise NameError, 'configuration ' + str(error).replace('name ', 'variable ', 1), sys.exc_info()[2]
	
	try:
		if not stdio:
			inname
			outname
		testcaseinname
		if tester:
			outname
			if ansname:
				testcaseoutname
		else:
			testcaseoutname
	except NameError, error:
		raise NameError, 'configuration ' + str(error).replace('name ', 'variable ', 1), sys.exc_info()[2]
	
	if not options.erase:
		try:
			inname
		except NameError:
			inname = taskname + '.in'
		try:
			outname
		except NameError:
			outname = taskname + '.out'
		try:
			ansname
		except NameError:
			ansname = taskname + '.ans'
	
	if options.pause:
		try:
			pause
		except NameError, error:
			if os.name == 'posix':
				pause = 'read -s -n 1'
				print 'Configuration ' + str(error).replace('name ', 'variable ') + '; it was devised automatically but the choice might be incorrect, so test.py might exit immediately after the testing is complete.'
			elif os.name == 'nt':
				pause = 'pause'
			else:
				raise NameError, 'configuration ' + str(error).replace('name ', 'variable ') + ' and cannot be devised automatically', sys.exc_info()[2]
	
	if not dummyinname:
		dummyinname = testcaseinname
	if not dummyoutname and (not tester or ansname):
		dummyoutname = testcaseoutname
	
	dummyinname = dummyinname.replace('%', taskname)
	dummyoutname = dummyoutname.replace('%', taskname)
	testcaseinname = testcaseinname.replace('%', taskname)
	if not stdio or not options.erase:
		inname = inname.replace('%', taskname)
		outname = outname.replace('%', taskname)
		try:
			ansname = ansname.replace('%', taskname)
		except NameError:
			pass
	if tester:
		try: inname = inname.replace('%', taskname)
		except NameError: pass
		outname = outname.replace('%', taskname)
		if ansname:
			ansname = ansname.replace('%', taskname)
			testcaseoutname = testcaseoutname.replace('%', taskname)
	else:
		testcaseoutname = testcaseoutname.replace('%', taskname)
	
	if isinstance(padwithzeroestolength, tuple):
		padwithzeroestolength, paddummieswithzeroestolength = padwithzeroestolength
	else:
		paddummieswithzeroestolength = padwithzeroestolength
	
	if options.python:
		dummies = ()
		s = ' '.join(args)
		tests = eval(s)
		try:
			tests.__iter__
		except AttributeError:
			tests = (tests,)
	elif len(args):
		if os.path.exists(args[0]):
			name = args[0]
			del args[0]
		if len(args) > 1:
			dummies = ()
			tests = args
		elif len(args):
			dummies = ()
			s = args[0]
			if len(s) < padwithzeroestolength:
				s = s.zfill(padwithzeroestolength)
			if existstestcase(testcaseinname.replace('$', s)):
				tests = (s,)
			else:
				try:
					tests = eval(args[0])
					try:
						tests.__iter__
					except AttributeError:
						tests = (tests,)
				except Exception:
					tests = (s,)
	
	if options.exclude:
		testsexcluded = []
		for i in options.exclude:
			v = eval(i)
			try:
				testsexcluded.extend(v)
			except TypeError:
				testsexcluded.append(v)
	
	# Windows doesn't like paths beginning with .\ and not ending with an extension
	name = os.path.normcase(name)
	if name.startswith('.\\'):
		name = name[2:]
	
	newpointmap = {}
	
	for i in pointmap:
		try:
			for j in i:
				newpointmap[j] = pointmap[i]
		except TypeError:
			newpointmap[i] = pointmap[i]
	
	pointmap = newpointmap
	
	if maxtime > 0:
		strmaxtime = '/%.3f' % maxtime
	else:
		strmaxtime = ''
	
	padoutputtolength = 0
	ntests = []
	
	for j in dummies:
		try:
			j.__iter__
		except AttributeError:
			j = (j,)
		ntests.append((j, True))
		for i in j:
			s = str(i)
			if len(s) < paddummieswithzeroestolength:
				s = s.zfill(paddummieswithzeroestolength)
			s = 'sample ' + s
			if padoutputtolength < len(s):
				padoutputtolength = len(s)
	
	for j in tests:
		try:
			j.__iter__
		except AttributeError:
			j = (j,)
		ntests.append((j, False))
		for i in j:
			s = str(i)
			if len(s) < padwithzeroestolength:
				s = s.zfill(padwithzeroestolength)
			if padoutputtolength < len(s):
				padoutputtolength = len(s)
	
	tests = ntests
	score = maxpoints = ncorrect = ntotal = ncorrectvalued = nvalued = 0
	
	if options.copyonly:
		j, isdummy = tests[-1]
		if isdummy:
			realinname = dummyinname
			realoutname = dummyoutname
		else:
			realinname = testcaseinname
			realoutname = testcaseoutname
		for i in j:
			if i in testsexcluded and not isdummy:
				continue
			s = str(i)
			if isdummy:
				if len(s) < paddummieswithzeroestolength:
					s = s.zfill(paddummieswithzeroestolength)
			else:
				if len(s) < padwithzeroestolength:
					s = s.zfill(padwithzeroestolength)
			copytestcase(realinname.replace('$', s), inname)
			if ansname:
				copytestcase(realoutname.replace('$', s), ansname)
		continue
	
	for j, isdummy in tests:
		ncorrectgrp = 0
		ntotalgrp = 0
		scoregrp = 0
		maxpointsgrp = 0
		if isdummy:
			realinname = dummyinname
			realoutname = dummyoutname
		else:
			realinname = testcaseinname
			realoutname = testcaseoutname
		for i in j:
			if i in testsexcluded and not isdummy:
				continue
			ntotalgrp += 1
			s = str(i)
			if isdummy:
				npoints = 0
				if len(s) < paddummieswithzeroestolength:
					s = s.zfill(paddummieswithzeroestolength)
				spref = 'sample '
			else:
				npoints = pointmap.get(None, 1)
				npoints = pointmap.get(i, npoints)
				maxpointsgrp += npoints
				if npoints:
					nvalued += 1
				if len(s) < padwithzeroestolength:
					s = s.zfill(padwithzeroestolength)
				spref = ''
			print ' ' * (padoutputtolength - len(spref + s)) + spref + s + ':',
			sys.stdout.flush()
			outputdata = open(os.devnull, 'w')
			if stdio:
				f = tempfile.NamedTemporaryFile(delete=False)
				inputdatafname = f.name
				f.close()
				copytestcase(realinname.replace('$', s), inputdatafname)
				inputdata = open(inputdatafname, 'rU')
				if options.erase:
					tempoutput = tempfile.TemporaryFile('w+')
				else:
					tempoutput = open(outname, 'w+')
				try:
					proc = subprocess.Popen(name, stdin=inputdata, stdout=tempoutput, stderr=outputdata, universal_newlines=True)
				except OSError, error:
					raise OSError, 'The program to be tested cannot be launched: ' + str(error), sys.exc_info()[2]
			else:
				if os.path.exists(outname):
					os.remove(outname)
				copytestcase(realinname.replace('$', s), inname)
				try:
					proc = subprocess.Popen(name, stdin=outputdata, stdout=outputdata, stderr=outputdata, universal_newlines=True)
				except OSError, error:
					raise OSError, 'The program to be tested cannot be launched: ' + str(error), sys.exc_info()[2]
			cl = clock()
			if maxtime > 0:
				while 1:
					proc.poll()
					elapsed = clock() - cl
					if proc.returncode == None:
						if elapsed >= maxtime:
							print '%.3f%s s, 0/%d, time limit exceeded' % (elapsed, strmaxtime, npoints)
							sys.stdout.flush()
							while proc.returncode == None:
								try:
									proc.terminate()
								except OSError:
									pass
								except AttributeError:
									try:
										os.kill(proc.pid, signal.SIGTERM)
									except Exception:
										pass
								proc.poll()
							outputdata.close()
							if stdio:
								tempoutput.close()
							break
					else:
						print '%.3f%s s,' % (elapsed, strmaxtime),
						sys.stdout.flush()
						elapsed = 0
						if stdio:
							tempoutput.seek(0)
							lines = tempoutput.readlines()
							tempoutput.close()
						break
				if elapsed >= maxtime:
					continue
			else:
				data = proc.communicate()
				elapsed = clock() - cl
				print '%.3f%s s,' % (elapsed, strmaxtime),
				sys.stdout.flush()
				if stdio:
					tempoutput.seek(0)
					lines = tempoutput.readlines()
					tempoutput.close()
			outputdata.close()
			if stdio:
				inputdata.close()
				try:
					os.unlink(inputdatafname)
				except Exception:
					pass
			if proc.returncode > 0:
				print '0/%d, non-zero return code %d' % (npoints, proc.returncode)
				sys.stdout.flush()
			elif proc.returncode < 0:
				print '0/%d, terminated by signal %d' % (npoints, -proc.returncode)
				sys.stdout.flush()
			else:
				if not tester:
					if stdio:
						outputdata = opentestcase(realoutname.replace('$', s))
						r = 0
						data = outputdata.read().splitlines(True)
						if len(lines) != len(data):
							r = 1
						else:
							for i in zip(lines, data):
								if i[0] != i[1]:
									r = 1
									break
						outputdata.close()
					else:
						try:
							inputdata = open(outname, 'rU')
						except IOError:
							print '0/%g, output file not created or not readable' % npoints
							sys.stdout.flush()
							r = None
						else:
							outputdata = opentestcase(realoutname.replace('$', s))
							r = 0
							lines = inputdata.readlines()
							data = outputdata.read().splitlines(True)
							if len(lines) != len(data):
								r = 1
							else:
								for i in zip(lines, data):
									if i[0] != i[1]:
										r = 1
										break
							inputdata.close()
							outputdata.close()
				else:
					if ansname:
						copytestcase(realoutname.replace('$', s), ansname)
					if stdio:
						try: copytestcase(realinname.replace('$', s), inname)
						except NameError: pass
						outputdata = open(outname, 'w')
						outputdata.writelines(lines)
						outputdata.close()
					try:
						proc = subprocess.Popen(tester, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
					except OSError, error:
						raise OSError, 'The tester application cannot be launched: ' + str(error), sys.exc_info()[2]
					data = proc.communicate()
					r = proc.returncode
				if tester and data[0]:
					data = ''.join((' (', data[0].strip(), ')'))
				else:
					data = ''
				if r:
					print '0/%g, wrong answer%s' % (npoints, data)
					sys.stdout.flush()
				elif r == 0:
					print '%g/%g, OK%s' % (npoints, npoints, data)
					sys.stdout.flush()
					scoregrp += npoints
					ncorrectgrp += 1
					if npoints:
						ncorrectvalued += 1
		if ntotalgrp:
			if scoregrp < maxpointsgrp:
				scoregrp = 0
			if ntotalgrp > 1:
				print 'Group total: %d/%d tests; %g/%g points' % (ncorrectgrp, ntotalgrp, scoregrp, maxpointsgrp)
				sys.stdout.flush()
			ncorrect += ncorrectgrp
			ntotal += ntotalgrp
			score += scoregrp
			maxpoints += maxpointsgrp
	
	if options.erase:
		if not stdio or tester:
			if os.path.exists(inname): os.remove(inname)
			if os.path.exists(outname): os.remove(outname)
		if tester and ansname:
			if os.path.exists(ansname): os.remove(ansname)
	elif stdio:
		copytestcase(realinname.replace('$', s), inname)
		copytestcase(realoutname.replace('$', s), ansname)
	if nvalued != ntotal:
		print 'Grand total: %d/%d tests (%d/%d valued); %g/%g points; weighted score: %g/%g' % (ncorrect, ntotal, ncorrectvalued, nvalued, score, maxpoints, (score*taskweight/maxpoints if not score*taskweight%maxpoints else float(score*taskweight)/maxpoints) if maxpoints else 0, taskweight)
	else:
		print 'Grand total: %d/%d tests; %g/%g points; weighted score: %g/%g' % (ncorrect, ntotal, score, maxpoints, (score*taskweight/maxpoints if not score*taskweight%maxpoints else float(score*taskweight)/maxpoints) if maxpoints else 0, taskweight)
	
	scoresumoveralltasks += (score*taskweight/maxpoints if not score*taskweight%maxpoints else float(score*taskweight)/maxpoints) if maxpoints else 0
	scoremaxoveralltasks += taskweight
	ntasks += 1
	nfulltasks += int((score == maxpoints) if maxpoints else (taskweight == 0))
	
	os.chdir(oldcwd)

if options.clean or options.copyonly:
	sys.exit()

if ntasks != 1:
	print
	print 'Grand grand total: %g/%g weighted points; %d/%d problems solved fully' % (scoresumoveralltasks, scoremaxoveralltasks, nfulltasks, ntasks)

if options.pause:
	print 'Press any key to exit... ',
	sys.stdout.flush()
	os.system(pause + ' >' + os.devnull)