# HG changeset patch # User Oleg Oshmyan # Date 1313427178 -10800 # Node ID 67088c1765b450046b85f9950c51e1aeb00f1c18 # Parent c2490e39fd709d57cfbdd8f1bc57c34cbcb19ddb Regexps now work with test archives Excuse me while I rewrite files.{File,regexp} almost from scratch... diff -r c2490e39fd70 -r 67088c1765b4 upreckon/files.py --- a/upreckon/files.py Sun Aug 14 01:02:10 2011 +0300 +++ b/upreckon/files.py Mon Aug 15 19:52:58 2011 +0300 @@ -63,19 +63,17 @@ def __init__(self, path): self._tarfile = tarfile.open(path) - files, dirs = {}, set() + files, dirs = {}, set(('/',)) for member in self._tarfile.getmembers(): - cutname = posixpath.normpath(member.name).lstrip('/') - while cutname.startswith('../'): - cutname = cutname[3:] - if cutname in ('.', '..'): + cutname = posixpath.normpath('/' + member.name) + if cutname == '/': continue if member.isfile(): files[cutname] = member cutname = posixpath.dirname(cutname) elif not member.isdir(): continue - while cutname: + while cutname != '/': dirs.add(cutname) cutname = posixpath.dirname(cutname) self._files = files @@ -83,26 +81,27 @@ self._names = self._dirs | frozenset(files) def extract(self, name, target): - member = self._files[posixpath.normpath(name)] + member = self._files[posixpath.normpath('/' + name)] member.name = target self._tarfile.extract(member) def open(self, name): - name = posixpath.normpath(name) + name = posixpath.normpath('/' + name) return self._tarfile.extractfile(self._files[name]) def exists(self, name): - return posixpath.normpath(name) in self._names + return posixpath.normpath('/' + name) in self._names def listdir(self, name): - normname = posixpath.normpath(name) + normname = posixpath.normpath('/' + name) if normname not in self._dirs: raise KeyError('No such directory: %r' % name) - normname += '/' - len_normname = len(normname) - return [fname for fname in self._names - if fname.startswith(normname) and - fname.find('/', len_normname) == -1] + if normname != '/': + normname += '/' + nnlen = len(normname) + return [fname[nnlen:] for fname in self._names + if fname.startswith(normname) and + fname.find('/', nnlen) == -1] def __enter__(self): if hasattr(self._tarfile, '__enter__'): @@ -132,17 +131,15 @@ def __init__(self, path): self._zipfile = zipfile.ZipFile(path) - files, dirs = {}, set() + files, dirs = {}, set(('/',)) for member in self._zipfile.infolist(): - cutname = posixpath.normpath(member.filename).lstrip('/') - while cutname.startswith('../'): - cutname = cutname[3:] - if cutname in ('.', '..'): + cutname = posixpath.normpath('/' + member.filename) + if cutname == '/': continue if not member.filename.endswith('/'): files[cutname] = member cutname = posixpath.dirname(cutname) - while cutname: + while cutname != '/': dirs.add(cutname) cutname = posixpath.dirname(cutname) self._files = files @@ -150,7 +147,7 @@ self._names = self._dirs | frozenset(files) def extract(self, name, target): - member = self._files[posixpath.normpath(name)] + member = self._files[posixpath.normpath('/' + name)] # FIXME: 2.5 lacks ZipFile.extract if os.path.isabs(target): # To my knowledge, this is as portable as it gets @@ -162,22 +159,23 @@ self._zipfile.extract(member) def open(self, name): - name = posixpath.normpath(name) + name = posixpath.normpath('/' + name) # FIXME: 2.5 lacks ZipFile.open return self._zipfile.open(self._files[name]) def exists(self, name): - return posixpath.normpath(name) in self._names + return posixpath.normpath('/' + name) in self._names def listdir(self, name): - normname = posixpath.normpath(name) + normname = posixpath.normpath('/' + name) if normname not in self._dirs: raise KeyError('No such directory: %r' % name) - normname += '/' - len_normname = len(normname) - return [fname for fname in self._names - if fname.startswith(normname) and - fname.find('/', len_normname) == -1] + if normname != '/': + normname += '/' + nnlen = len(normname) + return [fname[nnlen:] for fname in self._names + if fname.startswith(normname) and + fname.find('/', nnlen) == -1] def __enter__(self): if hasattr(self._zipfile, '__enter__'): @@ -291,24 +289,45 @@ else: shutil.copy(self.real_path, target) -def regexp(pattern, hastests=False, droplast=False): - # FIXME: add test archives +class RegexpMatchFile(object): + __slots__ = 'virtual_path', 'real_path', 'hastests', 'archive' + + def __init__(self, virtual_path, real_path, hastests=False, archive=None): + self.virtual_path = virtual_path + self.real_path = real_path + self.hastests = hastests + self.archive = archive + +def regexp(pattern): if not pattern: - yield os.curdir, '' + yield RegexpMatchFile('', os.curdir) return dirname, basename = posixpath.split(pattern) - if hastests: - dirnames = regexp(dirname, True) - else: - dirnames = itertools.chain(regexp(dirname), regexp(posixpath.join(dirname, 'tests'), True, True)) + dirs = regexp(dirname) reobj = re.compile(pattern + '$', re.UNICODE) - for dirname, vdirname in dirnames: - try: - names = os.listdir(dirname) - except OSError: - continue - for name in names: - path = os.path.join(dirname, name) - vpath = posixpath.join(vdirname, name) - if re.match(reobj, vpath): - yield path, vpath if not droplast else vdirname + while dirs: + newdirs = [] + for directory in dirs: + if directory.archive: + try: + names = directory.archive.listdir(directory.real_path) + except KeyError: + continue + join = posixpath.join + else: + try: + names = os.listdir(directory.real_path) + except OSError: + continue + join = posixpath.join + for name in names: + path = join(directory.real_path, name) + vpath = posixpath.join(directory.virtual_path, name) + if re.match(reobj, vpath): + yield RegexpMatchFile(vpath, path, directory.hastests, directory.archive) + if not directory.hastests: + if name == 'tests': + newdirs.append(RegexpMatchFile(directory.virtual_path, path, True, directory.archive)) + if not directory.archive and name in archives: + newdirs.append(RegexpMatchFile(directory.virtual_path, '', False, open_archive(path))) + dirs = newdirs diff -r c2490e39fd70 -r 67088c1765b4 upreckon/problem.py --- a/upreckon/problem.py Sun Aug 14 01:02:10 2011 +0300 +++ b/upreckon/problem.py Mon Aug 15 19:52:58 2011 +0300 @@ -355,13 +355,13 @@ reobj = re.compile(pattern, re.UNICODE) if not group: ids = [] - for path, vpath in files.regexp(pattern): - ids.append(re.match(reobj, vpath).group(1)) + for f in files.regexp(pattern): + ids.append(re.match(reobj, f.virtual_path).group(1)) return natsorted(ids) else: ids = {} - for path, vpath in files.regexp(pattern): - m = re.match(reobj, vpath) + for f in files.regexp(pattern): + m = re.match(reobj, f.virtual_path) g = m.group(group) ids.setdefault(g, []) ids[g].append(m.group(1))