From df830f12389984adeeba04659615b9fc230f2db3 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Thu, 30 Oct 2008 09:21:43 -0700 Subject: Remove import_tar, import_zip and the elements Now that repo relies only on the git data stream (as it is much faster to download through) we don't really need to be parsing the elements within manifest. Its a lot of complex code to convert the tar (or zip) through to a fast import stream, and we just aren't calling it anymore. Signed-off-by: Shawn O. Pearce --- import_ext.py | 422 ---------------------------------------------------------- 1 file changed, 422 deletions(-) delete mode 100644 import_ext.py (limited to 'import_ext.py') diff --git a/import_ext.py b/import_ext.py deleted file mode 100644 index 2a1ebf88..00000000 --- a/import_ext.py +++ /dev/null @@ -1,422 +0,0 @@ -# -# Copyright (C) 2008 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import random -import stat -import sys -import urllib2 -import StringIO - -from error import GitError, ImportError -from git_command import GitCommand - -class ImportExternal(object): - """Imports a single revision from a non-git data source. - Suitable for use to import a tar or zip based snapshot. - """ - def __init__(self): - self._marks = 0 - self._files = {} - self._tempref = 'refs/repo-external/import' - - self._urls = [] - self._remap = [] - self.parent = None - self._user_name = 'Upstream' - self._user_email = 'upstream-import@none' - self._user_when = 1000000 - - self.commit = None - - def Clone(self): - r = self.__class__() - - r.project = self.project - for u in self._urls: - r._urls.append(u) - for p in self._remap: - r._remap.append(_PathMap(r, p._old, p._new)) - - return r - - def SetProject(self, project): - self.project = project - - def SetVersion(self, version): - self.version = version - - def AddUrl(self, url): - self._urls.append(url) - - def SetParent(self, commit_hash): - self.parent = commit_hash - - def SetCommit(self, commit_hash): - self.commit = commit_hash - - def RemapPath(self, old, new, replace_version=True): - self._remap.append(_PathMap(self, old, new)) - - @property - def TagName(self): - v = '' - for c in self.version: - if c >= '0' and c <= '9': - v += c - elif c >= 'A' and c <= 'Z': - v += c - elif c >= 'a' and c <= 'z': - v += c - elif c in ('-', '_', '.', '/', '+', '@'): - v += c - return 'upstream/%s' % v - - @property - def PackageName(self): - n = self.project.name - if n.startswith('platform/'): - # This was not my finest moment... - # - n = n[len('platform/'):] - return n - - def Import(self): - self._need_graft = False - if self.parent: - try: - self.project.bare_git.cat_file('-e', self.parent) - except GitError: - self._need_graft = True - - gfi = GitCommand(self.project, - ['fast-import', '--force', '--quiet'], - bare = True, - provide_stdin = True) - try: - self._out = gfi.stdin - - try: - self._UnpackFiles() - self._MakeCommit() - self._out.flush() - finally: - rc = gfi.Wait() - if rc != 0: - raise ImportError('fast-import failed') - - if self._need_graft: - id = self._GraftCommit() - else: - id = self.project.bare_git.rev_parse('%s^0' % self._tempref) - - if self.commit and self.commit != id: - raise ImportError('checksum mismatch: %s expected,' - ' %s imported' % (self.commit, id)) - - self._MakeTag(id) - return id - finally: - try: - self.project.bare_git.DeleteRef(self._tempref) - except GitError: - pass - - def _PickUrl(self, failed): - u = map(lambda x: x.replace('%version%', self.version), self._urls) - for f in failed: - if f in u: - u.remove(f) - if len(u) == 0: - return None - return random.choice(u) - - def _OpenUrl(self): - failed = {} - while True: - url = self._PickUrl(failed.keys()) - if url is None: - why = 'Cannot download %s' % self.project.name - - if failed: - why += ': one or more mirrors are down\n' - bad_urls = list(failed.keys()) - bad_urls.sort() - for url in bad_urls: - why += ' %s: %s\n' % (url, failed[url]) - else: - why += ': no mirror URLs' - raise ImportError(why) - - print >>sys.stderr, "Getting %s ..." % url - try: - return urllib2.urlopen(url), url - except urllib2.HTTPError, e: - failed[url] = e.code - except urllib2.URLError, e: - failed[url] = e.reason[1] - except OSError, e: - failed[url] = e.strerror - - def _UnpackFiles(self): - raise NotImplementedError - - def _NextMark(self): - self._marks += 1 - return self._marks - - def _UnpackOneFile(self, mode, size, name, fd): - if stat.S_ISDIR(mode): # directory - return - else: - mode = self._CleanMode(mode, name) - - old_name = name - name = self._CleanName(name) - - if stat.S_ISLNK(mode) and self._remap: - # The link is relative to the old_name, and may need to - # be rewritten according to our remap rules if it goes - # up high enough in the tree structure. - # - dest = self._RewriteLink(fd.read(size), old_name, name) - fd = StringIO.StringIO(dest) - size = len(dest) - - fi = _File(mode, name, self._NextMark()) - - self._out.write('blob\n') - self._out.write('mark :%d\n' % fi.mark) - self._out.write('data %d\n' % size) - while size > 0: - n = min(2048, size) - self._out.write(fd.read(n)) - size -= n - self._out.write('\n') - self._files[fi.name] = fi - - def _SetFileMode(self, name, mode): - if not stat.S_ISDIR(mode): - mode = self._CleanMode(mode, name) - name = self._CleanName(name) - try: - fi = self._files[name] - except KeyError: - raise ImportError('file %s was not unpacked' % name) - fi.mode = mode - - def _RewriteLink(self, dest, relto_old, relto_new): - # Drop the last components of the symlink itself - # as the dest is relative to the directory its in. - # - relto_old = _TrimPath(relto_old) - relto_new = _TrimPath(relto_new) - - # Resolve the link to be absolute from the top of - # the archive, so we can remap its destination. - # - while dest.find('/./') >= 0 or dest.find('//') >= 0: - dest = dest.replace('/./', '/') - dest = dest.replace('//', '/') - - if dest.startswith('../') or dest.find('/../') > 0: - dest = _FoldPath('%s/%s' % (relto_old, dest)) - - for pm in self._remap: - if pm.Matches(dest): - dest = pm.Apply(dest) - break - - dest, relto_new = _StripCommonPrefix(dest, relto_new) - while relto_new: - i = relto_new.find('/') - if i > 0: - relto_new = relto_new[i + 1:] - else: - relto_new = '' - dest = '../' + dest - return dest - - def _CleanMode(self, mode, name): - if stat.S_ISREG(mode): # regular file - if (mode & 0111) == 0: - return 0644 - else: - return 0755 - elif stat.S_ISLNK(mode): # symlink - return stat.S_IFLNK - else: - raise ImportError('invalid mode %o in %s' % (mode, name)) - - def _CleanName(self, name): - old_name = name - for pm in self._remap: - if pm.Matches(name): - name = pm.Apply(name) - break - while name.startswith('/'): - name = name[1:] - if not name: - raise ImportError('path %s is empty after remap' % old_name) - if name.find('/./') >= 0 or name.find('/../') >= 0: - raise ImportError('path %s contains relative parts' % name) - return name - - def _MakeCommit(self): - msg = '%s %s\n' % (self.PackageName, self.version) - - self._out.write('commit %s\n' % self._tempref) - self._out.write('committer %s <%s> %d +0000\n' % ( - self._user_name, - self._user_email, - self._user_when)) - self._out.write('data %d\n' % len(msg)) - self._out.write(msg) - self._out.write('\n') - if self.parent and not self._need_graft: - self._out.write('from %s^0\n' % self.parent) - self._out.write('deleteall\n') - - for f in self._files.values(): - self._out.write('M %o :%d %s\n' % (f.mode, f.mark, f.name)) - self._out.write('\n') - - def _GraftCommit(self): - raw = self.project.bare_git.cat_file('commit', self._tempref) - raw = raw.split("\n") - while raw[1].startswith('parent '): - del raw[1] - raw.insert(1, 'parent %s' % self.parent) - id = self._WriteObject('commit', "\n".join(raw)) - - graft_file = os.path.join(self.project.gitdir, 'info/grafts') - if os.path.exists(graft_file): - graft_list = open(graft_file, 'rb').read().split("\n") - if graft_list and graft_list[-1] == '': - del graft_list[-1] - else: - graft_list = [] - - exists = False - for line in graft_list: - if line == id: - exists = True - break - - if not exists: - graft_list.append(id) - graft_list.append('') - fd = open(graft_file, 'wb') - fd.write("\n".join(graft_list)) - fd.close() - - return id - - def _MakeTag(self, id): - name = self.TagName - - raw = [] - raw.append('object %s' % id) - raw.append('type commit') - raw.append('tag %s' % name) - raw.append('tagger %s <%s> %d +0000' % ( - self._user_name, - self._user_email, - self._user_when)) - raw.append('') - raw.append('%s %s\n' % (self.PackageName, self.version)) - - tagid = self._WriteObject('tag', "\n".join(raw)) - self.project.bare_git.UpdateRef('refs/tags/%s' % name, tagid) - - def _WriteObject(self, type, data): - wo = GitCommand(self.project, - ['hash-object', '-t', type, '-w', '--stdin'], - bare = True, - provide_stdin = True, - capture_stdout = True, - capture_stderr = True) - wo.stdin.write(data) - if wo.Wait() != 0: - raise GitError('cannot create %s from (%s)' % (type, data)) - return wo.stdout[:-1] - - -def _TrimPath(path): - i = path.rfind('/') - if i > 0: - path = path[0:i] - return '' - -def _StripCommonPrefix(a, b): - while True: - ai = a.find('/') - bi = b.find('/') - if ai > 0 and bi > 0 and a[0:ai] == b[0:bi]: - a = a[ai + 1:] - b = b[bi + 1:] - else: - break - return a, b - -def _FoldPath(path): - while True: - if path.startswith('../'): - return path - - i = path.find('/../') - if i <= 0: - if path.startswith('/'): - return path[1:] - return path - - lhs = path[0:i] - rhs = path[i + 4:] - - i = lhs.rfind('/') - if i > 0: - path = lhs[0:i + 1] + rhs - else: - path = rhs - -class _File(object): - def __init__(self, mode, name, mark): - self.mode = mode - self.name = name - self.mark = mark - - -class _PathMap(object): - def __init__(self, imp, old, new): - self._imp = imp - self._old = old - self._new = new - - def _r(self, p): - return p.replace('%version%', self._imp.version) - - @property - def old(self): - return self._r(self._old) - - @property - def new(self): - return self._r(self._new) - - def Matches(self, name): - return name.startswith(self.old) - - def Apply(self, name): - return self.new + name[len(self.old):] -- cgit v1.2.3-54-g00ecf