summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorShawn O. Pearce <sop@google.com>2008-10-30 09:21:43 -0700
committerShawn O. Pearce <sop@google.com>2008-10-30 09:21:43 -0700
commitdf830f12389984adeeba04659615b9fc230f2db3 (patch)
treeea5857656f63f9b375dbb9851e8801062570c24d
parent90be5c0839762d14bdee75d121c3923ed91c8404 (diff)
downloadgit-repo-df830f12389984adeeba04659615b9fc230f2db3.tar.gz
Remove import_tar, import_zip and the <snapshot> elementsv1.0.7
Now that repo relies only on the git data stream (as it is much faster to download through) we don't really need to be parsing the <snapshot> elements within manifest. Its a lot of complex code to convert the tar (or zip) through to a fast import stream, and we just aren't calling it anymore. Signed-off-by: Shawn O. Pearce <sop@google.com>
Diffstat
-rw-r--r--import_ext.py422
-rw-r--r--import_tar.py206
-rw-r--r--import_zip.py345
-rw-r--r--manifest.py72
-rw-r--r--subcmds/compute_snapshot_check.py169
5 files changed, 0 insertions, 1214 deletions
diff --git a/import_ext.py b/import_ext.py
deleted file mode 100644
index 2a1ebf88..00000000
--- a/import_ext.py
+++ /dev/null
@@ -1,422 +0,0 @@
1#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import os
17import random
18import stat
19import sys
20import urllib2
21import StringIO
22
23from error import GitError, ImportError
24from git_command import GitCommand
25
26class ImportExternal(object):
27 """Imports a single revision from a non-git data source.
28 Suitable for use to import a tar or zip based snapshot.
29 """
30 def __init__(self):
31 self._marks = 0
32 self._files = {}
33 self._tempref = 'refs/repo-external/import'
34
35 self._urls = []
36 self._remap = []
37 self.parent = None
38 self._user_name = 'Upstream'
39 self._user_email = 'upstream-import@none'
40 self._user_when = 1000000
41
42 self.commit = None
43
44 def Clone(self):
45 r = self.__class__()
46
47 r.project = self.project
48 for u in self._urls:
49 r._urls.append(u)
50 for p in self._remap:
51 r._remap.append(_PathMap(r, p._old, p._new))
52
53 return r
54
55 def SetProject(self, project):
56 self.project = project
57
58 def SetVersion(self, version):
59 self.version = version
60
61 def AddUrl(self, url):
62 self._urls.append(url)
63
64 def SetParent(self, commit_hash):
65 self.parent = commit_hash
66
67 def SetCommit(self, commit_hash):
68 self.commit = commit_hash
69
70 def RemapPath(self, old, new, replace_version=True):
71 self._remap.append(_PathMap(self, old, new))
72
73 @property
74 def TagName(self):
75 v = ''
76 for c in self.version:
77 if c >= '0' and c <= '9':
78 v += c
79 elif c >= 'A' and c <= 'Z':
80 v += c
81 elif c >= 'a' and c <= 'z':
82 v += c
83 elif c in ('-', '_', '.', '/', '+', '@'):
84 v += c
85 return 'upstream/%s' % v
86
87 @property
88 def PackageName(self):
89 n = self.project.name
90 if n.startswith('platform/'):
91 # This was not my finest moment...
92 #
93 n = n[len('platform/'):]
94 return n
95
96 def Import(self):
97 self._need_graft = False
98 if self.parent:
99 try:
100 self.project.bare_git.cat_file('-e', self.parent)
101 except GitError:
102 self._need_graft = True
103
104 gfi = GitCommand(self.project,
105 ['fast-import', '--force', '--quiet'],
106 bare = True,
107 provide_stdin = True)
108 try:
109 self._out = gfi.stdin
110
111 try:
112 self._UnpackFiles()
113 self._MakeCommit()
114 self._out.flush()
115 finally:
116 rc = gfi.Wait()
117 if rc != 0:
118 raise ImportError('fast-import failed')
119
120 if self._need_graft:
121 id = self._GraftCommit()
122 else:
123 id = self.project.bare_git.rev_parse('%s^0' % self._tempref)
124
125 if self.commit and self.commit != id:
126 raise ImportError('checksum mismatch: %s expected,'
127 ' %s imported' % (self.commit, id))
128
129 self._MakeTag(id)
130 return id
131 finally:
132 try:
133 self.project.bare_git.DeleteRef(self._tempref)
134 except GitError:
135 pass
136
137 def _PickUrl(self, failed):
138 u = map(lambda x: x.replace('%version%', self.version), self._urls)
139 for f in failed:
140 if f in u:
141 u.remove(f)
142 if len(u) == 0:
143 return None
144 return random.choice(u)
145
146 def _OpenUrl(self):
147 failed = {}
148 while True:
149 url = self._PickUrl(failed.keys())
150 if url is None:
151 why = 'Cannot download %s' % self.project.name
152
153 if failed:
154 why += ': one or more mirrors are down\n'
155 bad_urls = list(failed.keys())
156 bad_urls.sort()
157 for url in bad_urls:
158 why += ' %s: %s\n' % (url, failed[url])
159 else:
160 why += ': no mirror URLs'
161 raise ImportError(why)
162
163 print >>sys.stderr, "Getting %s ..." % url
164 try:
165 return urllib2.urlopen(url), url
166 except urllib2.HTTPError, e:
167 failed[url] = e.code
168 except urllib2.URLError, e:
169 failed[url] = e.reason[1]
170 except OSError, e:
171 failed[url] = e.strerror
172
173 def _UnpackFiles(self):
174 raise NotImplementedError
175
176 def _NextMark(self):
177 self._marks += 1
178 return self._marks
179
180 def _UnpackOneFile(self, mode, size, name, fd):
181 if stat.S_ISDIR(mode): # directory
182 return
183 else:
184 mode = self._CleanMode(mode, name)
185
186 old_name = name
187 name = self._CleanName(name)
188
189 if stat.S_ISLNK(mode) and self._remap:
190 # The link is relative to the old_name, and may need to
191 # be rewritten according to our remap rules if it goes
192 # up high enough in the tree structure.
193 #
194 dest = self._RewriteLink(fd.read(size), old_name, name)
195 fd = StringIO.StringIO(dest)
196 size = len(dest)
197
198 fi = _File(mode, name, self._NextMark())
199
200 self._out.write('blob\n')
201 self._out.write('mark :%d\n' % fi.mark)
202 self._out.write('data %d\n' % size)
203 while size > 0:
204 n = min(2048, size)
205 self._out.write(fd.read(n))
206 size -= n
207 self._out.write('\n')
208 self._files[fi.name] = fi
209
210 def _SetFileMode(self, name, mode):
211 if not stat.S_ISDIR(mode):
212 mode = self._CleanMode(mode, name)
213 name = self._CleanName(name)
214 try:
215 fi = self._files[name]
216 except KeyError:
217 raise ImportError('file %s was not unpacked' % name)
218 fi.mode = mode
219
220 def _RewriteLink(self, dest, relto_old, relto_new):
221 # Drop the last components of the symlink itself
222 # as the dest is relative to the directory its in.
223 #
224 relto_old = _TrimPath(relto_old)
225 relto_new = _TrimPath(relto_new)
226
227 # Resolve the link to be absolute from the top of
228 # the archive, so we can remap its destination.
229 #
230 while dest.find('/./') >= 0 or dest.find('//') >= 0:
231 dest = dest.replace('/./', '/')
232 dest = dest.replace('//', '/')
233
234 if dest.startswith('../') or dest.find('/../') > 0:
235 dest = _FoldPath('%s/%s' % (relto_old, dest))
236
237 for pm in self._remap:
238 if pm.Matches(dest):
239 dest = pm.Apply(dest)
240 break
241
242 dest, relto_new = _StripCommonPrefix(dest, relto_new)
243 while relto_new:
244 i = relto_new.find('/')
245 if i > 0:
246 relto_new = relto_new[i + 1:]
247 else:
248 relto_new = ''
249 dest = '../' + dest
250 return dest
251
252 def _CleanMode(self, mode, name):
253 if stat.S_ISREG(mode): # regular file
254 if (mode & 0111) == 0:
255 return 0644
256 else:
257 return 0755
258 elif stat.S_ISLNK(mode): # symlink
259 return stat.S_IFLNK
260 else:
261 raise ImportError('invalid mode %o in %s' % (mode, name))
262
263 def _CleanName(self, name):
264 old_name = name
265 for pm in self._remap:
266 if pm.Matches(name):
267 name = pm.Apply(name)
268 break
269 while name.startswith('/'):
270 name = name[1:]
271 if not name:
272 raise ImportError('path %s is empty after remap' % old_name)
273 if name.find('/./') >= 0 or name.find('/../') >= 0:
274 raise ImportError('path %s contains relative parts' % name)
275 return name
276
277 def _MakeCommit(self):
278 msg = '%s %s\n' % (self.PackageName, self.version)
279
280 self._out.write('commit %s\n' % self._tempref)
281 self._out.write('committer %s <%s> %d +0000\n' % (
282 self._user_name,
283 self._user_email,
284 self._user_when))
285 self._out.write('data %d\n' % len(msg))
286 self._out.write(msg)
287 self._out.write('\n')
288 if self.parent and not self._need_graft:
289 self._out.write('from %s^0\n' % self.parent)
290 self._out.write('deleteall\n')
291
292 for f in self._files.values():
293 self._out.write('M %o :%d %s\n' % (f.mode, f.mark, f.name))
294 self._out.write('\n')
295
296 def _GraftCommit(self):
297 raw = self.project.bare_git.cat_file('commit', self._tempref)
298 raw = raw.split("\n")
299 while raw[1].startswith('parent '):
300 del raw[1]
301 raw.insert(1, 'parent %s' % self.parent)
302 id = self._WriteObject('commit', "\n".join(raw))
303
304 graft_file = os.path.join(self.project.gitdir, 'info/grafts')
305 if os.path.exists(graft_file):
306 graft_list = open(graft_file, 'rb').read().split("\n")
307 if graft_list and graft_list[-1] == '':
308 del graft_list[-1]
309 else:
310 graft_list = []
311
312 exists = False
313 for line in graft_list:
314 if line == id:
315 exists = True
316 break
317
318 if not exists:
319 graft_list.append(id)
320 graft_list.append('')
321 fd = open(graft_file, 'wb')
322 fd.write("\n".join(graft_list))
323 fd.close()
324
325 return id
326
327 def _MakeTag(self, id):
328 name = self.TagName
329
330 raw = []
331 raw.append('object %s' % id)
332 raw.append('type commit')
333 raw.append('tag %s' % name)
334 raw.append('tagger %s <%s> %d +0000' % (
335 self._user_name,
336 self._user_email,
337 self._user_when))
338 raw.append('')
339 raw.append('%s %s\n' % (self.PackageName, self.version))
340
341 tagid = self._WriteObject('tag', "\n".join(raw))
342 self.project.bare_git.UpdateRef('refs/tags/%s' % name, tagid)
343
344 def _WriteObject(self, type, data):
345 wo = GitCommand(self.project,
346 ['hash-object', '-t', type, '-w', '--stdin'],
347 bare = True,
348 provide_stdin = True,
349 capture_stdout = True,
350 capture_stderr = True)
351 wo.stdin.write(data)
352 if wo.Wait() != 0:
353 raise GitError('cannot create %s from (%s)' % (type, data))
354 return wo.stdout[:-1]
355
356
357def _TrimPath(path):
358 i = path.rfind('/')
359 if i > 0:
360 path = path[0:i]
361 return ''
362
363def _StripCommonPrefix(a, b):
364 while True:
365 ai = a.find('/')
366 bi = b.find('/')
367 if ai > 0 and bi > 0 and a[0:ai] == b[0:bi]:
368 a = a[ai + 1:]
369 b = b[bi + 1:]
370 else:
371 break
372 return a, b
373
374def _FoldPath(path):
375 while True:
376 if path.startswith('../'):
377 return path
378
379 i = path.find('/../')
380 if i <= 0:
381 if path.startswith('/'):
382 return path[1:]
383 return path
384
385 lhs = path[0:i]
386 rhs = path[i + 4:]
387
388 i = lhs.rfind('/')
389 if i > 0:
390 path = lhs[0:i + 1] + rhs
391 else:
392 path = rhs
393
394class _File(object):
395 def __init__(self, mode, name, mark):
396 self.mode = mode
397 self.name = name
398 self.mark = mark
399
400
401class _PathMap(object):
402 def __init__(self, imp, old, new):
403 self._imp = imp
404 self._old = old
405 self._new = new
406
407 def _r(self, p):
408 return p.replace('%version%', self._imp.version)
409
410 @property
411 def old(self):
412 return self._r(self._old)
413
414 @property
415 def new(self):
416 return self._r(self._new)
417
418 def Matches(self, name):
419 return name.startswith(self.old)
420
421 def Apply(self, name):
422 return self.new + name[len(self.old):]
diff --git a/import_tar.py b/import_tar.py
deleted file mode 100644
index d7ce14de..00000000
--- a/import_tar.py
+++ /dev/null
@@ -1,206 +0,0 @@
1#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import bz2
17import stat
18import tarfile
19import zlib
20import StringIO
21
22from import_ext import ImportExternal
23from error import ImportError
24
25class ImportTar(ImportExternal):
26 """Streams a (optionally compressed) tar file from the network
27 directly into a Project's Git repository.
28 """
29 @classmethod
30 def CanAccept(cls, url):
31 """Can this importer read and unpack the data stored at url?
32 """
33 if url.endswith('.tar.gz') or url.endswith('.tgz'):
34 return True
35 if url.endswith('.tar.bz2'):
36 return True
37 if url.endswith('.tar'):
38 return True
39 return False
40
41 def _UnpackFiles(self):
42 url_fd, url = self._OpenUrl()
43 try:
44 if url.endswith('.tar.gz') or url.endswith('.tgz'):
45 tar_fd = _Gzip(url_fd)
46 elif url.endswith('.tar.bz2'):
47 tar_fd = _Bzip2(url_fd)
48 elif url.endswith('.tar'):
49 tar_fd = _Raw(url_fd)
50 else:
51 raise ImportError('non-tar file extension: %s' % url)
52
53 try:
54 tar = tarfile.TarFile(name = url,
55 mode = 'r',
56 fileobj = tar_fd)
57 try:
58 for entry in tar:
59 mode = entry.mode
60
61 if (mode & 0170000) == 0:
62 if entry.isdir():
63 mode |= stat.S_IFDIR
64 elif entry.isfile() or entry.islnk(): # hard links as files
65 mode |= stat.S_IFREG
66 elif entry.issym():
67 mode |= stat.S_IFLNK
68
69 if stat.S_ISLNK(mode): # symlink
70 data_fd = StringIO.StringIO(entry.linkname)
71 data_sz = len(entry.linkname)
72 elif stat.S_ISDIR(mode): # directory
73 data_fd = StringIO.StringIO('')
74 data_sz = 0
75 else:
76 data_fd = tar.extractfile(entry)
77 data_sz = entry.size
78
79 self._UnpackOneFile(mode, data_sz, entry.name, data_fd)
80 finally:
81 tar.close()
82 finally:
83 tar_fd.close()
84 finally:
85 url_fd.close()
86
87
88
89class _DecompressStream(object):
90 """file like object to decompress a tar stream
91 """
92 def __init__(self, fd):
93 self._fd = fd
94 self._pos = 0
95 self._buf = None
96
97 def tell(self):
98 return self._pos
99
100 def seek(self, offset):
101 d = offset - self._pos
102 if d > 0:
103 self.read(d)
104 elif d == 0:
105 pass
106 else:
107 raise NotImplementedError, 'seek backwards'
108
109 def close(self):
110 self._fd = None
111
112 def read(self, size = -1):
113 if not self._fd:
114 raise EOFError, 'Reached EOF'
115
116 r = []
117 try:
118 if size >= 0:
119 self._ReadChunk(r, size)
120 else:
121 while True:
122 self._ReadChunk(r, 2048)
123 except EOFError:
124 pass
125
126 if len(r) == 1:
127 r = r[0]
128 else:
129 r = ''.join(r)
130 self._pos += len(r)
131 return r
132
133 def _ReadChunk(self, r, size):
134 b = self._buf
135 try:
136 while size > 0:
137 if b is None or len(b) == 0:
138 b = self._Decompress(self._fd.read(2048))
139 continue
140
141 use = min(size, len(b))
142 r.append(b[:use])
143 b = b[use:]
144 size -= use
145 finally:
146 self._buf = b
147
148 def _Decompress(self, b):
149 raise NotImplementedError, '_Decompress'
150
151
152class _Raw(_DecompressStream):
153 """file like object for an uncompressed stream
154 """
155 def __init__(self, fd):
156 _DecompressStream.__init__(self, fd)
157
158 def _Decompress(self, b):
159 return b
160
161
162class _Bzip2(_DecompressStream):
163 """file like object to decompress a .bz2 stream
164 """
165 def __init__(self, fd):
166 _DecompressStream.__init__(self, fd)
167 self._bz = bz2.BZ2Decompressor()
168
169 def _Decompress(self, b):
170 return self._bz.decompress(b)
171
172
173_FHCRC, _FEXTRA, _FNAME, _FCOMMENT = 2, 4, 8, 16
174class _Gzip(_DecompressStream):
175 """file like object to decompress a .gz stream
176 """
177 def __init__(self, fd):
178 _DecompressStream.__init__(self, fd)
179 self._z = zlib.decompressobj(-zlib.MAX_WBITS)
180
181 magic = fd.read(2)
182 if magic != '\037\213':
183 raise IOError, 'Not a gzipped file'
184
185 method = ord(fd.read(1))
186 if method != 8:
187 raise IOError, 'Unknown compression method'
188
189 flag = ord(fd.read(1))
190 fd.read(6)
191
192 if flag & _FEXTRA:
193 xlen = ord(fd.read(1))
194 xlen += 256 * ord(fd.read(1))
195 fd.read(xlen)
196 if flag & _FNAME:
197 while fd.read(1) != '\0':
198 pass
199 if flag & _FCOMMENT:
200 while fd.read(1) != '\0':
201 pass
202 if flag & _FHCRC:
203 fd.read(2)
204
205 def _Decompress(self, b):
206 return self._z.decompress(b)
diff --git a/import_zip.py b/import_zip.py
deleted file mode 100644
index 08aff326..00000000
--- a/import_zip.py
+++ /dev/null
@@ -1,345 +0,0 @@
1#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import stat
17import struct
18import zlib
19import cStringIO
20
21from import_ext import ImportExternal
22from error import ImportError
23
24class ImportZip(ImportExternal):
25 """Streams a zip file from the network directly into a Project's
26 Git repository.
27 """
28 @classmethod
29 def CanAccept(cls, url):
30 """Can this importer read and unpack the data stored at url?
31 """
32 if url.endswith('.zip') or url.endswith('.jar'):
33 return True
34 return False
35
36 def _UnpackFiles(self):
37 url_fd, url = self._OpenUrl()
38 try:
39 if not self.__class__.CanAccept(url):
40 raise ImportError('non-zip file extension: %s' % url)
41
42 zip = _ZipFile(url_fd)
43 for entry in zip.FileRecords():
44 data = zip.Open(entry).read()
45 sz = len(data)
46
47 if data and _SafeCRLF(data):
48 data = data.replace('\r\n', '\n')
49 sz = len(data)
50
51 fd = cStringIO.StringIO(data)
52 self._UnpackOneFile(entry.mode, sz, entry.name, fd)
53 zip.Close(entry)
54
55 for entry in zip.CentralDirectory():
56 self._SetFileMode(entry.name, entry.mode)
57
58 zip.CheckTail()
59 finally:
60 url_fd.close()
61
62
63def _SafeCRLF(data):
64 """Is it reasonably safe to perform a CRLF->LF conversion?
65
66 If the stream contains a NUL byte it is likely binary,
67 and thus a CRLF->LF conversion may damage the stream.
68
69 If the only NUL is in the last position of the stream,
70 but it otherwise can do a CRLF<->LF conversion we do
71 the CRLF conversion anyway. At least one source ZIP
72 file has this structure in its source code.
73
74 If every occurrance of a CR and LF is paired up as a
75 CRLF pair then the conversion is safely bi-directional.
76 s/\r\n/\n/g == s/\n/\r\\n/g can convert between them.
77 """
78 nul = data.find('\0')
79 if 0 <= nul and nul < (len(data) - 1):
80 return False
81
82 n_lf = 0
83 last = 0
84 while True:
85 lf = data.find('\n', last)
86 if lf < 0:
87 break
88 if lf == 0 or data[lf - 1] != '\r':
89 return False
90 last = lf + 1
91 n_lf += 1
92 return n_lf > 0
93
94class _ZipFile(object):
95 """Streaming iterator to parse a zip file on the fly.
96 """
97 def __init__(self, fd):
98 self._fd = _UngetStream(fd)
99
100 def FileRecords(self):
101 return _FileIter(self._fd)
102
103 def CentralDirectory(self):
104 return _CentIter(self._fd)
105
106 def CheckTail(self):
107 type_buf = self._fd.read(4)
108 type = struct.unpack('<I', type_buf)[0]
109 if type != 0x06054b50: # end of central directory
110 raise ImportError('zip record %x unsupported' % type)
111
112 def Open(self, entry):
113 if entry.is_compressed:
114 return _InflateStream(self._fd)
115 else:
116 if entry.has_trailer:
117 raise ImportError('unable to extract streamed zip')
118 return _FixedLengthStream(self._fd, entry.uncompressed_size)
119
120 def Close(self, entry):
121 if entry.has_trailer:
122 type = struct.unpack('<I', self._fd.read(4))[0]
123 if type == 0x08074b50:
124 # Not a formal type marker, but commonly seen in zips
125 # as the data descriptor signature.
126 #
127 struct.unpack('<3I', self._fd.read(12))
128 else:
129 # No signature for the data descriptor, so read the
130 # remaining fields out of the stream
131 #
132 self._fd.read(8)
133
134
135class _FileIter(object):
136 def __init__(self, fd):
137 self._fd = fd
138
139 def __iter__(self):
140 return self
141
142 def next(self):
143 fd = self._fd
144
145 type_buf = fd.read(4)
146 type = struct.unpack('<I', type_buf)[0]
147
148 if type != 0x04034b50: # local file header
149 fd.unread(type_buf)
150 raise StopIteration()
151
152 rec = _FileHeader(fd.read(26))
153 rec.name = fd.read(rec.name_len)
154 fd.read(rec.extra_len)
155
156 if rec.name.endswith('/'):
157 rec.name = rec.name[:-1]
158 rec.mode = stat.S_IFDIR | 0777
159 return rec
160
161
162class _FileHeader(object):
163 """Information about a single file in the archive.
164 0 version needed to extract 2 bytes
165 1 general purpose bit flag 2 bytes
166 2 compression method 2 bytes
167 3 last mod file time 2 bytes
168 4 last mod file date 2 bytes
169 5 crc-32 4 bytes
170 6 compressed size 4 bytes
171 7 uncompressed size 4 bytes
172 8 file name length 2 bytes
173 9 extra field length 2 bytes
174 """
175 def __init__(self, raw_bin):
176 rec = struct.unpack('<5H3I2H', raw_bin)
177
178 if rec[2] == 8:
179 self.is_compressed = True
180 elif rec[2] == 0:
181 self.is_compressed = False
182 else:
183 raise ImportError('unrecognized compression format')
184
185 if rec[1] & (1 << 3):
186 self.has_trailer = True
187 else:
188 self.has_trailer = False
189
190 self.compressed_size = rec[6]
191 self.uncompressed_size = rec[7]
192 self.name_len = rec[8]
193 self.extra_len = rec[9]
194 self.mode = stat.S_IFREG | 0644
195
196
197class _CentIter(object):
198 def __init__(self, fd):
199 self._fd = fd
200
201 def __iter__(self):
202 return self
203
204 def next(self):
205 fd = self._fd
206
207 type_buf = fd.read(4)
208 type = struct.unpack('<I', type_buf)[0]
209
210 if type != 0x02014b50: # central directory
211 fd.unread(type_buf)
212 raise StopIteration()
213
214 rec = _CentHeader(fd.read(42))
215 rec.name = fd.read(rec.name_len)
216 fd.read(rec.extra_len)
217 fd.read(rec.comment_len)
218
219 if rec.name.endswith('/'):
220 rec.name = rec.name[:-1]
221 rec.mode = stat.S_IFDIR | 0777
222 return rec
223
224
225class _CentHeader(object):
226 """Information about a single file in the archive.
227 0 version made by 2 bytes
228 1 version needed to extract 2 bytes
229 2 general purpose bit flag 2 bytes
230 3 compression method 2 bytes
231 4 last mod file time 2 bytes
232 5 last mod file date 2 bytes
233 6 crc-32 4 bytes
234 7 compressed size 4 bytes
235 8 uncompressed size 4 bytes
236 9 file name length 2 bytes
237 10 extra field length 2 bytes
238 11 file comment length 2 bytes
239 12 disk number start 2 bytes
240 13 internal file attributes 2 bytes
241 14 external file attributes 4 bytes
242 15 relative offset of local header 4 bytes
243 """
244 def __init__(self, raw_bin):
245 rec = struct.unpack('<6H3I5H2I', raw_bin)
246 self.name_len = rec[9]
247 self.extra_len = rec[10]
248 self.comment_len = rec[11]
249
250 if (rec[0] & 0xff00) == 0x0300: # UNIX
251 self.mode = rec[14] >> 16
252 else:
253 self.mode = stat.S_IFREG | 0644
254
255
256class _UngetStream(object):
257 """File like object to read and rewind a stream.
258 """
259 def __init__(self, fd):
260 self._fd = fd
261 self._buf = None
262
263 def read(self, size = -1):
264 r = []
265 try:
266 if size >= 0:
267 self._ReadChunk(r, size)
268 else:
269 while True:
270 self._ReadChunk(r, 2048)
271 except EOFError:
272 pass
273
274 if len(r) == 1:
275 return r[0]
276 return ''.join(r)
277
278 def unread(self, buf):
279 b = self._buf
280 if b is None or len(b) == 0:
281 self._buf = buf
282 else:
283 self._buf = buf + b
284
285 def _ReadChunk(self, r, size):
286 b = self._buf
287 try:
288 while size > 0:
289 if b is None or len(b) == 0:
290 b = self._Inflate(self._fd.read(2048))
291 if not b:
292 raise EOFError()
293 continue
294
295 use = min(size, len(b))
296 r.append(b[:use])
297 b = b[use:]
298 size -= use
299 finally:
300 self._buf = b
301
302 def _Inflate(self, b):
303 return b
304
305
306class _FixedLengthStream(_UngetStream):
307 """File like object to read a fixed length stream.
308 """
309 def __init__(self, fd, have):
310 _UngetStream.__init__(self, fd)
311 self._have = have
312
313 def _Inflate(self, b):
314 n = self._have
315 if n == 0:
316 self._fd.unread(b)
317 return None
318
319 if len(b) > n:
320 self._fd.unread(b[n:])
321 b = b[:n]
322 self._have -= len(b)
323 return b
324
325
326class _InflateStream(_UngetStream):
327 """Inflates the stream as it reads input.
328 """
329 def __init__(self, fd):
330 _UngetStream.__init__(self, fd)
331 self._z = zlib.decompressobj(-zlib.MAX_WBITS)
332
333 def _Inflate(self, b):
334 z = self._z
335 if not z:
336 self._fd.unread(b)
337 return None
338
339 b = z.decompress(b)
340 if z.unconsumed_tail != '':
341 self._fd.unread(z.unconsumed_tail)
342 elif z.unused_data != '':
343 self._fd.unread(z.unused_data)
344 self._z = None
345 return b
diff --git a/manifest.py b/manifest.py
index 8c5a8d6d..ffff14a5 100644
--- a/manifest.py
+++ b/manifest.py
@@ -18,8 +18,6 @@ import sys
18import xml.dom.minidom 18import xml.dom.minidom
19 19
20from git_config import GitConfig, IsId 20from git_config import GitConfig, IsId
21from import_tar import ImportTar
22from import_zip import ImportZip
23from project import Project, MetaProject, R_TAGS 21from project import Project, MetaProject, R_TAGS
24from remote import Remote 22from remote import Remote
25from error import ManifestParseError 23from error import ManifestParseError
@@ -245,78 +243,8 @@ class Manifest(object):
245 elif n.nodeName == 'copyfile': 243 elif n.nodeName == 'copyfile':
246 self._ParseCopyFile(project, n) 244 self._ParseCopyFile(project, n)
247 245
248 to_resolve = []
249 by_version = {}
250
251 for n in node.childNodes:
252 if n.nodeName == 'import':
253 self._ParseImport(project, n, to_resolve, by_version)
254
255 for pair in to_resolve:
256 sn, pr = pair
257 try:
258 sn.SetParent(by_version[pr].commit)
259 except KeyError:
260 raise ManifestParseError, \
261 'snapshot %s not in project %s in %s' % \
262 (pr, project.name, self.manifestFile)
263
264 return project 246 return project
265 247
266 def _ParseImport(self, project, import_node, to_resolve, by_version):
267 first_url = None
268 for node in import_node.childNodes:
269 if node.nodeName == 'mirror':
270 first_url = self._reqatt(node, 'url')
271 break
272 if not first_url:
273 raise ManifestParseError, \
274 'mirror url required for project %s in %s' % \
275 (project.name, self.manifestFile)
276
277 imp = None
278 for cls in [ImportTar, ImportZip]:
279 if cls.CanAccept(first_url):
280 imp = cls()
281 break
282 if not imp:
283 raise ManifestParseError, \
284 'snapshot %s unsupported for project %s in %s' % \
285 (first_url, project.name, self.manifestFile)
286
287 imp.SetProject(project)
288
289 for node in import_node.childNodes:
290 if node.nodeName == 'remap':
291 old = node.getAttribute('strip')
292 new = node.getAttribute('insert')
293 imp.RemapPath(old, new)
294
295 elif node.nodeName == 'mirror':
296 imp.AddUrl(self._reqatt(node, 'url'))
297
298 for node in import_node.childNodes:
299 if node.nodeName == 'snapshot':
300 sn = imp.Clone()
301 sn.SetVersion(self._reqatt(node, 'version'))
302 sn.SetCommit(node.getAttribute('check'))
303
304 pr = node.getAttribute('prior')
305 if pr:
306 if IsId(pr):
307 sn.SetParent(pr)
308 else:
309 to_resolve.append((sn, pr))
310
311 rev = R_TAGS + sn.TagName
312
313 if rev in project.snapshots:
314 raise ManifestParseError, \
315 'duplicate snapshot %s for project %s in %s' % \
316 (sn.version, project.name, self.manifestFile)
317 project.snapshots[rev] = sn
318 by_version[sn.version] = sn
319
320 def _ParseCopyFile(self, project, node): 248 def _ParseCopyFile(self, project, node):
321 src = self._reqatt(node, 'src') 249 src = self._reqatt(node, 'src')
322 dest = self._reqatt(node, 'dest') 250 dest = self._reqatt(node, 'dest')
diff --git a/subcmds/compute_snapshot_check.py b/subcmds/compute_snapshot_check.py
deleted file mode 100644
index 82db359a..00000000
--- a/subcmds/compute_snapshot_check.py
+++ /dev/null
@@ -1,169 +0,0 @@
1#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import os
17import sys
18import tempfile
19
20from command import Command
21from error import GitError, NoSuchProjectError
22from git_config import IsId
23from import_tar import ImportTar
24from import_zip import ImportZip
25from project import Project
26from remote import Remote
27
28def _ToCommit(project, rev):
29 return project.bare_git.rev_parse('--verify', '%s^0' % rev)
30
31def _Missing(project, rev):
32 return project._revlist('--objects', rev, '--not', '--all')
33
34
35class ComputeSnapshotCheck(Command):
36 common = False
37 helpSummary = "Compute the check value for a new snapshot"
38 helpUsage = """
39%prog -p NAME -v VERSION -s FILE [options]
40"""
41 helpDescription = """
42%prog computes and then displays the proper check value for a
43snapshot, so it can be pasted into the manifest file for a project.
44"""
45
46 def _Options(self, p):
47 g = p.add_option_group('Snapshot description options')
48 g.add_option('-p', '--project',
49 dest='project', metavar='NAME',
50 help='destination project name')
51 g.add_option('-v', '--version',
52 dest='version', metavar='VERSION',
53 help='upstream version/revision identifier')
54 g.add_option('-s', '--snapshot',
55 dest='snapshot', metavar='PATH',
56 help='local tarball path')
57 g.add_option('--new-project',
58 dest='new_project', action='store_true',
59 help='destinition is a new project')
60 g.add_option('--keep',
61 dest='keep_git', action='store_true',
62 help='keep the temporary git repository')
63
64 g = p.add_option_group('Base revision grafting options')
65 g.add_option('--prior',
66 dest='prior', metavar='COMMIT',
67 help='prior revision checksum')
68
69 g = p.add_option_group('Path mangling options')
70 g.add_option('--strip-prefix',
71 dest='strip_prefix', metavar='PREFIX',
72 help='remove prefix from all paths on import')
73 g.add_option('--insert-prefix',
74 dest='insert_prefix', metavar='PREFIX',
75 help='insert prefix before all paths on import')
76
77
78 def _Compute(self, opt):
79 try:
80 real_project = self.GetProjects([opt.project])[0]
81 except NoSuchProjectError:
82 if opt.new_project:
83 print >>sys.stderr, \
84 "warning: project '%s' does not exist" % opt.project
85 else:
86 raise NoSuchProjectError(opt.project)
87
88 self._tmpdir = tempfile.mkdtemp()
89 project = Project(manifest = self.manifest,
90 name = opt.project,
91 remote = Remote('origin'),
92 gitdir = os.path.join(self._tmpdir, '.git'),
93 worktree = self._tmpdir,
94 relpath = opt.project,
95 revision = 'refs/heads/master')
96 project._InitGitDir()
97
98 url = 'file://%s' % os.path.abspath(opt.snapshot)
99
100 imp = None
101 for cls in [ImportTar, ImportZip]:
102 if cls.CanAccept(url):
103 imp = cls()
104 break
105 if not imp:
106 print >>sys.stderr, 'error: %s unsupported' % opt.snapshot
107 sys.exit(1)
108
109 imp.SetProject(project)
110 imp.SetVersion(opt.version)
111 imp.AddUrl(url)
112
113 if opt.prior:
114 if opt.new_project:
115 if not IsId(opt.prior):
116 print >>sys.stderr, 'error: --prior=%s not valid' % opt.prior
117 sys.exit(1)
118 else:
119 try:
120 opt.prior = _ToCommit(real_project, opt.prior)
121 missing = _Missing(real_project, opt.prior)
122 except GitError, e:
123 print >>sys.stderr,\
124 'error: --prior=%s not valid\n%s' \
125 % (opt.prior, e)
126 sys.exit(1)
127 if missing:
128 print >>sys.stderr,\
129 'error: --prior=%s is valid, but is not reachable' \
130 % opt.prior
131 sys.exit(1)
132 imp.SetParent(opt.prior)
133
134 src = opt.strip_prefix
135 dst = opt.insert_prefix
136 if src or dst:
137 if src is None:
138 src = ''
139 if dst is None:
140 dst = ''
141 imp.RemapPath(src, dst)
142 commitId = imp.Import()
143
144 print >>sys.stderr,"%s\t%s" % (commitId, imp.version)
145 return project
146
147 def Execute(self, opt, args):
148 if args \
149 or not opt.project \
150 or not opt.version \
151 or not opt.snapshot:
152 self.Usage()
153
154 success = False
155 project = None
156 try:
157 self._tmpdir = None
158 project = self._Compute(opt)
159 finally:
160 if project and opt.keep_git:
161 print 'GIT_DIR = %s' % (project.gitdir)
162 elif self._tmpdir:
163 for root, dirs, files in os.walk(self._tmpdir, topdown=False):
164 for name in files:
165 os.remove(os.path.join(root, name))
166 for name in dirs:
167 os.rmdir(os.path.join(root, name))
168 os.rmdir(self._tmpdir)
169
generated by cgit v1.2.3-54-g00ecf (git 2.39.0) at 2025-05-13 16:26:24 +0000