diff options
Diffstat (limited to 'import_ext.py')
-rw-r--r-- | import_ext.py | 422 |
1 files changed, 0 insertions, 422 deletions
diff --git a/import_ext.py b/import_ext.py deleted file mode 100644 index 2a1ebf88..00000000 --- a/import_ext.py +++ /dev/null | |||
@@ -1,422 +0,0 @@ | |||
1 | # | ||
2 | # Copyright (C) 2008 The Android Open Source Project | ||
3 | # | ||
4 | # Licensed under the Apache License, Version 2.0 (the "License"); | ||
5 | # you may not use this file except in compliance with the License. | ||
6 | # You may obtain a copy of the License at | ||
7 | # | ||
8 | # http://www.apache.org/licenses/LICENSE-2.0 | ||
9 | # | ||
10 | # Unless required by applicable law or agreed to in writing, software | ||
11 | # distributed under the License is distributed on an "AS IS" BASIS, | ||
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
13 | # See the License for the specific language governing permissions and | ||
14 | # limitations under the License. | ||
15 | |||
16 | import os | ||
17 | import random | ||
18 | import stat | ||
19 | import sys | ||
20 | import urllib2 | ||
21 | import StringIO | ||
22 | |||
23 | from error import GitError, ImportError | ||
24 | from git_command import GitCommand | ||
25 | |||
26 | class ImportExternal(object): | ||
27 | """Imports a single revision from a non-git data source. | ||
28 | Suitable for use to import a tar or zip based snapshot. | ||
29 | """ | ||
30 | def __init__(self): | ||
31 | self._marks = 0 | ||
32 | self._files = {} | ||
33 | self._tempref = 'refs/repo-external/import' | ||
34 | |||
35 | self._urls = [] | ||
36 | self._remap = [] | ||
37 | self.parent = None | ||
38 | self._user_name = 'Upstream' | ||
39 | self._user_email = 'upstream-import@none' | ||
40 | self._user_when = 1000000 | ||
41 | |||
42 | self.commit = None | ||
43 | |||
44 | def Clone(self): | ||
45 | r = self.__class__() | ||
46 | |||
47 | r.project = self.project | ||
48 | for u in self._urls: | ||
49 | r._urls.append(u) | ||
50 | for p in self._remap: | ||
51 | r._remap.append(_PathMap(r, p._old, p._new)) | ||
52 | |||
53 | return r | ||
54 | |||
55 | def SetProject(self, project): | ||
56 | self.project = project | ||
57 | |||
58 | def SetVersion(self, version): | ||
59 | self.version = version | ||
60 | |||
61 | def AddUrl(self, url): | ||
62 | self._urls.append(url) | ||
63 | |||
64 | def SetParent(self, commit_hash): | ||
65 | self.parent = commit_hash | ||
66 | |||
67 | def SetCommit(self, commit_hash): | ||
68 | self.commit = commit_hash | ||
69 | |||
70 | def RemapPath(self, old, new, replace_version=True): | ||
71 | self._remap.append(_PathMap(self, old, new)) | ||
72 | |||
73 | @property | ||
74 | def TagName(self): | ||
75 | v = '' | ||
76 | for c in self.version: | ||
77 | if c >= '0' and c <= '9': | ||
78 | v += c | ||
79 | elif c >= 'A' and c <= 'Z': | ||
80 | v += c | ||
81 | elif c >= 'a' and c <= 'z': | ||
82 | v += c | ||
83 | elif c in ('-', '_', '.', '/', '+', '@'): | ||
84 | v += c | ||
85 | return 'upstream/%s' % v | ||
86 | |||
87 | @property | ||
88 | def PackageName(self): | ||
89 | n = self.project.name | ||
90 | if n.startswith('platform/'): | ||
91 | # This was not my finest moment... | ||
92 | # | ||
93 | n = n[len('platform/'):] | ||
94 | return n | ||
95 | |||
96 | def Import(self): | ||
97 | self._need_graft = False | ||
98 | if self.parent: | ||
99 | try: | ||
100 | self.project.bare_git.cat_file('-e', self.parent) | ||
101 | except GitError: | ||
102 | self._need_graft = True | ||
103 | |||
104 | gfi = GitCommand(self.project, | ||
105 | ['fast-import', '--force', '--quiet'], | ||
106 | bare = True, | ||
107 | provide_stdin = True) | ||
108 | try: | ||
109 | self._out = gfi.stdin | ||
110 | |||
111 | try: | ||
112 | self._UnpackFiles() | ||
113 | self._MakeCommit() | ||
114 | self._out.flush() | ||
115 | finally: | ||
116 | rc = gfi.Wait() | ||
117 | if rc != 0: | ||
118 | raise ImportError('fast-import failed') | ||
119 | |||
120 | if self._need_graft: | ||
121 | id = self._GraftCommit() | ||
122 | else: | ||
123 | id = self.project.bare_git.rev_parse('%s^0' % self._tempref) | ||
124 | |||
125 | if self.commit and self.commit != id: | ||
126 | raise ImportError('checksum mismatch: %s expected,' | ||
127 | ' %s imported' % (self.commit, id)) | ||
128 | |||
129 | self._MakeTag(id) | ||
130 | return id | ||
131 | finally: | ||
132 | try: | ||
133 | self.project.bare_git.DeleteRef(self._tempref) | ||
134 | except GitError: | ||
135 | pass | ||
136 | |||
137 | def _PickUrl(self, failed): | ||
138 | u = map(lambda x: x.replace('%version%', self.version), self._urls) | ||
139 | for f in failed: | ||
140 | if f in u: | ||
141 | u.remove(f) | ||
142 | if len(u) == 0: | ||
143 | return None | ||
144 | return random.choice(u) | ||
145 | |||
146 | def _OpenUrl(self): | ||
147 | failed = {} | ||
148 | while True: | ||
149 | url = self._PickUrl(failed.keys()) | ||
150 | if url is None: | ||
151 | why = 'Cannot download %s' % self.project.name | ||
152 | |||
153 | if failed: | ||
154 | why += ': one or more mirrors are down\n' | ||
155 | bad_urls = list(failed.keys()) | ||
156 | bad_urls.sort() | ||
157 | for url in bad_urls: | ||
158 | why += ' %s: %s\n' % (url, failed[url]) | ||
159 | else: | ||
160 | why += ': no mirror URLs' | ||
161 | raise ImportError(why) | ||
162 | |||
163 | print >>sys.stderr, "Getting %s ..." % url | ||
164 | try: | ||
165 | return urllib2.urlopen(url), url | ||
166 | except urllib2.HTTPError, e: | ||
167 | failed[url] = e.code | ||
168 | except urllib2.URLError, e: | ||
169 | failed[url] = e.reason[1] | ||
170 | except OSError, e: | ||
171 | failed[url] = e.strerror | ||
172 | |||
173 | def _UnpackFiles(self): | ||
174 | raise NotImplementedError | ||
175 | |||
176 | def _NextMark(self): | ||
177 | self._marks += 1 | ||
178 | return self._marks | ||
179 | |||
180 | def _UnpackOneFile(self, mode, size, name, fd): | ||
181 | if stat.S_ISDIR(mode): # directory | ||
182 | return | ||
183 | else: | ||
184 | mode = self._CleanMode(mode, name) | ||
185 | |||
186 | old_name = name | ||
187 | name = self._CleanName(name) | ||
188 | |||
189 | if stat.S_ISLNK(mode) and self._remap: | ||
190 | # The link is relative to the old_name, and may need to | ||
191 | # be rewritten according to our remap rules if it goes | ||
192 | # up high enough in the tree structure. | ||
193 | # | ||
194 | dest = self._RewriteLink(fd.read(size), old_name, name) | ||
195 | fd = StringIO.StringIO(dest) | ||
196 | size = len(dest) | ||
197 | |||
198 | fi = _File(mode, name, self._NextMark()) | ||
199 | |||
200 | self._out.write('blob\n') | ||
201 | self._out.write('mark :%d\n' % fi.mark) | ||
202 | self._out.write('data %d\n' % size) | ||
203 | while size > 0: | ||
204 | n = min(2048, size) | ||
205 | self._out.write(fd.read(n)) | ||
206 | size -= n | ||
207 | self._out.write('\n') | ||
208 | self._files[fi.name] = fi | ||
209 | |||
210 | def _SetFileMode(self, name, mode): | ||
211 | if not stat.S_ISDIR(mode): | ||
212 | mode = self._CleanMode(mode, name) | ||
213 | name = self._CleanName(name) | ||
214 | try: | ||
215 | fi = self._files[name] | ||
216 | except KeyError: | ||
217 | raise ImportError('file %s was not unpacked' % name) | ||
218 | fi.mode = mode | ||
219 | |||
220 | def _RewriteLink(self, dest, relto_old, relto_new): | ||
221 | # Drop the last components of the symlink itself | ||
222 | # as the dest is relative to the directory its in. | ||
223 | # | ||
224 | relto_old = _TrimPath(relto_old) | ||
225 | relto_new = _TrimPath(relto_new) | ||
226 | |||
227 | # Resolve the link to be absolute from the top of | ||
228 | # the archive, so we can remap its destination. | ||
229 | # | ||
230 | while dest.find('/./') >= 0 or dest.find('//') >= 0: | ||
231 | dest = dest.replace('/./', '/') | ||
232 | dest = dest.replace('//', '/') | ||
233 | |||
234 | if dest.startswith('../') or dest.find('/../') > 0: | ||
235 | dest = _FoldPath('%s/%s' % (relto_old, dest)) | ||
236 | |||
237 | for pm in self._remap: | ||
238 | if pm.Matches(dest): | ||
239 | dest = pm.Apply(dest) | ||
240 | break | ||
241 | |||
242 | dest, relto_new = _StripCommonPrefix(dest, relto_new) | ||
243 | while relto_new: | ||
244 | i = relto_new.find('/') | ||
245 | if i > 0: | ||
246 | relto_new = relto_new[i + 1:] | ||
247 | else: | ||
248 | relto_new = '' | ||
249 | dest = '../' + dest | ||
250 | return dest | ||
251 | |||
252 | def _CleanMode(self, mode, name): | ||
253 | if stat.S_ISREG(mode): # regular file | ||
254 | if (mode & 0111) == 0: | ||
255 | return 0644 | ||
256 | else: | ||
257 | return 0755 | ||
258 | elif stat.S_ISLNK(mode): # symlink | ||
259 | return stat.S_IFLNK | ||
260 | else: | ||
261 | raise ImportError('invalid mode %o in %s' % (mode, name)) | ||
262 | |||
263 | def _CleanName(self, name): | ||
264 | old_name = name | ||
265 | for pm in self._remap: | ||
266 | if pm.Matches(name): | ||
267 | name = pm.Apply(name) | ||
268 | break | ||
269 | while name.startswith('/'): | ||
270 | name = name[1:] | ||
271 | if not name: | ||
272 | raise ImportError('path %s is empty after remap' % old_name) | ||
273 | if name.find('/./') >= 0 or name.find('/../') >= 0: | ||
274 | raise ImportError('path %s contains relative parts' % name) | ||
275 | return name | ||
276 | |||
277 | def _MakeCommit(self): | ||
278 | msg = '%s %s\n' % (self.PackageName, self.version) | ||
279 | |||
280 | self._out.write('commit %s\n' % self._tempref) | ||
281 | self._out.write('committer %s <%s> %d +0000\n' % ( | ||
282 | self._user_name, | ||
283 | self._user_email, | ||
284 | self._user_when)) | ||
285 | self._out.write('data %d\n' % len(msg)) | ||
286 | self._out.write(msg) | ||
287 | self._out.write('\n') | ||
288 | if self.parent and not self._need_graft: | ||
289 | self._out.write('from %s^0\n' % self.parent) | ||
290 | self._out.write('deleteall\n') | ||
291 | |||
292 | for f in self._files.values(): | ||
293 | self._out.write('M %o :%d %s\n' % (f.mode, f.mark, f.name)) | ||
294 | self._out.write('\n') | ||
295 | |||
296 | def _GraftCommit(self): | ||
297 | raw = self.project.bare_git.cat_file('commit', self._tempref) | ||
298 | raw = raw.split("\n") | ||
299 | while raw[1].startswith('parent '): | ||
300 | del raw[1] | ||
301 | raw.insert(1, 'parent %s' % self.parent) | ||
302 | id = self._WriteObject('commit', "\n".join(raw)) | ||
303 | |||
304 | graft_file = os.path.join(self.project.gitdir, 'info/grafts') | ||
305 | if os.path.exists(graft_file): | ||
306 | graft_list = open(graft_file, 'rb').read().split("\n") | ||
307 | if graft_list and graft_list[-1] == '': | ||
308 | del graft_list[-1] | ||
309 | else: | ||
310 | graft_list = [] | ||
311 | |||
312 | exists = False | ||
313 | for line in graft_list: | ||
314 | if line == id: | ||
315 | exists = True | ||
316 | break | ||
317 | |||
318 | if not exists: | ||
319 | graft_list.append(id) | ||
320 | graft_list.append('') | ||
321 | fd = open(graft_file, 'wb') | ||
322 | fd.write("\n".join(graft_list)) | ||
323 | fd.close() | ||
324 | |||
325 | return id | ||
326 | |||
327 | def _MakeTag(self, id): | ||
328 | name = self.TagName | ||
329 | |||
330 | raw = [] | ||
331 | raw.append('object %s' % id) | ||
332 | raw.append('type commit') | ||
333 | raw.append('tag %s' % name) | ||
334 | raw.append('tagger %s <%s> %d +0000' % ( | ||
335 | self._user_name, | ||
336 | self._user_email, | ||
337 | self._user_when)) | ||
338 | raw.append('') | ||
339 | raw.append('%s %s\n' % (self.PackageName, self.version)) | ||
340 | |||
341 | tagid = self._WriteObject('tag', "\n".join(raw)) | ||
342 | self.project.bare_git.UpdateRef('refs/tags/%s' % name, tagid) | ||
343 | |||
344 | def _WriteObject(self, type, data): | ||
345 | wo = GitCommand(self.project, | ||
346 | ['hash-object', '-t', type, '-w', '--stdin'], | ||
347 | bare = True, | ||
348 | provide_stdin = True, | ||
349 | capture_stdout = True, | ||
350 | capture_stderr = True) | ||
351 | wo.stdin.write(data) | ||
352 | if wo.Wait() != 0: | ||
353 | raise GitError('cannot create %s from (%s)' % (type, data)) | ||
354 | return wo.stdout[:-1] | ||
355 | |||
356 | |||
357 | def _TrimPath(path): | ||
358 | i = path.rfind('/') | ||
359 | if i > 0: | ||
360 | path = path[0:i] | ||
361 | return '' | ||
362 | |||
363 | def _StripCommonPrefix(a, b): | ||
364 | while True: | ||
365 | ai = a.find('/') | ||
366 | bi = b.find('/') | ||
367 | if ai > 0 and bi > 0 and a[0:ai] == b[0:bi]: | ||
368 | a = a[ai + 1:] | ||
369 | b = b[bi + 1:] | ||
370 | else: | ||
371 | break | ||
372 | return a, b | ||
373 | |||
374 | def _FoldPath(path): | ||
375 | while True: | ||
376 | if path.startswith('../'): | ||
377 | return path | ||
378 | |||
379 | i = path.find('/../') | ||
380 | if i <= 0: | ||
381 | if path.startswith('/'): | ||
382 | return path[1:] | ||
383 | return path | ||
384 | |||
385 | lhs = path[0:i] | ||
386 | rhs = path[i + 4:] | ||
387 | |||
388 | i = lhs.rfind('/') | ||
389 | if i > 0: | ||
390 | path = lhs[0:i + 1] + rhs | ||
391 | else: | ||
392 | path = rhs | ||
393 | |||
394 | class _File(object): | ||
395 | def __init__(self, mode, name, mark): | ||
396 | self.mode = mode | ||
397 | self.name = name | ||
398 | self.mark = mark | ||
399 | |||
400 | |||
401 | class _PathMap(object): | ||
402 | def __init__(self, imp, old, new): | ||
403 | self._imp = imp | ||
404 | self._old = old | ||
405 | self._new = new | ||
406 | |||
407 | def _r(self, p): | ||
408 | return p.replace('%version%', self._imp.version) | ||
409 | |||
410 | @property | ||
411 | def old(self): | ||
412 | return self._r(self._old) | ||
413 | |||
414 | @property | ||
415 | def new(self): | ||
416 | return self._r(self._new) | ||
417 | |||
418 | def Matches(self, name): | ||
419 | return name.startswith(self.old) | ||
420 | |||
421 | def Apply(self, name): | ||
422 | return self.new + name[len(self.old):] | ||