diff options
Diffstat (limited to 'git_superproject.py')
-rw-r--r-- | git_superproject.py | 880 |
1 files changed, 485 insertions, 395 deletions
diff --git a/git_superproject.py b/git_superproject.py index 69a4d1fe..f1b4f231 100644 --- a/git_superproject.py +++ b/git_superproject.py | |||
@@ -12,7 +12,7 @@ | |||
12 | # See the License for the specific language governing permissions and | 12 | # See the License for the specific language governing permissions and |
13 | # limitations under the License. | 13 | # limitations under the License. |
14 | 14 | ||
15 | """Provide functionality to get all projects and their commit ids from Superproject. | 15 | """Provide functionality to get projects and their commit ids from Superproject. |
16 | 16 | ||
17 | For more information on superproject, check out: | 17 | For more information on superproject, check out: |
18 | https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects | 18 | https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects |
@@ -33,434 +33,524 @@ from git_command import git_require, GitCommand | |||
33 | from git_config import RepoConfig | 33 | from git_config import RepoConfig |
34 | from git_refs import GitRefs | 34 | from git_refs import GitRefs |
35 | 35 | ||
36 | _SUPERPROJECT_GIT_NAME = 'superproject.git' | 36 | _SUPERPROJECT_GIT_NAME = "superproject.git" |
37 | _SUPERPROJECT_MANIFEST_NAME = 'superproject_override.xml' | 37 | _SUPERPROJECT_MANIFEST_NAME = "superproject_override.xml" |
38 | 38 | ||
39 | 39 | ||
40 | class SyncResult(NamedTuple): | 40 | class SyncResult(NamedTuple): |
41 | """Return the status of sync and whether caller should exit.""" | 41 | """Return the status of sync and whether caller should exit.""" |
42 | 42 | ||
43 | # Whether the superproject sync was successful. | 43 | # Whether the superproject sync was successful. |
44 | success: bool | 44 | success: bool |
45 | # Whether the caller should exit. | 45 | # Whether the caller should exit. |
46 | fatal: bool | 46 | fatal: bool |
47 | 47 | ||
48 | 48 | ||
49 | class CommitIdsResult(NamedTuple): | 49 | class CommitIdsResult(NamedTuple): |
50 | """Return the commit ids and whether caller should exit.""" | 50 | """Return the commit ids and whether caller should exit.""" |
51 | 51 | ||
52 | # A dictionary with the projects/commit ids on success, otherwise None. | 52 | # A dictionary with the projects/commit ids on success, otherwise None. |
53 | commit_ids: dict | 53 | commit_ids: dict |
54 | # Whether the caller should exit. | 54 | # Whether the caller should exit. |
55 | fatal: bool | 55 | fatal: bool |
56 | 56 | ||
57 | 57 | ||
58 | class UpdateProjectsResult(NamedTuple): | 58 | class UpdateProjectsResult(NamedTuple): |
59 | """Return the overriding manifest file and whether caller should exit.""" | 59 | """Return the overriding manifest file and whether caller should exit.""" |
60 | 60 | ||
61 | # Path name of the overriding manifest file if successful, otherwise None. | 61 | # Path name of the overriding manifest file if successful, otherwise None. |
62 | manifest_path: str | 62 | manifest_path: str |
63 | # Whether the caller should exit. | 63 | # Whether the caller should exit. |
64 | fatal: bool | 64 | fatal: bool |
65 | 65 | ||
66 | 66 | ||
67 | class Superproject(object): | 67 | class Superproject(object): |
68 | """Get commit ids from superproject. | 68 | """Get commit ids from superproject. |
69 | 69 | ||
70 | Initializes a local copy of a superproject for the manifest. This allows | 70 | Initializes a local copy of a superproject for the manifest. This allows |
71 | lookup of commit ids for all projects. It contains _project_commit_ids which | 71 | lookup of commit ids for all projects. It contains _project_commit_ids which |
72 | is a dictionary with project/commit id entries. | 72 | is a dictionary with project/commit id entries. |
73 | """ | ||
74 | def __init__(self, manifest, name, remote, revision, | ||
75 | superproject_dir='exp-superproject'): | ||
76 | """Initializes superproject. | ||
77 | |||
78 | Args: | ||
79 | manifest: A Manifest object that is to be written to a file. | ||
80 | name: The unique name of the superproject | ||
81 | remote: The RemoteSpec for the remote. | ||
82 | revision: The name of the git branch to track. | ||
83 | superproject_dir: Relative path under |manifest.subdir| to checkout | ||
84 | superproject. | ||
85 | """ | ||
86 | self._project_commit_ids = None | ||
87 | self._manifest = manifest | ||
88 | self.name = name | ||
89 | self.remote = remote | ||
90 | self.revision = self._branch = revision | ||
91 | self._repodir = manifest.repodir | ||
92 | self._superproject_dir = superproject_dir | ||
93 | self._superproject_path = manifest.SubmanifestInfoDir(manifest.path_prefix, | ||
94 | superproject_dir) | ||
95 | self._manifest_path = os.path.join(self._superproject_path, | ||
96 | _SUPERPROJECT_MANIFEST_NAME) | ||
97 | git_name = hashlib.md5(remote.name.encode('utf8')).hexdigest() + '-' | ||
98 | self._remote_url = remote.url | ||
99 | self._work_git_name = git_name + _SUPERPROJECT_GIT_NAME | ||
100 | self._work_git = os.path.join(self._superproject_path, self._work_git_name) | ||
101 | |||
102 | # The following are command arguemnts, rather than superproject attributes, | ||
103 | # and were included here originally. They should eventually become | ||
104 | # arguments that are passed down from the public methods, instead of being | ||
105 | # treated as attributes. | ||
106 | self._git_event_log = None | ||
107 | self._quiet = False | ||
108 | self._print_messages = False | ||
109 | |||
110 | def SetQuiet(self, value): | ||
111 | """Set the _quiet attribute.""" | ||
112 | self._quiet = value | ||
113 | |||
114 | def SetPrintMessages(self, value): | ||
115 | """Set the _print_messages attribute.""" | ||
116 | self._print_messages = value | ||
117 | |||
118 | @property | ||
119 | def project_commit_ids(self): | ||
120 | """Returns a dictionary of projects and their commit ids.""" | ||
121 | return self._project_commit_ids | ||
122 | |||
123 | @property | ||
124 | def manifest_path(self): | ||
125 | """Returns the manifest path if the path exists or None.""" | ||
126 | return self._manifest_path if os.path.exists(self._manifest_path) else None | ||
127 | |||
128 | def _LogMessage(self, fmt, *inputs): | ||
129 | """Logs message to stderr and _git_event_log.""" | ||
130 | message = f'{self._LogMessagePrefix()} {fmt.format(*inputs)}' | ||
131 | if self._print_messages: | ||
132 | print(message, file=sys.stderr) | ||
133 | self._git_event_log.ErrorEvent(message, fmt) | ||
134 | |||
135 | def _LogMessagePrefix(self): | ||
136 | """Returns the prefix string to be logged in each log message""" | ||
137 | return f'repo superproject branch: {self._branch} url: {self._remote_url}' | ||
138 | |||
139 | def _LogError(self, fmt, *inputs): | ||
140 | """Logs error message to stderr and _git_event_log.""" | ||
141 | self._LogMessage(f'error: {fmt}', *inputs) | ||
142 | |||
143 | def _LogWarning(self, fmt, *inputs): | ||
144 | """Logs warning message to stderr and _git_event_log.""" | ||
145 | self._LogMessage(f'warning: {fmt}', *inputs) | ||
146 | |||
147 | def _Init(self): | ||
148 | """Sets up a local Git repository to get a copy of a superproject. | ||
149 | |||
150 | Returns: | ||
151 | True if initialization is successful, or False. | ||
152 | """ | ||
153 | if not os.path.exists(self._superproject_path): | ||
154 | os.mkdir(self._superproject_path) | ||
155 | if not self._quiet and not os.path.exists(self._work_git): | ||
156 | print('%s: Performing initial setup for superproject; this might take ' | ||
157 | 'several minutes.' % self._work_git) | ||
158 | cmd = ['init', '--bare', self._work_git_name] | ||
159 | p = GitCommand(None, | ||
160 | cmd, | ||
161 | cwd=self._superproject_path, | ||
162 | capture_stdout=True, | ||
163 | capture_stderr=True) | ||
164 | retval = p.Wait() | ||
165 | if retval: | ||
166 | self._LogWarning('git init call failed, command: git {}, ' | ||
167 | 'return code: {}, stderr: {}', cmd, retval, p.stderr) | ||
168 | return False | ||
169 | return True | ||
170 | |||
171 | def _Fetch(self): | ||
172 | """Fetches a local copy of a superproject for the manifest based on |_remote_url|. | ||
173 | |||
174 | Returns: | ||
175 | True if fetch is successful, or False. | ||
176 | """ | ||
177 | if not os.path.exists(self._work_git): | ||
178 | self._LogWarning('git fetch missing directory: {}', self._work_git) | ||
179 | return False | ||
180 | if not git_require((2, 28, 0)): | ||
181 | self._LogWarning('superproject requires a git version 2.28 or later') | ||
182 | return False | ||
183 | cmd = ['fetch', self._remote_url, '--depth', '1', '--force', '--no-tags', | ||
184 | '--filter', 'blob:none'] | ||
185 | |||
186 | # Check if there is a local ref that we can pass to --negotiation-tip. | ||
187 | # If this is the first fetch, it does not exist yet. | ||
188 | # We use --negotiation-tip to speed up the fetch. Superproject branches do | ||
189 | # not share commits. So this lets git know it only needs to send commits | ||
190 | # reachable from the specified local refs. | ||
191 | rev_commit = GitRefs(self._work_git).get(f'refs/heads/{self.revision}') | ||
192 | if rev_commit: | ||
193 | cmd.extend(['--negotiation-tip', rev_commit]) | ||
194 | |||
195 | if self._branch: | ||
196 | cmd += [self._branch + ':' + self._branch] | ||
197 | p = GitCommand(None, | ||
198 | cmd, | ||
199 | cwd=self._work_git, | ||
200 | capture_stdout=True, | ||
201 | capture_stderr=True) | ||
202 | retval = p.Wait() | ||
203 | if retval: | ||
204 | self._LogWarning('git fetch call failed, command: git {}, ' | ||
205 | 'return code: {}, stderr: {}', cmd, retval, p.stderr) | ||
206 | return False | ||
207 | return True | ||
208 | |||
209 | def _LsTree(self): | ||
210 | """Gets the commit ids for all projects. | ||
211 | |||
212 | Works only in git repositories. | ||
213 | |||
214 | Returns: | ||
215 | data: data returned from 'git ls-tree ...' instead of None. | ||
216 | """ | ||
217 | if not os.path.exists(self._work_git): | ||
218 | self._LogWarning('git ls-tree missing directory: {}', self._work_git) | ||
219 | return None | ||
220 | data = None | ||
221 | branch = 'HEAD' if not self._branch else self._branch | ||
222 | cmd = ['ls-tree', '-z', '-r', branch] | ||
223 | |||
224 | p = GitCommand(None, | ||
225 | cmd, | ||
226 | cwd=self._work_git, | ||
227 | capture_stdout=True, | ||
228 | capture_stderr=True) | ||
229 | retval = p.Wait() | ||
230 | if retval == 0: | ||
231 | data = p.stdout | ||
232 | else: | ||
233 | self._LogWarning('git ls-tree call failed, command: git {}, ' | ||
234 | 'return code: {}, stderr: {}', cmd, retval, p.stderr) | ||
235 | return data | ||
236 | |||
237 | def Sync(self, git_event_log): | ||
238 | """Gets a local copy of a superproject for the manifest. | ||
239 | |||
240 | Args: | ||
241 | git_event_log: an EventLog, for git tracing. | ||
242 | |||
243 | Returns: | ||
244 | SyncResult | ||
245 | """ | ||
246 | self._git_event_log = git_event_log | ||
247 | if not self._manifest.superproject: | ||
248 | self._LogWarning('superproject tag is not defined in manifest: {}', | ||
249 | self._manifest.manifestFile) | ||
250 | return SyncResult(False, False) | ||
251 | |||
252 | _PrintBetaNotice() | ||
253 | |||
254 | should_exit = True | ||
255 | if not self._remote_url: | ||
256 | self._LogWarning('superproject URL is not defined in manifest: {}', | ||
257 | self._manifest.manifestFile) | ||
258 | return SyncResult(False, should_exit) | ||
259 | |||
260 | if not self._Init(): | ||
261 | return SyncResult(False, should_exit) | ||
262 | if not self._Fetch(): | ||
263 | return SyncResult(False, should_exit) | ||
264 | if not self._quiet: | ||
265 | print('%s: Initial setup for superproject completed.' % self._work_git) | ||
266 | return SyncResult(True, False) | ||
267 | |||
268 | def _GetAllProjectsCommitIds(self): | ||
269 | """Get commit ids for all projects from superproject and save them in _project_commit_ids. | ||
270 | |||
271 | Returns: | ||
272 | CommitIdsResult | ||
273 | """ | ||
274 | sync_result = self.Sync(self._git_event_log) | ||
275 | if not sync_result.success: | ||
276 | return CommitIdsResult(None, sync_result.fatal) | ||
277 | |||
278 | data = self._LsTree() | ||
279 | if not data: | ||
280 | self._LogWarning('git ls-tree failed to return data for manifest: {}', | ||
281 | self._manifest.manifestFile) | ||
282 | return CommitIdsResult(None, True) | ||
283 | |||
284 | # Parse lines like the following to select lines starting with '160000' and | ||
285 | # build a dictionary with project path (last element) and its commit id (3rd element). | ||
286 | # | ||
287 | # 160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00 | ||
288 | # 120000 blob acc2cbdf438f9d2141f0ae424cec1d8fc4b5d97f\tbootstrap.bash\x00 | ||
289 | commit_ids = {} | ||
290 | for line in data.split('\x00'): | ||
291 | ls_data = line.split(None, 3) | ||
292 | if not ls_data: | ||
293 | break | ||
294 | if ls_data[0] == '160000': | ||
295 | commit_ids[ls_data[3]] = ls_data[2] | ||
296 | |||
297 | self._project_commit_ids = commit_ids | ||
298 | return CommitIdsResult(commit_ids, False) | ||
299 | |||
300 | def _WriteManifestFile(self): | ||
301 | """Writes manifest to a file. | ||
302 | |||
303 | Returns: | ||
304 | manifest_path: Path name of the file into which manifest is written instead of None. | ||
305 | """ | ||
306 | if not os.path.exists(self._superproject_path): | ||
307 | self._LogWarning('missing superproject directory: {}', self._superproject_path) | ||
308 | return None | ||
309 | manifest_str = self._manifest.ToXml(groups=self._manifest.GetGroupsStr(), | ||
310 | omit_local=True).toxml() | ||
311 | manifest_path = self._manifest_path | ||
312 | try: | ||
313 | with open(manifest_path, 'w', encoding='utf-8') as fp: | ||
314 | fp.write(manifest_str) | ||
315 | except IOError as e: | ||
316 | self._LogError('cannot write manifest to : {} {}', | ||
317 | manifest_path, e) | ||
318 | return None | ||
319 | return manifest_path | ||
320 | |||
321 | def _SkipUpdatingProjectRevisionId(self, project): | ||
322 | """Checks if a project's revision id needs to be updated or not. | ||
323 | |||
324 | Revision id for projects from local manifest will not be updated. | ||
325 | |||
326 | Args: | ||
327 | project: project whose revision id is being updated. | ||
328 | |||
329 | Returns: | ||
330 | True if a project's revision id should not be updated, or False, | ||
331 | """ | 73 | """ |
332 | path = project.relpath | ||
333 | if not path: | ||
334 | return True | ||
335 | # Skip the project with revisionId. | ||
336 | if project.revisionId: | ||
337 | return True | ||
338 | # Skip the project if it comes from the local manifest. | ||
339 | return project.manifest.IsFromLocalManifest(project) | ||
340 | |||
341 | def UpdateProjectsRevisionId(self, projects, git_event_log): | ||
342 | """Update revisionId of every project in projects with the commit id. | ||
343 | |||
344 | Args: | ||
345 | projects: a list of projects whose revisionId needs to be updated. | ||
346 | git_event_log: an EventLog, for git tracing. | ||
347 | 74 | ||
348 | Returns: | 75 | def __init__( |
349 | UpdateProjectsResult | 76 | self, |
350 | """ | 77 | manifest, |
351 | self._git_event_log = git_event_log | 78 | name, |
352 | commit_ids_result = self._GetAllProjectsCommitIds() | 79 | remote, |
353 | commit_ids = commit_ids_result.commit_ids | 80 | revision, |
354 | if not commit_ids: | 81 | superproject_dir="exp-superproject", |
355 | return UpdateProjectsResult(None, commit_ids_result.fatal) | 82 | ): |
356 | 83 | """Initializes superproject. | |
357 | projects_missing_commit_ids = [] | 84 | |
358 | for project in projects: | 85 | Args: |
359 | if self._SkipUpdatingProjectRevisionId(project): | 86 | manifest: A Manifest object that is to be written to a file. |
360 | continue | 87 | name: The unique name of the superproject |
361 | path = project.relpath | 88 | remote: The RemoteSpec for the remote. |
362 | commit_id = commit_ids.get(path) | 89 | revision: The name of the git branch to track. |
363 | if not commit_id: | 90 | superproject_dir: Relative path under |manifest.subdir| to checkout |
364 | projects_missing_commit_ids.append(path) | 91 | superproject. |
365 | 92 | """ | |
366 | # If superproject doesn't have a commit id for a project, then report an | 93 | self._project_commit_ids = None |
367 | # error event and continue as if do not use superproject is specified. | 94 | self._manifest = manifest |
368 | if projects_missing_commit_ids: | 95 | self.name = name |
369 | self._LogWarning('please file a bug using {} to report missing ' | 96 | self.remote = remote |
370 | 'commit_ids for: {}', self._manifest.contactinfo.bugurl, | 97 | self.revision = self._branch = revision |
371 | projects_missing_commit_ids) | 98 | self._repodir = manifest.repodir |
372 | return UpdateProjectsResult(None, False) | 99 | self._superproject_dir = superproject_dir |
373 | 100 | self._superproject_path = manifest.SubmanifestInfoDir( | |
374 | for project in projects: | 101 | manifest.path_prefix, superproject_dir |
375 | if not self._SkipUpdatingProjectRevisionId(project): | 102 | ) |
376 | project.SetRevisionId(commit_ids.get(project.relpath)) | 103 | self._manifest_path = os.path.join( |
377 | 104 | self._superproject_path, _SUPERPROJECT_MANIFEST_NAME | |
378 | manifest_path = self._WriteManifestFile() | 105 | ) |
379 | return UpdateProjectsResult(manifest_path, False) | 106 | git_name = hashlib.md5(remote.name.encode("utf8")).hexdigest() + "-" |
107 | self._remote_url = remote.url | ||
108 | self._work_git_name = git_name + _SUPERPROJECT_GIT_NAME | ||
109 | self._work_git = os.path.join( | ||
110 | self._superproject_path, self._work_git_name | ||
111 | ) | ||
112 | |||
113 | # The following are command arguemnts, rather than superproject | ||
114 | # attributes, and were included here originally. They should eventually | ||
115 | # become arguments that are passed down from the public methods, instead | ||
116 | # of being treated as attributes. | ||
117 | self._git_event_log = None | ||
118 | self._quiet = False | ||
119 | self._print_messages = False | ||
120 | |||
121 | def SetQuiet(self, value): | ||
122 | """Set the _quiet attribute.""" | ||
123 | self._quiet = value | ||
124 | |||
125 | def SetPrintMessages(self, value): | ||
126 | """Set the _print_messages attribute.""" | ||
127 | self._print_messages = value | ||
128 | |||
129 | @property | ||
130 | def project_commit_ids(self): | ||
131 | """Returns a dictionary of projects and their commit ids.""" | ||
132 | return self._project_commit_ids | ||
133 | |||
134 | @property | ||
135 | def manifest_path(self): | ||
136 | """Returns the manifest path if the path exists or None.""" | ||
137 | return ( | ||
138 | self._manifest_path if os.path.exists(self._manifest_path) else None | ||
139 | ) | ||
140 | |||
141 | def _LogMessage(self, fmt, *inputs): | ||
142 | """Logs message to stderr and _git_event_log.""" | ||
143 | message = f"{self._LogMessagePrefix()} {fmt.format(*inputs)}" | ||
144 | if self._print_messages: | ||
145 | print(message, file=sys.stderr) | ||
146 | self._git_event_log.ErrorEvent(message, fmt) | ||
147 | |||
148 | def _LogMessagePrefix(self): | ||
149 | """Returns the prefix string to be logged in each log message""" | ||
150 | return ( | ||
151 | f"repo superproject branch: {self._branch} url: {self._remote_url}" | ||
152 | ) | ||
153 | |||
154 | def _LogError(self, fmt, *inputs): | ||
155 | """Logs error message to stderr and _git_event_log.""" | ||
156 | self._LogMessage(f"error: {fmt}", *inputs) | ||
157 | |||
158 | def _LogWarning(self, fmt, *inputs): | ||
159 | """Logs warning message to stderr and _git_event_log.""" | ||
160 | self._LogMessage(f"warning: {fmt}", *inputs) | ||
161 | |||
162 | def _Init(self): | ||
163 | """Sets up a local Git repository to get a copy of a superproject. | ||
164 | |||
165 | Returns: | ||
166 | True if initialization is successful, or False. | ||
167 | """ | ||
168 | if not os.path.exists(self._superproject_path): | ||
169 | os.mkdir(self._superproject_path) | ||
170 | if not self._quiet and not os.path.exists(self._work_git): | ||
171 | print( | ||
172 | "%s: Performing initial setup for superproject; this might " | ||
173 | "take several minutes." % self._work_git | ||
174 | ) | ||
175 | cmd = ["init", "--bare", self._work_git_name] | ||
176 | p = GitCommand( | ||
177 | None, | ||
178 | cmd, | ||
179 | cwd=self._superproject_path, | ||
180 | capture_stdout=True, | ||
181 | capture_stderr=True, | ||
182 | ) | ||
183 | retval = p.Wait() | ||
184 | if retval: | ||
185 | self._LogWarning( | ||
186 | "git init call failed, command: git {}, " | ||
187 | "return code: {}, stderr: {}", | ||
188 | cmd, | ||
189 | retval, | ||
190 | p.stderr, | ||
191 | ) | ||
192 | return False | ||
193 | return True | ||
194 | |||
195 | def _Fetch(self): | ||
196 | """Fetches a superproject for the manifest based on |_remote_url|. | ||
197 | |||
198 | This runs git fetch which stores a local copy the superproject. | ||
199 | |||
200 | Returns: | ||
201 | True if fetch is successful, or False. | ||
202 | """ | ||
203 | if not os.path.exists(self._work_git): | ||
204 | self._LogWarning("git fetch missing directory: {}", self._work_git) | ||
205 | return False | ||
206 | if not git_require((2, 28, 0)): | ||
207 | self._LogWarning( | ||
208 | "superproject requires a git version 2.28 or later" | ||
209 | ) | ||
210 | return False | ||
211 | cmd = [ | ||
212 | "fetch", | ||
213 | self._remote_url, | ||
214 | "--depth", | ||
215 | "1", | ||
216 | "--force", | ||
217 | "--no-tags", | ||
218 | "--filter", | ||
219 | "blob:none", | ||
220 | ] | ||
221 | |||
222 | # Check if there is a local ref that we can pass to --negotiation-tip. | ||
223 | # If this is the first fetch, it does not exist yet. | ||
224 | # We use --negotiation-tip to speed up the fetch. Superproject branches | ||
225 | # do not share commits. So this lets git know it only needs to send | ||
226 | # commits reachable from the specified local refs. | ||
227 | rev_commit = GitRefs(self._work_git).get(f"refs/heads/{self.revision}") | ||
228 | if rev_commit: | ||
229 | cmd.extend(["--negotiation-tip", rev_commit]) | ||
230 | |||
231 | if self._branch: | ||
232 | cmd += [self._branch + ":" + self._branch] | ||
233 | p = GitCommand( | ||
234 | None, | ||
235 | cmd, | ||
236 | cwd=self._work_git, | ||
237 | capture_stdout=True, | ||
238 | capture_stderr=True, | ||
239 | ) | ||
240 | retval = p.Wait() | ||
241 | if retval: | ||
242 | self._LogWarning( | ||
243 | "git fetch call failed, command: git {}, " | ||
244 | "return code: {}, stderr: {}", | ||
245 | cmd, | ||
246 | retval, | ||
247 | p.stderr, | ||
248 | ) | ||
249 | return False | ||
250 | return True | ||
251 | |||
252 | def _LsTree(self): | ||
253 | """Gets the commit ids for all projects. | ||
254 | |||
255 | Works only in git repositories. | ||
256 | |||
257 | Returns: | ||
258 | data: data returned from 'git ls-tree ...' instead of None. | ||
259 | """ | ||
260 | if not os.path.exists(self._work_git): | ||
261 | self._LogWarning( | ||
262 | "git ls-tree missing directory: {}", self._work_git | ||
263 | ) | ||
264 | return None | ||
265 | data = None | ||
266 | branch = "HEAD" if not self._branch else self._branch | ||
267 | cmd = ["ls-tree", "-z", "-r", branch] | ||
268 | |||
269 | p = GitCommand( | ||
270 | None, | ||
271 | cmd, | ||
272 | cwd=self._work_git, | ||
273 | capture_stdout=True, | ||
274 | capture_stderr=True, | ||
275 | ) | ||
276 | retval = p.Wait() | ||
277 | if retval == 0: | ||
278 | data = p.stdout | ||
279 | else: | ||
280 | self._LogWarning( | ||
281 | "git ls-tree call failed, command: git {}, " | ||
282 | "return code: {}, stderr: {}", | ||
283 | cmd, | ||
284 | retval, | ||
285 | p.stderr, | ||
286 | ) | ||
287 | return data | ||
288 | |||
289 | def Sync(self, git_event_log): | ||
290 | """Gets a local copy of a superproject for the manifest. | ||
291 | |||
292 | Args: | ||
293 | git_event_log: an EventLog, for git tracing. | ||
294 | |||
295 | Returns: | ||
296 | SyncResult | ||
297 | """ | ||
298 | self._git_event_log = git_event_log | ||
299 | if not self._manifest.superproject: | ||
300 | self._LogWarning( | ||
301 | "superproject tag is not defined in manifest: {}", | ||
302 | self._manifest.manifestFile, | ||
303 | ) | ||
304 | return SyncResult(False, False) | ||
305 | |||
306 | _PrintBetaNotice() | ||
307 | |||
308 | should_exit = True | ||
309 | if not self._remote_url: | ||
310 | self._LogWarning( | ||
311 | "superproject URL is not defined in manifest: {}", | ||
312 | self._manifest.manifestFile, | ||
313 | ) | ||
314 | return SyncResult(False, should_exit) | ||
315 | |||
316 | if not self._Init(): | ||
317 | return SyncResult(False, should_exit) | ||
318 | if not self._Fetch(): | ||
319 | return SyncResult(False, should_exit) | ||
320 | if not self._quiet: | ||
321 | print( | ||
322 | "%s: Initial setup for superproject completed." % self._work_git | ||
323 | ) | ||
324 | return SyncResult(True, False) | ||
325 | |||
326 | def _GetAllProjectsCommitIds(self): | ||
327 | """Get commit ids for all projects from superproject and save them. | ||
328 | |||
329 | Commit ids are saved in _project_commit_ids. | ||
330 | |||
331 | Returns: | ||
332 | CommitIdsResult | ||
333 | """ | ||
334 | sync_result = self.Sync(self._git_event_log) | ||
335 | if not sync_result.success: | ||
336 | return CommitIdsResult(None, sync_result.fatal) | ||
337 | |||
338 | data = self._LsTree() | ||
339 | if not data: | ||
340 | self._LogWarning( | ||
341 | "git ls-tree failed to return data for manifest: {}", | ||
342 | self._manifest.manifestFile, | ||
343 | ) | ||
344 | return CommitIdsResult(None, True) | ||
345 | |||
346 | # Parse lines like the following to select lines starting with '160000' | ||
347 | # and build a dictionary with project path (last element) and its commit | ||
348 | # id (3rd element). | ||
349 | # | ||
350 | # 160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00 | ||
351 | # 120000 blob acc2cbdf438f9d2141f0ae424cec1d8fc4b5d97f\tbootstrap.bash\x00 # noqa: E501 | ||
352 | commit_ids = {} | ||
353 | for line in data.split("\x00"): | ||
354 | ls_data = line.split(None, 3) | ||
355 | if not ls_data: | ||
356 | break | ||
357 | if ls_data[0] == "160000": | ||
358 | commit_ids[ls_data[3]] = ls_data[2] | ||
359 | |||
360 | self._project_commit_ids = commit_ids | ||
361 | return CommitIdsResult(commit_ids, False) | ||
362 | |||
363 | def _WriteManifestFile(self): | ||
364 | """Writes manifest to a file. | ||
365 | |||
366 | Returns: | ||
367 | manifest_path: Path name of the file into which manifest is written | ||
368 | instead of None. | ||
369 | """ | ||
370 | if not os.path.exists(self._superproject_path): | ||
371 | self._LogWarning( | ||
372 | "missing superproject directory: {}", self._superproject_path | ||
373 | ) | ||
374 | return None | ||
375 | manifest_str = self._manifest.ToXml( | ||
376 | groups=self._manifest.GetGroupsStr(), omit_local=True | ||
377 | ).toxml() | ||
378 | manifest_path = self._manifest_path | ||
379 | try: | ||
380 | with open(manifest_path, "w", encoding="utf-8") as fp: | ||
381 | fp.write(manifest_str) | ||
382 | except IOError as e: | ||
383 | self._LogError("cannot write manifest to : {} {}", manifest_path, e) | ||
384 | return None | ||
385 | return manifest_path | ||
386 | |||
387 | def _SkipUpdatingProjectRevisionId(self, project): | ||
388 | """Checks if a project's revision id needs to be updated or not. | ||
389 | |||
390 | Revision id for projects from local manifest will not be updated. | ||
391 | |||
392 | Args: | ||
393 | project: project whose revision id is being updated. | ||
394 | |||
395 | Returns: | ||
396 | True if a project's revision id should not be updated, or False, | ||
397 | """ | ||
398 | path = project.relpath | ||
399 | if not path: | ||
400 | return True | ||
401 | # Skip the project with revisionId. | ||
402 | if project.revisionId: | ||
403 | return True | ||
404 | # Skip the project if it comes from the local manifest. | ||
405 | return project.manifest.IsFromLocalManifest(project) | ||
406 | |||
407 | def UpdateProjectsRevisionId(self, projects, git_event_log): | ||
408 | """Update revisionId of every project in projects with the commit id. | ||
409 | |||
410 | Args: | ||
411 | projects: a list of projects whose revisionId needs to be updated. | ||
412 | git_event_log: an EventLog, for git tracing. | ||
413 | |||
414 | Returns: | ||
415 | UpdateProjectsResult | ||
416 | """ | ||
417 | self._git_event_log = git_event_log | ||
418 | commit_ids_result = self._GetAllProjectsCommitIds() | ||
419 | commit_ids = commit_ids_result.commit_ids | ||
420 | if not commit_ids: | ||
421 | return UpdateProjectsResult(None, commit_ids_result.fatal) | ||
422 | |||
423 | projects_missing_commit_ids = [] | ||
424 | for project in projects: | ||
425 | if self._SkipUpdatingProjectRevisionId(project): | ||
426 | continue | ||
427 | path = project.relpath | ||
428 | commit_id = commit_ids.get(path) | ||
429 | if not commit_id: | ||
430 | projects_missing_commit_ids.append(path) | ||
431 | |||
432 | # If superproject doesn't have a commit id for a project, then report an | ||
433 | # error event and continue as if do not use superproject is specified. | ||
434 | if projects_missing_commit_ids: | ||
435 | self._LogWarning( | ||
436 | "please file a bug using {} to report missing " | ||
437 | "commit_ids for: {}", | ||
438 | self._manifest.contactinfo.bugurl, | ||
439 | projects_missing_commit_ids, | ||
440 | ) | ||
441 | return UpdateProjectsResult(None, False) | ||
442 | |||
443 | for project in projects: | ||
444 | if not self._SkipUpdatingProjectRevisionId(project): | ||
445 | project.SetRevisionId(commit_ids.get(project.relpath)) | ||
446 | |||
447 | manifest_path = self._WriteManifestFile() | ||
448 | return UpdateProjectsResult(manifest_path, False) | ||
380 | 449 | ||
381 | 450 | ||
382 | @functools.lru_cache(maxsize=10) | 451 | @functools.lru_cache(maxsize=10) |
383 | def _PrintBetaNotice(): | 452 | def _PrintBetaNotice(): |
384 | """Print the notice of beta status.""" | 453 | """Print the notice of beta status.""" |
385 | print('NOTICE: --use-superproject is in beta; report any issues to the ' | 454 | print( |
386 | 'address described in `repo version`', file=sys.stderr) | 455 | "NOTICE: --use-superproject is in beta; report any issues to the " |
456 | "address described in `repo version`", | ||
457 | file=sys.stderr, | ||
458 | ) | ||
387 | 459 | ||
388 | 460 | ||
389 | @functools.lru_cache(maxsize=None) | 461 | @functools.lru_cache(maxsize=None) |
390 | def _UseSuperprojectFromConfiguration(): | 462 | def _UseSuperprojectFromConfiguration(): |
391 | """Returns the user choice of whether to use superproject.""" | 463 | """Returns the user choice of whether to use superproject.""" |
392 | user_cfg = RepoConfig.ForUser() | 464 | user_cfg = RepoConfig.ForUser() |
393 | time_now = int(time.time()) | 465 | time_now = int(time.time()) |
394 | 466 | ||
395 | user_value = user_cfg.GetBoolean('repo.superprojectChoice') | 467 | user_value = user_cfg.GetBoolean("repo.superprojectChoice") |
396 | if user_value is not None: | 468 | if user_value is not None: |
397 | user_expiration = user_cfg.GetInt('repo.superprojectChoiceExpire') | 469 | user_expiration = user_cfg.GetInt("repo.superprojectChoiceExpire") |
398 | if user_expiration is None or user_expiration <= 0 or user_expiration >= time_now: | 470 | if ( |
399 | # TODO(b/190688390) - Remove prompt when we are comfortable with the new | 471 | user_expiration is None |
400 | # default value. | 472 | or user_expiration <= 0 |
401 | if user_value: | 473 | or user_expiration >= time_now |
402 | print(('You are currently enrolled in Git submodules experiment ' | 474 | ): |
403 | '(go/android-submodules-quickstart). Use --no-use-superproject ' | 475 | # TODO(b/190688390) - Remove prompt when we are comfortable with the |
404 | 'to override.\n'), file=sys.stderr) | 476 | # new default value. |
405 | else: | 477 | if user_value: |
406 | print(('You are not currently enrolled in Git submodules experiment ' | 478 | print( |
407 | '(go/android-submodules-quickstart). Use --use-superproject ' | 479 | ( |
408 | 'to override.\n'), file=sys.stderr) | 480 | "You are currently enrolled in Git submodules " |
409 | return user_value | 481 | "experiment (go/android-submodules-quickstart). Use " |
410 | 482 | "--no-use-superproject to override.\n" | |
411 | # We don't have an unexpired choice, ask for one. | 483 | ), |
412 | system_cfg = RepoConfig.ForSystem() | 484 | file=sys.stderr, |
413 | system_value = system_cfg.GetBoolean('repo.superprojectChoice') | 485 | ) |
414 | if system_value: | 486 | else: |
415 | # The system configuration is proposing that we should enable the | 487 | print( |
416 | # use of superproject. Treat the user as enrolled for two weeks. | 488 | ( |
417 | # | 489 | "You are not currently enrolled in Git submodules " |
418 | # TODO(b/190688390) - Remove prompt when we are comfortable with the new | 490 | "experiment (go/android-submodules-quickstart). Use " |
419 | # default value. | 491 | "--use-superproject to override.\n" |
420 | userchoice = True | 492 | ), |
421 | time_choiceexpire = time_now + (86400 * 14) | 493 | file=sys.stderr, |
422 | user_cfg.SetString('repo.superprojectChoiceExpire', str(time_choiceexpire)) | 494 | ) |
423 | user_cfg.SetBoolean('repo.superprojectChoice', userchoice) | 495 | return user_value |
424 | print('You are automatically enrolled in Git submodules experiment ' | 496 | |
425 | '(go/android-submodules-quickstart) for another two weeks.\n', | 497 | # We don't have an unexpired choice, ask for one. |
426 | file=sys.stderr) | 498 | system_cfg = RepoConfig.ForSystem() |
427 | return True | 499 | system_value = system_cfg.GetBoolean("repo.superprojectChoice") |
428 | 500 | if system_value: | |
429 | # For all other cases, we would not use superproject by default. | 501 | # The system configuration is proposing that we should enable the |
430 | return False | 502 | # use of superproject. Treat the user as enrolled for two weeks. |
503 | # | ||
504 | # TODO(b/190688390) - Remove prompt when we are comfortable with the new | ||
505 | # default value. | ||
506 | userchoice = True | ||
507 | time_choiceexpire = time_now + (86400 * 14) | ||
508 | user_cfg.SetString( | ||
509 | "repo.superprojectChoiceExpire", str(time_choiceexpire) | ||
510 | ) | ||
511 | user_cfg.SetBoolean("repo.superprojectChoice", userchoice) | ||
512 | print( | ||
513 | "You are automatically enrolled in Git submodules experiment " | ||
514 | "(go/android-submodules-quickstart) for another two weeks.\n", | ||
515 | file=sys.stderr, | ||
516 | ) | ||
517 | return True | ||
518 | |||
519 | # For all other cases, we would not use superproject by default. | ||
520 | return False | ||
431 | 521 | ||
432 | 522 | ||
433 | def PrintMessages(use_superproject, manifest): | 523 | def PrintMessages(use_superproject, manifest): |
434 | """Returns a boolean if error/warning messages are to be printed. | 524 | """Returns a boolean if error/warning messages are to be printed. |
435 | 525 | ||
436 | Args: | 526 | Args: |
437 | use_superproject: option value from optparse. | 527 | use_superproject: option value from optparse. |
438 | manifest: manifest to use. | 528 | manifest: manifest to use. |
439 | """ | 529 | """ |
440 | return use_superproject is not None or bool(manifest.superproject) | 530 | return use_superproject is not None or bool(manifest.superproject) |
441 | 531 | ||
442 | 532 | ||
443 | def UseSuperproject(use_superproject, manifest): | 533 | def UseSuperproject(use_superproject, manifest): |
444 | """Returns a boolean if use-superproject option is enabled. | 534 | """Returns a boolean if use-superproject option is enabled. |
445 | 535 | ||
446 | Args: | 536 | Args: |
447 | use_superproject: option value from optparse. | 537 | use_superproject: option value from optparse. |
448 | manifest: manifest to use. | 538 | manifest: manifest to use. |
449 | 539 | ||
450 | Returns: | 540 | Returns: |
451 | Whether the superproject should be used. | 541 | Whether the superproject should be used. |
452 | """ | 542 | """ |
453 | 543 | ||
454 | if not manifest.superproject: | 544 | if not manifest.superproject: |
455 | # This (sub) manifest does not have a superproject definition. | 545 | # This (sub) manifest does not have a superproject definition. |
456 | return False | 546 | return False |
457 | elif use_superproject is not None: | 547 | elif use_superproject is not None: |
458 | return use_superproject | 548 | return use_superproject |
459 | else: | ||
460 | client_value = manifest.manifestProject.use_superproject | ||
461 | if client_value is not None: | ||
462 | return client_value | ||
463 | elif manifest.superproject: | ||
464 | return _UseSuperprojectFromConfiguration() | ||
465 | else: | 549 | else: |
466 | return False | 550 | client_value = manifest.manifestProject.use_superproject |
551 | if client_value is not None: | ||
552 | return client_value | ||
553 | elif manifest.superproject: | ||
554 | return _UseSuperprojectFromConfiguration() | ||
555 | else: | ||
556 | return False | ||