diff options
Diffstat (limited to 'subcmds/sync.py')
-rw-r--r-- | subcmds/sync.py | 3130 |
1 files changed, 1730 insertions, 1400 deletions
diff --git a/subcmds/sync.py b/subcmds/sync.py index 9a8ca8f7..eabaa68b 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py | |||
@@ -33,18 +33,21 @@ import xml.parsers.expat | |||
33 | import xmlrpc.client | 33 | import xmlrpc.client |
34 | 34 | ||
35 | try: | 35 | try: |
36 | import threading as _threading | 36 | import threading as _threading |
37 | except ImportError: | 37 | except ImportError: |
38 | import dummy_threading as _threading | 38 | import dummy_threading as _threading |
39 | 39 | ||
40 | try: | 40 | try: |
41 | import resource | 41 | import resource |
42 | |||
43 | def _rlimit_nofile(): | ||
44 | return resource.getrlimit(resource.RLIMIT_NOFILE) | ||
42 | 45 | ||
43 | def _rlimit_nofile(): | ||
44 | return resource.getrlimit(resource.RLIMIT_NOFILE) | ||
45 | except ImportError: | 46 | except ImportError: |
46 | def _rlimit_nofile(): | 47 | |
47 | return (256, 256) | 48 | def _rlimit_nofile(): |
49 | return (256, 256) | ||
50 | |||
48 | 51 | ||
49 | import event_log | 52 | import event_log |
50 | from git_command import git_require | 53 | from git_command import git_require |
@@ -54,7 +57,12 @@ import git_superproject | |||
54 | import gitc_utils | 57 | import gitc_utils |
55 | from project import Project | 58 | from project import Project |
56 | from project import RemoteSpec | 59 | from project import RemoteSpec |
57 | from command import Command, DEFAULT_LOCAL_JOBS, MirrorSafeCommand, WORKER_BATCH_SIZE | 60 | from command import ( |
61 | Command, | ||
62 | DEFAULT_LOCAL_JOBS, | ||
63 | MirrorSafeCommand, | ||
64 | WORKER_BATCH_SIZE, | ||
65 | ) | ||
58 | from error import RepoChangedException, GitError | 66 | from error import RepoChangedException, GitError |
59 | import platform_utils | 67 | import platform_utils |
60 | from project import SyncBuffer | 68 | from project import SyncBuffer |
@@ -68,70 +76,74 @@ _ONE_DAY_S = 24 * 60 * 60 | |||
68 | 76 | ||
69 | # Env var to implicitly turn auto-gc back on. This was added to allow a user to | 77 | # Env var to implicitly turn auto-gc back on. This was added to allow a user to |
70 | # revert a change in default behavior in v2.29.9. Remove after 2023-04-01. | 78 | # revert a change in default behavior in v2.29.9. Remove after 2023-04-01. |
71 | _REPO_AUTO_GC = 'REPO_AUTO_GC' | 79 | _REPO_AUTO_GC = "REPO_AUTO_GC" |
72 | _AUTO_GC = os.environ.get(_REPO_AUTO_GC) == '1' | 80 | _AUTO_GC = os.environ.get(_REPO_AUTO_GC) == "1" |
73 | 81 | ||
74 | 82 | ||
75 | class _FetchOneResult(NamedTuple): | 83 | class _FetchOneResult(NamedTuple): |
76 | """_FetchOne return value. | 84 | """_FetchOne return value. |
77 | 85 | ||
78 | Attributes: | 86 | Attributes: |
79 | success (bool): True if successful. | 87 | success (bool): True if successful. |
80 | project (Project): The fetched project. | 88 | project (Project): The fetched project. |
81 | start (float): The starting time.time(). | 89 | start (float): The starting time.time(). |
82 | finish (float): The ending time.time(). | 90 | finish (float): The ending time.time(). |
83 | remote_fetched (bool): True if the remote was actually queried. | 91 | remote_fetched (bool): True if the remote was actually queried. |
84 | """ | 92 | """ |
85 | success: bool | 93 | |
86 | project: Project | 94 | success: bool |
87 | start: float | 95 | project: Project |
88 | finish: float | 96 | start: float |
89 | remote_fetched: bool | 97 | finish: float |
98 | remote_fetched: bool | ||
90 | 99 | ||
91 | 100 | ||
92 | class _FetchResult(NamedTuple): | 101 | class _FetchResult(NamedTuple): |
93 | """_Fetch return value. | 102 | """_Fetch return value. |
103 | |||
104 | Attributes: | ||
105 | success (bool): True if successful. | ||
106 | projects (Set[str]): The names of the git directories of fetched projects. | ||
107 | """ | ||
94 | 108 | ||
95 | Attributes: | 109 | success: bool |
96 | success (bool): True if successful. | 110 | projects: Set[str] |
97 | projects (Set[str]): The names of the git directories of fetched projects. | ||
98 | """ | ||
99 | success: bool | ||
100 | projects: Set[str] | ||
101 | 111 | ||
102 | 112 | ||
103 | class _FetchMainResult(NamedTuple): | 113 | class _FetchMainResult(NamedTuple): |
104 | """_FetchMain return value. | 114 | """_FetchMain return value. |
115 | |||
116 | Attributes: | ||
117 | all_projects (List[Project]): The fetched projects. | ||
118 | """ | ||
105 | 119 | ||
106 | Attributes: | 120 | all_projects: List[Project] |
107 | all_projects (List[Project]): The fetched projects. | ||
108 | """ | ||
109 | all_projects: List[Project] | ||
110 | 121 | ||
111 | 122 | ||
112 | class _CheckoutOneResult(NamedTuple): | 123 | class _CheckoutOneResult(NamedTuple): |
113 | """_CheckoutOne return value. | 124 | """_CheckoutOne return value. |
125 | |||
126 | Attributes: | ||
127 | success (bool): True if successful. | ||
128 | project (Project): The project. | ||
129 | start (float): The starting time.time(). | ||
130 | finish (float): The ending time.time(). | ||
131 | """ | ||
114 | 132 | ||
115 | Attributes: | 133 | success: bool |
116 | success (bool): True if successful. | 134 | project: Project |
117 | project (Project): The project. | 135 | start: float |
118 | start (float): The starting time.time(). | 136 | finish: float |
119 | finish (float): The ending time.time(). | ||
120 | """ | ||
121 | success: bool | ||
122 | project: Project | ||
123 | start: float | ||
124 | finish: float | ||
125 | 137 | ||
126 | 138 | ||
127 | class Sync(Command, MirrorSafeCommand): | 139 | class Sync(Command, MirrorSafeCommand): |
128 | COMMON = True | 140 | COMMON = True |
129 | MULTI_MANIFEST_SUPPORT = True | 141 | MULTI_MANIFEST_SUPPORT = True |
130 | helpSummary = "Update working tree to the latest revision" | 142 | helpSummary = "Update working tree to the latest revision" |
131 | helpUsage = """ | 143 | helpUsage = """ |
132 | %prog [<project>...] | 144 | %prog [<project>...] |
133 | """ | 145 | """ |
134 | helpDescription = """ | 146 | helpDescription = """ |
135 | The '%prog' command synchronizes local project directories | 147 | The '%prog' command synchronizes local project directories |
136 | with the remote repositories specified in the manifest. If a local | 148 | with the remote repositories specified in the manifest. If a local |
137 | project does not yet exist, it will clone a new local directory from | 149 | project does not yet exist, it will clone a new local directory from |
@@ -230,1293 +242,1604 @@ If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or | |||
230 | later is required to fix a server side protocol bug. | 242 | later is required to fix a server side protocol bug. |
231 | 243 | ||
232 | """ | 244 | """ |
233 | # A value of 0 means we want parallel jobs, but we'll determine the default | 245 | # A value of 0 means we want parallel jobs, but we'll determine the default |
234 | # value later on. | 246 | # value later on. |
235 | PARALLEL_JOBS = 0 | 247 | PARALLEL_JOBS = 0 |
236 | 248 | ||
237 | def _Options(self, p, show_smart=True): | 249 | def _Options(self, p, show_smart=True): |
238 | p.add_option('--jobs-network', default=None, type=int, metavar='JOBS', | 250 | p.add_option( |
239 | help='number of network jobs to run in parallel (defaults to --jobs or 1)') | 251 | "--jobs-network", |
240 | p.add_option('--jobs-checkout', default=None, type=int, metavar='JOBS', | 252 | default=None, |
241 | help='number of local checkout jobs to run in parallel (defaults to --jobs or ' | 253 | type=int, |
242 | f'{DEFAULT_LOCAL_JOBS})') | 254 | metavar="JOBS", |
243 | 255 | help="number of network jobs to run in parallel (defaults to " | |
244 | p.add_option('-f', '--force-broken', | 256 | "--jobs or 1)", |
245 | dest='force_broken', action='store_true', | 257 | ) |
246 | help='obsolete option (to be deleted in the future)') | 258 | p.add_option( |
247 | p.add_option('--fail-fast', | 259 | "--jobs-checkout", |
248 | dest='fail_fast', action='store_true', | 260 | default=None, |
249 | help='stop syncing after first error is hit') | 261 | type=int, |
250 | p.add_option('--force-sync', | 262 | metavar="JOBS", |
251 | dest='force_sync', action='store_true', | 263 | help="number of local checkout jobs to run in parallel (defaults " |
252 | help="overwrite an existing git directory if it needs to " | 264 | f"to --jobs or {DEFAULT_LOCAL_JOBS})", |
253 | "point to a different object directory. WARNING: this " | 265 | ) |
254 | "may cause loss of data") | 266 | |
255 | p.add_option('--force-remove-dirty', | 267 | p.add_option( |
256 | dest='force_remove_dirty', action='store_true', | 268 | "-f", |
257 | help="force remove projects with uncommitted modifications if " | 269 | "--force-broken", |
258 | "projects no longer exist in the manifest. " | 270 | dest="force_broken", |
259 | "WARNING: this may cause loss of data") | 271 | action="store_true", |
260 | p.add_option('-l', '--local-only', | 272 | help="obsolete option (to be deleted in the future)", |
261 | dest='local_only', action='store_true', | 273 | ) |
262 | help="only update working tree, don't fetch") | 274 | p.add_option( |
263 | p.add_option('--no-manifest-update', '--nmu', | 275 | "--fail-fast", |
264 | dest='mp_update', action='store_false', default='true', | 276 | dest="fail_fast", |
265 | help='use the existing manifest checkout as-is. ' | 277 | action="store_true", |
266 | '(do not update to the latest revision)') | 278 | help="stop syncing after first error is hit", |
267 | p.add_option('-n', '--network-only', | 279 | ) |
268 | dest='network_only', action='store_true', | 280 | p.add_option( |
269 | help="fetch only, don't update working tree") | 281 | "--force-sync", |
270 | p.add_option('-d', '--detach', | 282 | dest="force_sync", |
271 | dest='detach_head', action='store_true', | 283 | action="store_true", |
272 | help='detach projects back to manifest revision') | 284 | help="overwrite an existing git directory if it needs to " |
273 | p.add_option('-c', '--current-branch', | 285 | "point to a different object directory. WARNING: this " |
274 | dest='current_branch_only', action='store_true', | 286 | "may cause loss of data", |
275 | help='fetch only current branch from server') | 287 | ) |
276 | p.add_option('--no-current-branch', | 288 | p.add_option( |
277 | dest='current_branch_only', action='store_false', | 289 | "--force-remove-dirty", |
278 | help='fetch all branches from server') | 290 | dest="force_remove_dirty", |
279 | p.add_option('-m', '--manifest-name', | 291 | action="store_true", |
280 | dest='manifest_name', | 292 | help="force remove projects with uncommitted modifications if " |
281 | help='temporary manifest to use for this sync', metavar='NAME.xml') | 293 | "projects no longer exist in the manifest. " |
282 | p.add_option('--clone-bundle', action='store_true', | 294 | "WARNING: this may cause loss of data", |
283 | help='enable use of /clone.bundle on HTTP/HTTPS') | 295 | ) |
284 | p.add_option('--no-clone-bundle', dest='clone_bundle', action='store_false', | 296 | p.add_option( |
285 | help='disable use of /clone.bundle on HTTP/HTTPS') | 297 | "-l", |
286 | p.add_option('-u', '--manifest-server-username', action='store', | 298 | "--local-only", |
287 | dest='manifest_server_username', | 299 | dest="local_only", |
288 | help='username to authenticate with the manifest server') | 300 | action="store_true", |
289 | p.add_option('-p', '--manifest-server-password', action='store', | 301 | help="only update working tree, don't fetch", |
290 | dest='manifest_server_password', | 302 | ) |
291 | help='password to authenticate with the manifest server') | 303 | p.add_option( |
292 | p.add_option('--fetch-submodules', | 304 | "--no-manifest-update", |
293 | dest='fetch_submodules', action='store_true', | 305 | "--nmu", |
294 | help='fetch submodules from server') | 306 | dest="mp_update", |
295 | p.add_option('--use-superproject', action='store_true', | 307 | action="store_false", |
296 | help='use the manifest superproject to sync projects; implies -c') | 308 | default="true", |
297 | p.add_option('--no-use-superproject', action='store_false', | 309 | help="use the existing manifest checkout as-is. " |
298 | dest='use_superproject', | 310 | "(do not update to the latest revision)", |
299 | help='disable use of manifest superprojects') | 311 | ) |
300 | p.add_option('--tags', action='store_true', | 312 | p.add_option( |
301 | help='fetch tags') | 313 | "-n", |
302 | p.add_option('--no-tags', | 314 | "--network-only", |
303 | dest='tags', action='store_false', | 315 | dest="network_only", |
304 | help="don't fetch tags (default)") | 316 | action="store_true", |
305 | p.add_option('--optimized-fetch', | 317 | help="fetch only, don't update working tree", |
306 | dest='optimized_fetch', action='store_true', | 318 | ) |
307 | help='only fetch projects fixed to sha1 if revision does not exist locally') | 319 | p.add_option( |
308 | p.add_option('--retry-fetches', | 320 | "-d", |
309 | default=0, action='store', type='int', | 321 | "--detach", |
310 | help='number of times to retry fetches on transient errors') | 322 | dest="detach_head", |
311 | p.add_option('--prune', action='store_true', | 323 | action="store_true", |
312 | help='delete refs that no longer exist on the remote (default)') | 324 | help="detach projects back to manifest revision", |
313 | p.add_option('--no-prune', dest='prune', action='store_false', | 325 | ) |
314 | help='do not delete refs that no longer exist on the remote') | 326 | p.add_option( |
315 | p.add_option('--auto-gc', action='store_true', default=None, | 327 | "-c", |
316 | help='run garbage collection on all synced projects') | 328 | "--current-branch", |
317 | p.add_option('--no-auto-gc', dest='auto_gc', action='store_false', | 329 | dest="current_branch_only", |
318 | help='do not run garbage collection on any projects (default)') | 330 | action="store_true", |
319 | if show_smart: | 331 | help="fetch only current branch from server", |
320 | p.add_option('-s', '--smart-sync', | 332 | ) |
321 | dest='smart_sync', action='store_true', | 333 | p.add_option( |
322 | help='smart sync using manifest from the latest known good build') | 334 | "--no-current-branch", |
323 | p.add_option('-t', '--smart-tag', | 335 | dest="current_branch_only", |
324 | dest='smart_tag', action='store', | 336 | action="store_false", |
325 | help='smart sync using manifest from a known tag') | 337 | help="fetch all branches from server", |
326 | 338 | ) | |
327 | g = p.add_option_group('repo Version options') | 339 | p.add_option( |
328 | g.add_option('--no-repo-verify', | 340 | "-m", |
329 | dest='repo_verify', default=True, action='store_false', | 341 | "--manifest-name", |
330 | help='do not verify repo source code') | 342 | dest="manifest_name", |
331 | g.add_option('--repo-upgraded', | 343 | help="temporary manifest to use for this sync", |
332 | dest='repo_upgraded', action='store_true', | 344 | metavar="NAME.xml", |
333 | help=SUPPRESS_HELP) | 345 | ) |
334 | 346 | p.add_option( | |
335 | def _GetBranch(self, manifest_project): | 347 | "--clone-bundle", |
336 | """Returns the branch name for getting the approved smartsync manifest. | 348 | action="store_true", |
337 | 349 | help="enable use of /clone.bundle on HTTP/HTTPS", | |
338 | Args: | 350 | ) |
339 | manifest_project: the manifestProject to query. | 351 | p.add_option( |
340 | """ | 352 | "--no-clone-bundle", |
341 | b = manifest_project.GetBranch(manifest_project.CurrentBranch) | 353 | dest="clone_bundle", |
342 | branch = b.merge | 354 | action="store_false", |
343 | if branch.startswith(R_HEADS): | 355 | help="disable use of /clone.bundle on HTTP/HTTPS", |
344 | branch = branch[len(R_HEADS):] | 356 | ) |
345 | return branch | 357 | p.add_option( |
346 | 358 | "-u", | |
347 | def _GetCurrentBranchOnly(self, opt, manifest): | 359 | "--manifest-server-username", |
348 | """Returns whether current-branch or use-superproject options are enabled. | 360 | action="store", |
349 | 361 | dest="manifest_server_username", | |
350 | Args: | 362 | help="username to authenticate with the manifest server", |
351 | opt: Program options returned from optparse. See _Options(). | 363 | ) |
352 | manifest: The manifest to use. | 364 | p.add_option( |
353 | 365 | "-p", | |
354 | Returns: | 366 | "--manifest-server-password", |
355 | True if a superproject is requested, otherwise the value of the | 367 | action="store", |
356 | current_branch option (True, False or None). | 368 | dest="manifest_server_password", |
357 | """ | 369 | help="password to authenticate with the manifest server", |
358 | return git_superproject.UseSuperproject(opt.use_superproject, manifest) or opt.current_branch_only | 370 | ) |
359 | 371 | p.add_option( | |
360 | def _UpdateProjectsRevisionId(self, opt, args, superproject_logging_data, | 372 | "--fetch-submodules", |
361 | manifest): | 373 | dest="fetch_submodules", |
362 | """Update revisionId of projects with the commit hash from the superproject. | 374 | action="store_true", |
363 | 375 | help="fetch submodules from server", | |
364 | This function updates each project's revisionId with the commit hash from | 376 | ) |
365 | the superproject. It writes the updated manifest into a file and reloads | 377 | p.add_option( |
366 | the manifest from it. When appropriate, sub manifests are also processed. | 378 | "--use-superproject", |
367 | 379 | action="store_true", | |
368 | Args: | 380 | help="use the manifest superproject to sync projects; implies -c", |
369 | opt: Program options returned from optparse. See _Options(). | 381 | ) |
370 | args: Arguments to pass to GetProjects. See the GetProjects | 382 | p.add_option( |
371 | docstring for details. | 383 | "--no-use-superproject", |
372 | superproject_logging_data: A dictionary of superproject data to log. | 384 | action="store_false", |
373 | manifest: The manifest to use. | 385 | dest="use_superproject", |
374 | """ | 386 | help="disable use of manifest superprojects", |
375 | have_superproject = manifest.superproject or any( | 387 | ) |
376 | m.superproject for m in manifest.all_children) | 388 | p.add_option("--tags", action="store_true", help="fetch tags") |
377 | if not have_superproject: | 389 | p.add_option( |
378 | return | 390 | "--no-tags", |
379 | 391 | dest="tags", | |
380 | if opt.local_only and manifest.superproject: | 392 | action="store_false", |
381 | manifest_path = manifest.superproject.manifest_path | 393 | help="don't fetch tags (default)", |
382 | if manifest_path: | 394 | ) |
383 | self._ReloadManifest(manifest_path, manifest) | 395 | p.add_option( |
384 | return | 396 | "--optimized-fetch", |
385 | 397 | dest="optimized_fetch", | |
386 | all_projects = self.GetProjects(args, | 398 | action="store_true", |
387 | missing_ok=True, | 399 | help="only fetch projects fixed to sha1 if revision does not exist " |
388 | submodules_ok=opt.fetch_submodules, | 400 | "locally", |
389 | manifest=manifest, | 401 | ) |
390 | all_manifests=not opt.this_manifest_only) | 402 | p.add_option( |
391 | 403 | "--retry-fetches", | |
392 | per_manifest = collections.defaultdict(list) | 404 | default=0, |
393 | manifest_paths = {} | 405 | action="store", |
394 | if opt.this_manifest_only: | 406 | type="int", |
395 | per_manifest[manifest.path_prefix] = all_projects | 407 | help="number of times to retry fetches on transient errors", |
396 | else: | 408 | ) |
397 | for p in all_projects: | 409 | p.add_option( |
398 | per_manifest[p.manifest.path_prefix].append(p) | 410 | "--prune", |
399 | 411 | action="store_true", | |
400 | superproject_logging_data = {} | 412 | help="delete refs that no longer exist on the remote (default)", |
401 | need_unload = False | 413 | ) |
402 | for m in self.ManifestList(opt): | 414 | p.add_option( |
403 | if not m.path_prefix in per_manifest: | 415 | "--no-prune", |
404 | continue | 416 | dest="prune", |
405 | use_super = git_superproject.UseSuperproject(opt.use_superproject, m) | 417 | action="store_false", |
406 | if superproject_logging_data: | 418 | help="do not delete refs that no longer exist on the remote", |
407 | superproject_logging_data['multimanifest'] = True | 419 | ) |
408 | superproject_logging_data.update( | 420 | p.add_option( |
409 | superproject=use_super, | 421 | "--auto-gc", |
410 | haslocalmanifests=bool(m.HasLocalManifests), | 422 | action="store_true", |
411 | hassuperprojecttag=bool(m.superproject), | 423 | default=None, |
412 | ) | 424 | help="run garbage collection on all synced projects", |
413 | if use_super and (m.IsMirror or m.IsArchive): | 425 | ) |
414 | # Don't use superproject, because we have no working tree. | 426 | p.add_option( |
415 | use_super = False | 427 | "--no-auto-gc", |
416 | superproject_logging_data['superproject'] = False | 428 | dest="auto_gc", |
417 | superproject_logging_data['noworktree'] = True | 429 | action="store_false", |
418 | if opt.use_superproject is not False: | 430 | help="do not run garbage collection on any projects (default)", |
419 | print(f'{m.path_prefix}: not using superproject because there is no ' | 431 | ) |
420 | 'working tree.') | 432 | if show_smart: |
421 | 433 | p.add_option( | |
422 | if not use_super: | 434 | "-s", |
423 | continue | 435 | "--smart-sync", |
424 | m.superproject.SetQuiet(opt.quiet) | 436 | dest="smart_sync", |
425 | print_messages = git_superproject.PrintMessages(opt.use_superproject, m) | 437 | action="store_true", |
426 | m.superproject.SetPrintMessages(print_messages) | 438 | help="smart sync using manifest from the latest known good " |
427 | update_result = m.superproject.UpdateProjectsRevisionId( | 439 | "build", |
428 | per_manifest[m.path_prefix], git_event_log=self.git_event_log) | 440 | ) |
429 | manifest_path = update_result.manifest_path | 441 | p.add_option( |
430 | superproject_logging_data['updatedrevisionid'] = bool(manifest_path) | 442 | "-t", |
431 | if manifest_path: | 443 | "--smart-tag", |
432 | m.SetManifestOverride(manifest_path) | 444 | dest="smart_tag", |
433 | need_unload = True | 445 | action="store", |
434 | else: | 446 | help="smart sync using manifest from a known tag", |
435 | if print_messages: | 447 | ) |
436 | print(f'{m.path_prefix}: warning: Update of revisionId from ' | 448 | |
437 | 'superproject has failed, repo sync will not use superproject ' | 449 | g = p.add_option_group("repo Version options") |
438 | 'to fetch the source. ', | 450 | g.add_option( |
439 | 'Please resync with the --no-use-superproject option to avoid ' | 451 | "--no-repo-verify", |
440 | 'this repo warning.', | 452 | dest="repo_verify", |
441 | file=sys.stderr) | 453 | default=True, |
442 | if update_result.fatal and opt.use_superproject is not None: | 454 | action="store_false", |
443 | sys.exit(1) | 455 | help="do not verify repo source code", |
444 | if need_unload: | 456 | ) |
445 | m.outer_client.manifest.Unload() | 457 | g.add_option( |
446 | 458 | "--repo-upgraded", | |
447 | def _FetchProjectList(self, opt, projects): | 459 | dest="repo_upgraded", |
448 | """Main function of the fetch worker. | 460 | action="store_true", |
449 | 461 | help=SUPPRESS_HELP, | |
450 | The projects we're given share the same underlying git object store, so we | 462 | ) |
451 | have to fetch them in serial. | ||
452 | |||
453 | Delegates most of the work to _FetchHelper. | ||
454 | |||
455 | Args: | ||
456 | opt: Program options returned from optparse. See _Options(). | ||
457 | projects: Projects to fetch. | ||
458 | """ | ||
459 | return [self._FetchOne(opt, x) for x in projects] | ||
460 | 463 | ||
461 | def _FetchOne(self, opt, project): | 464 | def _GetBranch(self, manifest_project): |
462 | """Fetch git objects for a single project. | 465 | """Returns the branch name for getting the approved smartsync manifest. |
466 | |||
467 | Args: | ||
468 | manifest_project: The manifestProject to query. | ||
469 | """ | ||
470 | b = manifest_project.GetBranch(manifest_project.CurrentBranch) | ||
471 | branch = b.merge | ||
472 | if branch.startswith(R_HEADS): | ||
473 | branch = branch[len(R_HEADS) :] | ||
474 | return branch | ||
475 | |||
476 | def _GetCurrentBranchOnly(self, opt, manifest): | ||
477 | """Returns whether current-branch or use-superproject options are | ||
478 | enabled. | ||
479 | |||
480 | Args: | ||
481 | opt: Program options returned from optparse. See _Options(). | ||
482 | manifest: The manifest to use. | ||
483 | |||
484 | Returns: | ||
485 | True if a superproject is requested, otherwise the value of the | ||
486 | current_branch option (True, False or None). | ||
487 | """ | ||
488 | return ( | ||
489 | git_superproject.UseSuperproject(opt.use_superproject, manifest) | ||
490 | or opt.current_branch_only | ||
491 | ) | ||
463 | 492 | ||
464 | Args: | 493 | def _UpdateProjectsRevisionId( |
465 | opt: Program options returned from optparse. See _Options(). | 494 | self, opt, args, superproject_logging_data, manifest |
466 | project: Project object for the project to fetch. | 495 | ): |
496 | """Update revisionId of projects with the commit from the superproject. | ||
497 | |||
498 | This function updates each project's revisionId with the commit hash | ||
499 | from the superproject. It writes the updated manifest into a file and | ||
500 | reloads the manifest from it. When appropriate, sub manifests are also | ||
501 | processed. | ||
502 | |||
503 | Args: | ||
504 | opt: Program options returned from optparse. See _Options(). | ||
505 | args: Arguments to pass to GetProjects. See the GetProjects | ||
506 | docstring for details. | ||
507 | superproject_logging_data: A dictionary of superproject data to log. | ||
508 | manifest: The manifest to use. | ||
509 | """ | ||
510 | have_superproject = manifest.superproject or any( | ||
511 | m.superproject for m in manifest.all_children | ||
512 | ) | ||
513 | if not have_superproject: | ||
514 | return | ||
515 | |||
516 | if opt.local_only and manifest.superproject: | ||
517 | manifest_path = manifest.superproject.manifest_path | ||
518 | if manifest_path: | ||
519 | self._ReloadManifest(manifest_path, manifest) | ||
520 | return | ||
521 | |||
522 | all_projects = self.GetProjects( | ||
523 | args, | ||
524 | missing_ok=True, | ||
525 | submodules_ok=opt.fetch_submodules, | ||
526 | manifest=manifest, | ||
527 | all_manifests=not opt.this_manifest_only, | ||
528 | ) | ||
467 | 529 | ||
468 | Returns: | 530 | per_manifest = collections.defaultdict(list) |
469 | Whether the fetch was successful. | 531 | if opt.this_manifest_only: |
470 | """ | 532 | per_manifest[manifest.path_prefix] = all_projects |
471 | start = time.time() | 533 | else: |
472 | success = False | 534 | for p in all_projects: |
473 | remote_fetched = False | 535 | per_manifest[p.manifest.path_prefix].append(p) |
474 | buf = io.StringIO() | 536 | |
475 | try: | 537 | superproject_logging_data = {} |
476 | sync_result = project.Sync_NetworkHalf( | 538 | need_unload = False |
477 | quiet=opt.quiet, | 539 | for m in self.ManifestList(opt): |
478 | verbose=opt.verbose, | 540 | if m.path_prefix not in per_manifest: |
479 | output_redir=buf, | 541 | continue |
480 | current_branch_only=self._GetCurrentBranchOnly(opt, project.manifest), | 542 | use_super = git_superproject.UseSuperproject( |
481 | force_sync=opt.force_sync, | 543 | opt.use_superproject, m |
482 | clone_bundle=opt.clone_bundle, | 544 | ) |
483 | tags=opt.tags, archive=project.manifest.IsArchive, | 545 | if superproject_logging_data: |
484 | optimized_fetch=opt.optimized_fetch, | 546 | superproject_logging_data["multimanifest"] = True |
485 | retry_fetches=opt.retry_fetches, | 547 | superproject_logging_data.update( |
486 | prune=opt.prune, | 548 | superproject=use_super, |
487 | ssh_proxy=self.ssh_proxy, | 549 | haslocalmanifests=bool(m.HasLocalManifests), |
488 | clone_filter=project.manifest.CloneFilter, | 550 | hassuperprojecttag=bool(m.superproject), |
489 | partial_clone_exclude=project.manifest.PartialCloneExclude) | 551 | ) |
490 | success = sync_result.success | 552 | if use_super and (m.IsMirror or m.IsArchive): |
491 | remote_fetched = sync_result.remote_fetched | 553 | # Don't use superproject, because we have no working tree. |
492 | 554 | use_super = False | |
493 | output = buf.getvalue() | 555 | superproject_logging_data["superproject"] = False |
494 | if (opt.verbose or not success) and output: | 556 | superproject_logging_data["noworktree"] = True |
495 | print('\n' + output.rstrip()) | 557 | if opt.use_superproject is not False: |
496 | 558 | print( | |
497 | if not success: | 559 | f"{m.path_prefix}: not using superproject because " |
498 | print('error: Cannot fetch %s from %s' | 560 | "there is no working tree." |
499 | % (project.name, project.remote.url), | 561 | ) |
500 | file=sys.stderr) | 562 | |
501 | except KeyboardInterrupt: | 563 | if not use_super: |
502 | print(f'Keyboard interrupt while processing {project.name}') | 564 | continue |
503 | except GitError as e: | 565 | m.superproject.SetQuiet(opt.quiet) |
504 | print('error.GitError: Cannot fetch %s' % str(e), file=sys.stderr) | 566 | print_messages = git_superproject.PrintMessages( |
505 | except Exception as e: | 567 | opt.use_superproject, m |
506 | print('error: Cannot fetch %s (%s: %s)' | 568 | ) |
507 | % (project.name, type(e).__name__, str(e)), file=sys.stderr) | 569 | m.superproject.SetPrintMessages(print_messages) |
508 | raise | 570 | update_result = m.superproject.UpdateProjectsRevisionId( |
509 | 571 | per_manifest[m.path_prefix], git_event_log=self.git_event_log | |
510 | finish = time.time() | 572 | ) |
511 | return _FetchOneResult(success, project, start, finish, remote_fetched) | 573 | manifest_path = update_result.manifest_path |
512 | 574 | superproject_logging_data["updatedrevisionid"] = bool(manifest_path) | |
513 | @classmethod | 575 | if manifest_path: |
514 | def _FetchInitChild(cls, ssh_proxy): | 576 | m.SetManifestOverride(manifest_path) |
515 | cls.ssh_proxy = ssh_proxy | 577 | need_unload = True |
516 | 578 | else: | |
517 | def _Fetch(self, projects, opt, err_event, ssh_proxy): | 579 | if print_messages: |
518 | ret = True | 580 | print( |
519 | 581 | f"{m.path_prefix}: warning: Update of revisionId from " | |
520 | jobs = opt.jobs_network | 582 | "superproject has failed, repo sync will not use " |
521 | fetched = set() | 583 | "superproject to fetch the source. ", |
522 | remote_fetched = set() | 584 | "Please resync with the --no-use-superproject option " |
523 | pm = Progress('Fetching', len(projects), delay=False, quiet=opt.quiet) | 585 | "to avoid this repo warning.", |
524 | 586 | file=sys.stderr, | |
525 | objdir_project_map = dict() | 587 | ) |
526 | for project in projects: | 588 | if update_result.fatal and opt.use_superproject is not None: |
527 | objdir_project_map.setdefault(project.objdir, []).append(project) | 589 | sys.exit(1) |
528 | projects_list = list(objdir_project_map.values()) | 590 | if need_unload: |
529 | 591 | m.outer_client.manifest.Unload() | |
530 | def _ProcessResults(results_sets): | 592 | |
531 | ret = True | 593 | def _FetchProjectList(self, opt, projects): |
532 | for results in results_sets: | 594 | """Main function of the fetch worker. |
533 | for result in results: | 595 | |
534 | success = result.success | 596 | The projects we're given share the same underlying git object store, so |
535 | project = result.project | 597 | we have to fetch them in serial. |
536 | start = result.start | 598 | |
537 | finish = result.finish | 599 | Delegates most of the work to _FetchHelper. |
538 | self._fetch_times.Set(project, finish - start) | 600 | |
539 | self.event_log.AddSync(project, event_log.TASK_SYNC_NETWORK, | 601 | Args: |
540 | start, finish, success) | 602 | opt: Program options returned from optparse. See _Options(). |
541 | if result.remote_fetched: | 603 | projects: Projects to fetch. |
542 | remote_fetched.add(project) | 604 | """ |
543 | # Check for any errors before running any more tasks. | 605 | return [self._FetchOne(opt, x) for x in projects] |
544 | # ...we'll let existing jobs finish, though. | 606 | |
545 | if not success: | 607 | def _FetchOne(self, opt, project): |
546 | ret = False | 608 | """Fetch git objects for a single project. |
547 | else: | 609 | |
548 | fetched.add(project.gitdir) | 610 | Args: |
549 | pm.update(msg=f'Last synced: {project.name}') | 611 | opt: Program options returned from optparse. See _Options(). |
550 | if not ret and opt.fail_fast: | 612 | project: Project object for the project to fetch. |
551 | break | 613 | |
552 | return ret | 614 | Returns: |
553 | 615 | Whether the fetch was successful. | |
554 | # We pass the ssh proxy settings via the class. This allows multiprocessing | 616 | """ |
555 | # to pickle it up when spawning children. We can't pass it as an argument | 617 | start = time.time() |
556 | # to _FetchProjectList below as multiprocessing is unable to pickle those. | 618 | success = False |
557 | Sync.ssh_proxy = None | 619 | remote_fetched = False |
558 | 620 | buf = io.StringIO() | |
559 | # NB: Multiprocessing is heavy, so don't spin it up for one job. | 621 | try: |
560 | if len(projects_list) == 1 or jobs == 1: | 622 | sync_result = project.Sync_NetworkHalf( |
561 | self._FetchInitChild(ssh_proxy) | 623 | quiet=opt.quiet, |
562 | if not _ProcessResults(self._FetchProjectList(opt, x) for x in projects_list): | 624 | verbose=opt.verbose, |
563 | ret = False | 625 | output_redir=buf, |
564 | else: | 626 | current_branch_only=self._GetCurrentBranchOnly( |
565 | # Favor throughput over responsiveness when quiet. It seems that imap() | 627 | opt, project.manifest |
566 | # will yield results in batches relative to chunksize, so even as the | 628 | ), |
567 | # children finish a sync, we won't see the result until one child finishes | 629 | force_sync=opt.force_sync, |
568 | # ~chunksize jobs. When using a large --jobs with large chunksize, this | 630 | clone_bundle=opt.clone_bundle, |
569 | # can be jarring as there will be a large initial delay where repo looks | 631 | tags=opt.tags, |
570 | # like it isn't doing anything and sits at 0%, but then suddenly completes | 632 | archive=project.manifest.IsArchive, |
571 | # a lot of jobs all at once. Since this code is more network bound, we | 633 | optimized_fetch=opt.optimized_fetch, |
572 | # can accept a bit more CPU overhead with a smaller chunksize so that the | 634 | retry_fetches=opt.retry_fetches, |
573 | # user sees more immediate & continuous feedback. | 635 | prune=opt.prune, |
574 | if opt.quiet: | 636 | ssh_proxy=self.ssh_proxy, |
575 | chunksize = WORKER_BATCH_SIZE | 637 | clone_filter=project.manifest.CloneFilter, |
576 | else: | 638 | partial_clone_exclude=project.manifest.PartialCloneExclude, |
577 | pm.update(inc=0, msg='warming up') | 639 | ) |
578 | chunksize = 4 | 640 | success = sync_result.success |
579 | with multiprocessing.Pool(jobs, initializer=self._FetchInitChild, | 641 | remote_fetched = sync_result.remote_fetched |
580 | initargs=(ssh_proxy,)) as pool: | 642 | |
581 | results = pool.imap_unordered( | 643 | output = buf.getvalue() |
582 | functools.partial(self._FetchProjectList, opt), | 644 | if (opt.verbose or not success) and output: |
583 | projects_list, | 645 | print("\n" + output.rstrip()) |
584 | chunksize=chunksize) | 646 | |
585 | if not _ProcessResults(results): | 647 | if not success: |
586 | ret = False | 648 | print( |
587 | pool.close() | 649 | "error: Cannot fetch %s from %s" |
588 | 650 | % (project.name, project.remote.url), | |
589 | # Cleanup the reference now that we're done with it, and we're going to | 651 | file=sys.stderr, |
590 | # release any resources it points to. If we don't, later multiprocessing | 652 | ) |
591 | # usage (e.g. checkouts) will try to pickle and then crash. | 653 | except KeyboardInterrupt: |
592 | del Sync.ssh_proxy | 654 | print(f"Keyboard interrupt while processing {project.name}") |
593 | 655 | except GitError as e: | |
594 | pm.end() | 656 | print("error.GitError: Cannot fetch %s" % str(e), file=sys.stderr) |
595 | self._fetch_times.Save() | 657 | except Exception as e: |
596 | 658 | print( | |
597 | if not self.outer_client.manifest.IsArchive: | 659 | "error: Cannot fetch %s (%s: %s)" |
598 | self._GCProjects(projects, opt, err_event) | 660 | % (project.name, type(e).__name__, str(e)), |
599 | 661 | file=sys.stderr, | |
600 | return _FetchResult(ret, fetched) | 662 | ) |
601 | 663 | raise | |
602 | def _FetchMain(self, opt, args, all_projects, err_event, | 664 | |
603 | ssh_proxy, manifest): | 665 | finish = time.time() |
604 | """The main network fetch loop. | 666 | return _FetchOneResult(success, project, start, finish, remote_fetched) |
605 | 667 | ||
606 | Args: | 668 | @classmethod |
607 | opt: Program options returned from optparse. See _Options(). | 669 | def _FetchInitChild(cls, ssh_proxy): |
608 | args: Command line args used to filter out projects. | 670 | cls.ssh_proxy = ssh_proxy |
609 | all_projects: List of all projects that should be fetched. | 671 | |
610 | err_event: Whether an error was hit while processing. | 672 | def _Fetch(self, projects, opt, err_event, ssh_proxy): |
611 | ssh_proxy: SSH manager for clients & masters. | 673 | ret = True |
612 | manifest: The manifest to use. | 674 | |
613 | 675 | jobs = opt.jobs_network | |
614 | Returns: | 676 | fetched = set() |
615 | List of all projects that should be checked out. | 677 | remote_fetched = set() |
616 | """ | 678 | pm = Progress("Fetching", len(projects), delay=False, quiet=opt.quiet) |
617 | rp = manifest.repoProject | 679 | |
618 | 680 | objdir_project_map = dict() | |
619 | to_fetch = [] | 681 | for project in projects: |
620 | now = time.time() | 682 | objdir_project_map.setdefault(project.objdir, []).append(project) |
621 | if _ONE_DAY_S <= (now - rp.LastFetch): | 683 | projects_list = list(objdir_project_map.values()) |
622 | to_fetch.append(rp) | 684 | |
623 | to_fetch.extend(all_projects) | 685 | def _ProcessResults(results_sets): |
624 | to_fetch.sort(key=self._fetch_times.Get, reverse=True) | 686 | ret = True |
625 | 687 | for results in results_sets: | |
626 | result = self._Fetch(to_fetch, opt, err_event, ssh_proxy) | 688 | for result in results: |
627 | success = result.success | 689 | success = result.success |
628 | fetched = result.projects | 690 | project = result.project |
629 | if not success: | 691 | start = result.start |
630 | err_event.set() | 692 | finish = result.finish |
631 | 693 | self._fetch_times.Set(project, finish - start) | |
632 | _PostRepoFetch(rp, opt.repo_verify) | 694 | self.event_log.AddSync( |
633 | if opt.network_only: | 695 | project, |
634 | # bail out now; the rest touches the working tree | 696 | event_log.TASK_SYNC_NETWORK, |
635 | if err_event.is_set(): | 697 | start, |
636 | print('\nerror: Exited sync due to fetch errors.\n', file=sys.stderr) | 698 | finish, |
637 | sys.exit(1) | 699 | success, |
638 | return _FetchMainResult([]) | 700 | ) |
639 | 701 | if result.remote_fetched: | |
640 | # Iteratively fetch missing and/or nested unregistered submodules | 702 | remote_fetched.add(project) |
641 | previously_missing_set = set() | 703 | # Check for any errors before running any more tasks. |
642 | while True: | 704 | # ...we'll let existing jobs finish, though. |
643 | self._ReloadManifest(None, manifest) | 705 | if not success: |
644 | all_projects = self.GetProjects(args, | 706 | ret = False |
645 | missing_ok=True, | 707 | else: |
646 | submodules_ok=opt.fetch_submodules, | 708 | fetched.add(project.gitdir) |
647 | manifest=manifest, | 709 | pm.update(msg=f"Last synced: {project.name}") |
648 | all_manifests=not opt.this_manifest_only) | 710 | if not ret and opt.fail_fast: |
649 | missing = [] | 711 | break |
650 | for project in all_projects: | 712 | return ret |
651 | if project.gitdir not in fetched: | ||
652 | missing.append(project) | ||
653 | if not missing: | ||
654 | break | ||
655 | # Stop us from non-stopped fetching actually-missing repos: If set of | ||
656 | # missing repos has not been changed from last fetch, we break. | ||
657 | missing_set = set(p.name for p in missing) | ||
658 | if previously_missing_set == missing_set: | ||
659 | break | ||
660 | previously_missing_set = missing_set | ||
661 | result = self._Fetch(missing, opt, err_event, ssh_proxy) | ||
662 | success = result.success | ||
663 | new_fetched = result.projects | ||
664 | if not success: | ||
665 | err_event.set() | ||
666 | fetched.update(new_fetched) | ||
667 | |||
668 | return _FetchMainResult(all_projects) | ||
669 | |||
670 | def _CheckoutOne(self, detach_head, force_sync, project): | ||
671 | """Checkout work tree for one project | ||
672 | |||
673 | Args: | ||
674 | detach_head: Whether to leave a detached HEAD. | ||
675 | force_sync: Force checking out of the repo. | ||
676 | project: Project object for the project to checkout. | ||
677 | |||
678 | Returns: | ||
679 | Whether the fetch was successful. | ||
680 | """ | ||
681 | start = time.time() | ||
682 | syncbuf = SyncBuffer(project.manifest.manifestProject.config, | ||
683 | detach_head=detach_head) | ||
684 | success = False | ||
685 | try: | ||
686 | project.Sync_LocalHalf(syncbuf, force_sync=force_sync) | ||
687 | success = syncbuf.Finish() | ||
688 | except GitError as e: | ||
689 | print('error.GitError: Cannot checkout %s: %s' % | ||
690 | (project.name, str(e)), file=sys.stderr) | ||
691 | except Exception as e: | ||
692 | print('error: Cannot checkout %s: %s: %s' % | ||
693 | (project.name, type(e).__name__, str(e)), | ||
694 | file=sys.stderr) | ||
695 | raise | ||
696 | |||
697 | if not success: | ||
698 | print('error: Cannot checkout %s' % (project.name), file=sys.stderr) | ||
699 | finish = time.time() | ||
700 | return _CheckoutOneResult(success, project, start, finish) | ||
701 | |||
702 | def _Checkout(self, all_projects, opt, err_results): | ||
703 | """Checkout projects listed in all_projects | ||
704 | |||
705 | Args: | ||
706 | all_projects: List of all projects that should be checked out. | ||
707 | opt: Program options returned from optparse. See _Options(). | ||
708 | err_results: A list of strings, paths to git repos where checkout failed. | ||
709 | """ | ||
710 | # Only checkout projects with worktrees. | ||
711 | all_projects = [x for x in all_projects if x.worktree] | ||
712 | 713 | ||
713 | def _ProcessResults(pool, pm, results): | 714 | # We pass the ssh proxy settings via the class. This allows |
714 | ret = True | 715 | # multiprocessing to pickle it up when spawning children. We can't pass |
715 | for result in results: | 716 | # it as an argument to _FetchProjectList below as multiprocessing is |
717 | # unable to pickle those. | ||
718 | Sync.ssh_proxy = None | ||
719 | |||
720 | # NB: Multiprocessing is heavy, so don't spin it up for one job. | ||
721 | if len(projects_list) == 1 or jobs == 1: | ||
722 | self._FetchInitChild(ssh_proxy) | ||
723 | if not _ProcessResults( | ||
724 | self._FetchProjectList(opt, x) for x in projects_list | ||
725 | ): | ||
726 | ret = False | ||
727 | else: | ||
728 | # Favor throughput over responsiveness when quiet. It seems that | ||
729 | # imap() will yield results in batches relative to chunksize, so | ||
730 | # even as the children finish a sync, we won't see the result until | ||
731 | # one child finishes ~chunksize jobs. When using a large --jobs | ||
732 | # with large chunksize, this can be jarring as there will be a large | ||
733 | # initial delay where repo looks like it isn't doing anything and | ||
734 | # sits at 0%, but then suddenly completes a lot of jobs all at once. | ||
735 | # Since this code is more network bound, we can accept a bit more | ||
736 | # CPU overhead with a smaller chunksize so that the user sees more | ||
737 | # immediate & continuous feedback. | ||
738 | if opt.quiet: | ||
739 | chunksize = WORKER_BATCH_SIZE | ||
740 | else: | ||
741 | pm.update(inc=0, msg="warming up") | ||
742 | chunksize = 4 | ||
743 | with multiprocessing.Pool( | ||
744 | jobs, initializer=self._FetchInitChild, initargs=(ssh_proxy,) | ||
745 | ) as pool: | ||
746 | results = pool.imap_unordered( | ||
747 | functools.partial(self._FetchProjectList, opt), | ||
748 | projects_list, | ||
749 | chunksize=chunksize, | ||
750 | ) | ||
751 | if not _ProcessResults(results): | ||
752 | ret = False | ||
753 | pool.close() | ||
754 | |||
755 | # Cleanup the reference now that we're done with it, and we're going to | ||
756 | # release any resources it points to. If we don't, later | ||
757 | # multiprocessing usage (e.g. checkouts) will try to pickle and then | ||
758 | # crash. | ||
759 | del Sync.ssh_proxy | ||
760 | |||
761 | pm.end() | ||
762 | self._fetch_times.Save() | ||
763 | |||
764 | if not self.outer_client.manifest.IsArchive: | ||
765 | self._GCProjects(projects, opt, err_event) | ||
766 | |||
767 | return _FetchResult(ret, fetched) | ||
768 | |||
769 | def _FetchMain( | ||
770 | self, opt, args, all_projects, err_event, ssh_proxy, manifest | ||
771 | ): | ||
772 | """The main network fetch loop. | ||
773 | |||
774 | Args: | ||
775 | opt: Program options returned from optparse. See _Options(). | ||
776 | args: Command line args used to filter out projects. | ||
777 | all_projects: List of all projects that should be fetched. | ||
778 | err_event: Whether an error was hit while processing. | ||
779 | ssh_proxy: SSH manager for clients & masters. | ||
780 | manifest: The manifest to use. | ||
781 | |||
782 | Returns: | ||
783 | List of all projects that should be checked out. | ||
784 | """ | ||
785 | rp = manifest.repoProject | ||
786 | |||
787 | to_fetch = [] | ||
788 | now = time.time() | ||
789 | if _ONE_DAY_S <= (now - rp.LastFetch): | ||
790 | to_fetch.append(rp) | ||
791 | to_fetch.extend(all_projects) | ||
792 | to_fetch.sort(key=self._fetch_times.Get, reverse=True) | ||
793 | |||
794 | result = self._Fetch(to_fetch, opt, err_event, ssh_proxy) | ||
716 | success = result.success | 795 | success = result.success |
717 | project = result.project | 796 | fetched = result.projects |
718 | start = result.start | ||
719 | finish = result.finish | ||
720 | self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL, | ||
721 | start, finish, success) | ||
722 | # Check for any errors before running any more tasks. | ||
723 | # ...we'll let existing jobs finish, though. | ||
724 | if not success: | 797 | if not success: |
725 | ret = False | 798 | err_event.set() |
726 | err_results.append(project.RelPath(local=opt.this_manifest_only)) | 799 | |
727 | if opt.fail_fast: | 800 | _PostRepoFetch(rp, opt.repo_verify) |
728 | if pool: | 801 | if opt.network_only: |
729 | pool.close() | 802 | # Bail out now; the rest touches the working tree. |
803 | if err_event.is_set(): | ||
804 | print( | ||
805 | "\nerror: Exited sync due to fetch errors.\n", | ||
806 | file=sys.stderr, | ||
807 | ) | ||
808 | sys.exit(1) | ||
809 | return _FetchMainResult([]) | ||
810 | |||
811 | # Iteratively fetch missing and/or nested unregistered submodules. | ||
812 | previously_missing_set = set() | ||
813 | while True: | ||
814 | self._ReloadManifest(None, manifest) | ||
815 | all_projects = self.GetProjects( | ||
816 | args, | ||
817 | missing_ok=True, | ||
818 | submodules_ok=opt.fetch_submodules, | ||
819 | manifest=manifest, | ||
820 | all_manifests=not opt.this_manifest_only, | ||
821 | ) | ||
822 | missing = [] | ||
823 | for project in all_projects: | ||
824 | if project.gitdir not in fetched: | ||
825 | missing.append(project) | ||
826 | if not missing: | ||
827 | break | ||
828 | # Stop us from non-stopped fetching actually-missing repos: If set | ||
829 | # of missing repos has not been changed from last fetch, we break. | ||
830 | missing_set = set(p.name for p in missing) | ||
831 | if previously_missing_set == missing_set: | ||
832 | break | ||
833 | previously_missing_set = missing_set | ||
834 | result = self._Fetch(missing, opt, err_event, ssh_proxy) | ||
835 | success = result.success | ||
836 | new_fetched = result.projects | ||
837 | if not success: | ||
838 | err_event.set() | ||
839 | fetched.update(new_fetched) | ||
840 | |||
841 | return _FetchMainResult(all_projects) | ||
842 | |||
843 | def _CheckoutOne(self, detach_head, force_sync, project): | ||
844 | """Checkout work tree for one project | ||
845 | |||
846 | Args: | ||
847 | detach_head: Whether to leave a detached HEAD. | ||
848 | force_sync: Force checking out of the repo. | ||
849 | project: Project object for the project to checkout. | ||
850 | |||
851 | Returns: | ||
852 | Whether the fetch was successful. | ||
853 | """ | ||
854 | start = time.time() | ||
855 | syncbuf = SyncBuffer( | ||
856 | project.manifest.manifestProject.config, detach_head=detach_head | ||
857 | ) | ||
858 | success = False | ||
859 | try: | ||
860 | project.Sync_LocalHalf(syncbuf, force_sync=force_sync) | ||
861 | success = syncbuf.Finish() | ||
862 | except GitError as e: | ||
863 | print( | ||
864 | "error.GitError: Cannot checkout %s: %s" | ||
865 | % (project.name, str(e)), | ||
866 | file=sys.stderr, | ||
867 | ) | ||
868 | except Exception as e: | ||
869 | print( | ||
870 | "error: Cannot checkout %s: %s: %s" | ||
871 | % (project.name, type(e).__name__, str(e)), | ||
872 | file=sys.stderr, | ||
873 | ) | ||
874 | raise | ||
875 | |||
876 | if not success: | ||
877 | print("error: Cannot checkout %s" % (project.name), file=sys.stderr) | ||
878 | finish = time.time() | ||
879 | return _CheckoutOneResult(success, project, start, finish) | ||
880 | |||
881 | def _Checkout(self, all_projects, opt, err_results): | ||
882 | """Checkout projects listed in all_projects | ||
883 | |||
884 | Args: | ||
885 | all_projects: List of all projects that should be checked out. | ||
886 | opt: Program options returned from optparse. See _Options(). | ||
887 | err_results: A list of strings, paths to git repos where checkout | ||
888 | failed. | ||
889 | """ | ||
890 | # Only checkout projects with worktrees. | ||
891 | all_projects = [x for x in all_projects if x.worktree] | ||
892 | |||
893 | def _ProcessResults(pool, pm, results): | ||
894 | ret = True | ||
895 | for result in results: | ||
896 | success = result.success | ||
897 | project = result.project | ||
898 | start = result.start | ||
899 | finish = result.finish | ||
900 | self.event_log.AddSync( | ||
901 | project, event_log.TASK_SYNC_LOCAL, start, finish, success | ||
902 | ) | ||
903 | # Check for any errors before running any more tasks. | ||
904 | # ...we'll let existing jobs finish, though. | ||
905 | if not success: | ||
906 | ret = False | ||
907 | err_results.append( | ||
908 | project.RelPath(local=opt.this_manifest_only) | ||
909 | ) | ||
910 | if opt.fail_fast: | ||
911 | if pool: | ||
912 | pool.close() | ||
913 | return ret | ||
914 | pm.update(msg=project.name) | ||
730 | return ret | 915 | return ret |
731 | pm.update(msg=project.name) | ||
732 | return ret | ||
733 | |||
734 | return self.ExecuteInParallel( | ||
735 | opt.jobs_checkout, | ||
736 | functools.partial(self._CheckoutOne, opt.detach_head, opt.force_sync), | ||
737 | all_projects, | ||
738 | callback=_ProcessResults, | ||
739 | output=Progress('Checking out', len(all_projects), quiet=opt.quiet)) and not err_results | ||
740 | |||
741 | @staticmethod | ||
742 | def _GetPreciousObjectsState(project: Project, opt): | ||
743 | """Get the preciousObjects state for the project. | ||
744 | |||
745 | Args: | ||
746 | project (Project): the project to examine, and possibly correct. | ||
747 | opt (optparse.Values): options given to sync. | ||
748 | |||
749 | Returns: | ||
750 | Expected state of extensions.preciousObjects: | ||
751 | False: Should be disabled. (not present) | ||
752 | True: Should be enabled. | ||
753 | """ | ||
754 | if project.use_git_worktrees: | ||
755 | return False | ||
756 | projects = project.manifest.GetProjectsWithName(project.name, | ||
757 | all_manifests=True) | ||
758 | if len(projects) == 1: | ||
759 | return False | ||
760 | relpath = project.RelPath(local=opt.this_manifest_only) | ||
761 | if len(projects) > 1: | ||
762 | # Objects are potentially shared with another project. | ||
763 | # See the logic in Project.Sync_NetworkHalf regarding UseAlternates. | ||
764 | # - When False, shared projects share (via symlink) | ||
765 | # .repo/project-objects/{PROJECT_NAME}.git as the one-and-only objects | ||
766 | # directory. All objects are precious, since there is no project with a | ||
767 | # complete set of refs. | ||
768 | # - When True, shared projects share (via info/alternates) | ||
769 | # .repo/project-objects/{PROJECT_NAME}.git as an alternate object store, | ||
770 | # which is written only on the first clone of the project, and is not | ||
771 | # written subsequently. (When Sync_NetworkHalf sees that it exists, it | ||
772 | # makes sure that the alternates file points there, and uses a | ||
773 | # project-local .git/objects directory for all syncs going forward. | ||
774 | # We do not support switching between the options. The environment | ||
775 | # variable is present for testing and migration only. | ||
776 | return not project.UseAlternates | ||
777 | |||
778 | return False | ||
779 | |||
780 | def _SetPreciousObjectsState(self, project: Project, opt): | ||
781 | """Correct the preciousObjects state for the project. | ||
782 | |||
783 | Args: | ||
784 | project: the project to examine, and possibly correct. | ||
785 | opt: options given to sync. | ||
786 | """ | ||
787 | expected = self._GetPreciousObjectsState(project, opt) | ||
788 | actual = project.config.GetBoolean('extensions.preciousObjects') or False | ||
789 | relpath = project.RelPath(local=opt.this_manifest_only) | ||
790 | |||
791 | if expected != actual: | ||
792 | # If this is unexpected, log it and repair. | ||
793 | Trace(f'{relpath} expected preciousObjects={expected}, got {actual}') | ||
794 | if expected: | ||
795 | if not opt.quiet: | ||
796 | print('\r%s: Shared project %s found, disabling pruning.' % | ||
797 | (relpath, project.name)) | ||
798 | if git_require((2, 7, 0)): | ||
799 | project.EnableRepositoryExtension('preciousObjects') | ||
800 | else: | ||
801 | # This isn't perfect, but it's the best we can do with old git. | ||
802 | print('\r%s: WARNING: shared projects are unreliable when using ' | ||
803 | 'old versions of git; please upgrade to git-2.7.0+.' | ||
804 | % (relpath,), | ||
805 | file=sys.stderr) | ||
806 | project.config.SetString('gc.pruneExpire', 'never') | ||
807 | else: | ||
808 | if not opt.quiet: | ||
809 | print(f'\r{relpath}: not shared, disabling pruning.') | ||
810 | project.config.SetString('extensions.preciousObjects', None) | ||
811 | project.config.SetString('gc.pruneExpire', None) | ||
812 | 916 | ||
813 | def _GCProjects(self, projects, opt, err_event): | 917 | return ( |
814 | """Perform garbage collection. | 918 | self.ExecuteInParallel( |
919 | opt.jobs_checkout, | ||
920 | functools.partial( | ||
921 | self._CheckoutOne, opt.detach_head, opt.force_sync | ||
922 | ), | ||
923 | all_projects, | ||
924 | callback=_ProcessResults, | ||
925 | output=Progress( | ||
926 | "Checking out", len(all_projects), quiet=opt.quiet | ||
927 | ), | ||
928 | ) | ||
929 | and not err_results | ||
930 | ) | ||
815 | 931 | ||
816 | If We are skipping garbage collection (opt.auto_gc not set), we still want | 932 | @staticmethod |
817 | to potentially mark objects precious, so that `git gc` does not discard | 933 | def _GetPreciousObjectsState(project: Project, opt): |
818 | shared objects. | 934 | """Get the preciousObjects state for the project. |
819 | """ | 935 | |
820 | if not opt.auto_gc: | 936 | Args: |
821 | # Just repair preciousObjects state, and return. | 937 | project (Project): the project to examine, and possibly correct. |
822 | for project in projects: | 938 | opt (optparse.Values): options given to sync. |
823 | self._SetPreciousObjectsState(project, opt) | 939 | |
824 | return | 940 | Returns: |
825 | 941 | Expected state of extensions.preciousObjects: | |
826 | pm = Progress('Garbage collecting', len(projects), delay=False, | 942 | False: Should be disabled. (not present) |
827 | quiet=opt.quiet) | 943 | True: Should be enabled. |
828 | pm.update(inc=0, msg='prescan') | 944 | """ |
829 | 945 | if project.use_git_worktrees: | |
830 | tidy_dirs = {} | 946 | return False |
831 | for project in projects: | 947 | projects = project.manifest.GetProjectsWithName( |
832 | self._SetPreciousObjectsState(project, opt) | 948 | project.name, all_manifests=True |
833 | |||
834 | project.config.SetString('gc.autoDetach', 'false') | ||
835 | # Only call git gc once per objdir, but call pack-refs for the remainder. | ||
836 | if project.objdir not in tidy_dirs: | ||
837 | tidy_dirs[project.objdir] = ( | ||
838 | True, # Run a full gc. | ||
839 | project.bare_git, | ||
840 | ) | 949 | ) |
841 | elif project.gitdir not in tidy_dirs: | 950 | if len(projects) == 1: |
842 | tidy_dirs[project.gitdir] = ( | 951 | return False |
843 | False, # Do not run a full gc; just run pack-refs. | 952 | if len(projects) > 1: |
844 | project.bare_git, | 953 | # Objects are potentially shared with another project. |
954 | # See the logic in Project.Sync_NetworkHalf regarding UseAlternates. | ||
955 | # - When False, shared projects share (via symlink) | ||
956 | # .repo/project-objects/{PROJECT_NAME}.git as the one-and-only | ||
957 | # objects directory. All objects are precious, since there is no | ||
958 | # project with a complete set of refs. | ||
959 | # - When True, shared projects share (via info/alternates) | ||
960 | # .repo/project-objects/{PROJECT_NAME}.git as an alternate object | ||
961 | # store, which is written only on the first clone of the project, | ||
962 | # and is not written subsequently. (When Sync_NetworkHalf sees | ||
963 | # that it exists, it makes sure that the alternates file points | ||
964 | # there, and uses a project-local .git/objects directory for all | ||
965 | # syncs going forward. | ||
966 | # We do not support switching between the options. The environment | ||
967 | # variable is present for testing and migration only. | ||
968 | return not project.UseAlternates | ||
969 | |||
970 | return False | ||
971 | |||
972 | def _SetPreciousObjectsState(self, project: Project, opt): | ||
973 | """Correct the preciousObjects state for the project. | ||
974 | |||
975 | Args: | ||
976 | project: the project to examine, and possibly correct. | ||
977 | opt: options given to sync. | ||
978 | """ | ||
979 | expected = self._GetPreciousObjectsState(project, opt) | ||
980 | actual = ( | ||
981 | project.config.GetBoolean("extensions.preciousObjects") or False | ||
845 | ) | 982 | ) |
846 | 983 | relpath = project.RelPath(local=opt.this_manifest_only) | |
847 | jobs = opt.jobs | 984 | |
848 | 985 | if expected != actual: | |
849 | if jobs < 2: | 986 | # If this is unexpected, log it and repair. |
850 | for (run_gc, bare_git) in tidy_dirs.values(): | 987 | Trace( |
851 | pm.update(msg=bare_git._project.name) | 988 | f"{relpath} expected preciousObjects={expected}, got {actual}" |
852 | 989 | ) | |
853 | if run_gc: | 990 | if expected: |
854 | bare_git.gc('--auto') | 991 | if not opt.quiet: |
992 | print( | ||
993 | "\r%s: Shared project %s found, disabling pruning." | ||
994 | % (relpath, project.name) | ||
995 | ) | ||
996 | if git_require((2, 7, 0)): | ||
997 | project.EnableRepositoryExtension("preciousObjects") | ||
998 | else: | ||
999 | # This isn't perfect, but it's the best we can do with old | ||
1000 | # git. | ||
1001 | print( | ||
1002 | "\r%s: WARNING: shared projects are unreliable when " | ||
1003 | "using old versions of git; please upgrade to " | ||
1004 | "git-2.7.0+." % (relpath,), | ||
1005 | file=sys.stderr, | ||
1006 | ) | ||
1007 | project.config.SetString("gc.pruneExpire", "never") | ||
1008 | else: | ||
1009 | if not opt.quiet: | ||
1010 | print(f"\r{relpath}: not shared, disabling pruning.") | ||
1011 | project.config.SetString("extensions.preciousObjects", None) | ||
1012 | project.config.SetString("gc.pruneExpire", None) | ||
1013 | |||
1014 | def _GCProjects(self, projects, opt, err_event): | ||
1015 | """Perform garbage collection. | ||
1016 | |||
1017 | If We are skipping garbage collection (opt.auto_gc not set), we still | ||
1018 | want to potentially mark objects precious, so that `git gc` does not | ||
1019 | discard shared objects. | ||
1020 | """ | ||
1021 | if not opt.auto_gc: | ||
1022 | # Just repair preciousObjects state, and return. | ||
1023 | for project in projects: | ||
1024 | self._SetPreciousObjectsState(project, opt) | ||
1025 | return | ||
1026 | |||
1027 | pm = Progress( | ||
1028 | "Garbage collecting", len(projects), delay=False, quiet=opt.quiet | ||
1029 | ) | ||
1030 | pm.update(inc=0, msg="prescan") | ||
1031 | |||
1032 | tidy_dirs = {} | ||
1033 | for project in projects: | ||
1034 | self._SetPreciousObjectsState(project, opt) | ||
1035 | |||
1036 | project.config.SetString("gc.autoDetach", "false") | ||
1037 | # Only call git gc once per objdir, but call pack-refs for the | ||
1038 | # remainder. | ||
1039 | if project.objdir not in tidy_dirs: | ||
1040 | tidy_dirs[project.objdir] = ( | ||
1041 | True, # Run a full gc. | ||
1042 | project.bare_git, | ||
1043 | ) | ||
1044 | elif project.gitdir not in tidy_dirs: | ||
1045 | tidy_dirs[project.gitdir] = ( | ||
1046 | False, # Do not run a full gc; just run pack-refs. | ||
1047 | project.bare_git, | ||
1048 | ) | ||
1049 | |||
1050 | jobs = opt.jobs | ||
1051 | |||
1052 | if jobs < 2: | ||
1053 | for run_gc, bare_git in tidy_dirs.values(): | ||
1054 | pm.update(msg=bare_git._project.name) | ||
1055 | |||
1056 | if run_gc: | ||
1057 | bare_git.gc("--auto") | ||
1058 | else: | ||
1059 | bare_git.pack_refs() | ||
1060 | pm.end() | ||
1061 | return | ||
1062 | |||
1063 | cpu_count = os.cpu_count() | ||
1064 | config = {"pack.threads": cpu_count // jobs if cpu_count > jobs else 1} | ||
1065 | |||
1066 | threads = set() | ||
1067 | sem = _threading.Semaphore(jobs) | ||
1068 | |||
1069 | def tidy_up(run_gc, bare_git): | ||
1070 | pm.start(bare_git._project.name) | ||
1071 | try: | ||
1072 | try: | ||
1073 | if run_gc: | ||
1074 | bare_git.gc("--auto", config=config) | ||
1075 | else: | ||
1076 | bare_git.pack_refs(config=config) | ||
1077 | except GitError: | ||
1078 | err_event.set() | ||
1079 | except Exception: | ||
1080 | err_event.set() | ||
1081 | raise | ||
1082 | finally: | ||
1083 | pm.finish(bare_git._project.name) | ||
1084 | sem.release() | ||
1085 | |||
1086 | for run_gc, bare_git in tidy_dirs.values(): | ||
1087 | if err_event.is_set() and opt.fail_fast: | ||
1088 | break | ||
1089 | sem.acquire() | ||
1090 | t = _threading.Thread( | ||
1091 | target=tidy_up, | ||
1092 | args=( | ||
1093 | run_gc, | ||
1094 | bare_git, | ||
1095 | ), | ||
1096 | ) | ||
1097 | t.daemon = True | ||
1098 | threads.add(t) | ||
1099 | t.start() | ||
1100 | |||
1101 | for t in threads: | ||
1102 | t.join() | ||
1103 | pm.end() | ||
1104 | |||
1105 | def _ReloadManifest(self, manifest_name, manifest): | ||
1106 | """Reload the manfiest from the file specified by the |manifest_name|. | ||
1107 | |||
1108 | It unloads the manifest if |manifest_name| is None. | ||
1109 | |||
1110 | Args: | ||
1111 | manifest_name: Manifest file to be reloaded. | ||
1112 | manifest: The manifest to use. | ||
1113 | """ | ||
1114 | if manifest_name: | ||
1115 | # Override calls Unload already. | ||
1116 | manifest.Override(manifest_name) | ||
855 | else: | 1117 | else: |
856 | bare_git.pack_refs() | 1118 | manifest.Unload() |
857 | pm.end() | 1119 | |
858 | return | 1120 | def UpdateProjectList(self, opt, manifest): |
859 | 1121 | """Update the cached projects list for |manifest| | |
860 | cpu_count = os.cpu_count() | 1122 | |
861 | config = {'pack.threads': cpu_count // jobs if cpu_count > jobs else 1} | 1123 | In a multi-manifest checkout, each manifest has its own project.list. |
862 | 1124 | ||
863 | threads = set() | 1125 | Args: |
864 | sem = _threading.Semaphore(jobs) | 1126 | opt: Program options returned from optparse. See _Options(). |
1127 | manifest: The manifest to use. | ||
1128 | |||
1129 | Returns: | ||
1130 | 0: success | ||
1131 | 1: failure | ||
1132 | """ | ||
1133 | new_project_paths = [] | ||
1134 | for project in self.GetProjects( | ||
1135 | None, missing_ok=True, manifest=manifest, all_manifests=False | ||
1136 | ): | ||
1137 | if project.relpath: | ||
1138 | new_project_paths.append(project.relpath) | ||
1139 | file_name = "project.list" | ||
1140 | file_path = os.path.join(manifest.subdir, file_name) | ||
1141 | old_project_paths = [] | ||
1142 | |||
1143 | if os.path.exists(file_path): | ||
1144 | with open(file_path, "r") as fd: | ||
1145 | old_project_paths = fd.read().split("\n") | ||
1146 | # In reversed order, so subfolders are deleted before parent folder. | ||
1147 | for path in sorted(old_project_paths, reverse=True): | ||
1148 | if not path: | ||
1149 | continue | ||
1150 | if path not in new_project_paths: | ||
1151 | # If the path has already been deleted, we don't need to do | ||
1152 | # it. | ||
1153 | gitdir = os.path.join(manifest.topdir, path, ".git") | ||
1154 | if os.path.exists(gitdir): | ||
1155 | project = Project( | ||
1156 | manifest=manifest, | ||
1157 | name=path, | ||
1158 | remote=RemoteSpec("origin"), | ||
1159 | gitdir=gitdir, | ||
1160 | objdir=gitdir, | ||
1161 | use_git_worktrees=os.path.isfile(gitdir), | ||
1162 | worktree=os.path.join(manifest.topdir, path), | ||
1163 | relpath=path, | ||
1164 | revisionExpr="HEAD", | ||
1165 | revisionId=None, | ||
1166 | groups=None, | ||
1167 | ) | ||
1168 | if not project.DeleteWorktree( | ||
1169 | quiet=opt.quiet, force=opt.force_remove_dirty | ||
1170 | ): | ||
1171 | return 1 | ||
1172 | |||
1173 | new_project_paths.sort() | ||
1174 | with open(file_path, "w") as fd: | ||
1175 | fd.write("\n".join(new_project_paths)) | ||
1176 | fd.write("\n") | ||
1177 | return 0 | ||
1178 | |||
1179 | def UpdateCopyLinkfileList(self, manifest): | ||
1180 | """Save all dests of copyfile and linkfile, and update them if needed. | ||
1181 | |||
1182 | Returns: | ||
1183 | Whether update was successful. | ||
1184 | """ | ||
1185 | new_paths = {} | ||
1186 | new_linkfile_paths = [] | ||
1187 | new_copyfile_paths = [] | ||
1188 | for project in self.GetProjects( | ||
1189 | None, missing_ok=True, manifest=manifest, all_manifests=False | ||
1190 | ): | ||
1191 | new_linkfile_paths.extend(x.dest for x in project.linkfiles) | ||
1192 | new_copyfile_paths.extend(x.dest for x in project.copyfiles) | ||
1193 | |||
1194 | new_paths = { | ||
1195 | "linkfile": new_linkfile_paths, | ||
1196 | "copyfile": new_copyfile_paths, | ||
1197 | } | ||
1198 | |||
1199 | copylinkfile_name = "copy-link-files.json" | ||
1200 | copylinkfile_path = os.path.join(manifest.subdir, copylinkfile_name) | ||
1201 | old_copylinkfile_paths = {} | ||
1202 | |||
1203 | if os.path.exists(copylinkfile_path): | ||
1204 | with open(copylinkfile_path, "rb") as fp: | ||
1205 | try: | ||
1206 | old_copylinkfile_paths = json.load(fp) | ||
1207 | except Exception: | ||
1208 | print( | ||
1209 | "error: %s is not a json formatted file." | ||
1210 | % copylinkfile_path, | ||
1211 | file=sys.stderr, | ||
1212 | ) | ||
1213 | platform_utils.remove(copylinkfile_path) | ||
1214 | return False | ||
1215 | |||
1216 | need_remove_files = [] | ||
1217 | need_remove_files.extend( | ||
1218 | set(old_copylinkfile_paths.get("linkfile", [])) | ||
1219 | - set(new_linkfile_paths) | ||
1220 | ) | ||
1221 | need_remove_files.extend( | ||
1222 | set(old_copylinkfile_paths.get("copyfile", [])) | ||
1223 | - set(new_copyfile_paths) | ||
1224 | ) | ||
1225 | |||
1226 | for need_remove_file in need_remove_files: | ||
1227 | # Try to remove the updated copyfile or linkfile. | ||
1228 | # So, if the file is not exist, nothing need to do. | ||
1229 | platform_utils.remove(need_remove_file, missing_ok=True) | ||
1230 | |||
1231 | # Create copy-link-files.json, save dest path of "copyfile" and | ||
1232 | # "linkfile". | ||
1233 | with open(copylinkfile_path, "w", encoding="utf-8") as fp: | ||
1234 | json.dump(new_paths, fp) | ||
1235 | return True | ||
1236 | |||
1237 | def _SmartSyncSetup(self, opt, smart_sync_manifest_path, manifest): | ||
1238 | if not manifest.manifest_server: | ||
1239 | print( | ||
1240 | "error: cannot smart sync: no manifest server defined in " | ||
1241 | "manifest", | ||
1242 | file=sys.stderr, | ||
1243 | ) | ||
1244 | sys.exit(1) | ||
1245 | |||
1246 | manifest_server = manifest.manifest_server | ||
1247 | if not opt.quiet: | ||
1248 | print("Using manifest server %s" % manifest_server) | ||
1249 | |||
1250 | if "@" not in manifest_server: | ||
1251 | username = None | ||
1252 | password = None | ||
1253 | if opt.manifest_server_username and opt.manifest_server_password: | ||
1254 | username = opt.manifest_server_username | ||
1255 | password = opt.manifest_server_password | ||
1256 | else: | ||
1257 | try: | ||
1258 | info = netrc.netrc() | ||
1259 | except IOError: | ||
1260 | # .netrc file does not exist or could not be opened. | ||
1261 | pass | ||
1262 | else: | ||
1263 | try: | ||
1264 | parse_result = urllib.parse.urlparse(manifest_server) | ||
1265 | if parse_result.hostname: | ||
1266 | auth = info.authenticators(parse_result.hostname) | ||
1267 | if auth: | ||
1268 | username, _account, password = auth | ||
1269 | else: | ||
1270 | print( | ||
1271 | "No credentials found for %s in .netrc" | ||
1272 | % parse_result.hostname, | ||
1273 | file=sys.stderr, | ||
1274 | ) | ||
1275 | except netrc.NetrcParseError as e: | ||
1276 | print( | ||
1277 | "Error parsing .netrc file: %s" % e, file=sys.stderr | ||
1278 | ) | ||
1279 | |||
1280 | if username and password: | ||
1281 | manifest_server = manifest_server.replace( | ||
1282 | "://", "://%s:%s@" % (username, password), 1 | ||
1283 | ) | ||
1284 | |||
1285 | transport = PersistentTransport(manifest_server) | ||
1286 | if manifest_server.startswith("persistent-"): | ||
1287 | manifest_server = manifest_server[len("persistent-") :] | ||
865 | 1288 | ||
866 | def tidy_up(run_gc, bare_git): | ||
867 | pm.start(bare_git._project.name) | ||
868 | try: | ||
869 | try: | 1289 | try: |
870 | if run_gc: | 1290 | server = xmlrpc.client.Server(manifest_server, transport=transport) |
871 | bare_git.gc('--auto', config=config) | 1291 | if opt.smart_sync: |
872 | else: | 1292 | branch = self._GetBranch(manifest.manifestProject) |
873 | bare_git.pack_refs(config=config) | 1293 | |
874 | except GitError: | 1294 | if "SYNC_TARGET" in os.environ: |
875 | err_event.set() | 1295 | target = os.environ["SYNC_TARGET"] |
876 | except Exception: | 1296 | [success, manifest_str] = server.GetApprovedManifest( |
877 | err_event.set() | 1297 | branch, target |
878 | raise | 1298 | ) |
879 | finally: | 1299 | elif ( |
880 | pm.finish(bare_git._project.name) | 1300 | "TARGET_PRODUCT" in os.environ |
881 | sem.release() | 1301 | and "TARGET_BUILD_VARIANT" in os.environ |
882 | 1302 | ): | |
883 | for (run_gc, bare_git) in tidy_dirs.values(): | 1303 | target = "%s-%s" % ( |
884 | if err_event.is_set() and opt.fail_fast: | 1304 | os.environ["TARGET_PRODUCT"], |
885 | break | 1305 | os.environ["TARGET_BUILD_VARIANT"], |
886 | sem.acquire() | 1306 | ) |
887 | t = _threading.Thread(target=tidy_up, args=(run_gc, bare_git,)) | 1307 | [success, manifest_str] = server.GetApprovedManifest( |
888 | t.daemon = True | 1308 | branch, target |
889 | threads.add(t) | 1309 | ) |
890 | t.start() | 1310 | else: |
891 | 1311 | [success, manifest_str] = server.GetApprovedManifest(branch) | |
892 | for t in threads: | 1312 | else: |
893 | t.join() | 1313 | assert opt.smart_tag |
894 | pm.end() | 1314 | [success, manifest_str] = server.GetManifest(opt.smart_tag) |
895 | 1315 | ||
896 | def _ReloadManifest(self, manifest_name, manifest): | 1316 | if success: |
897 | """Reload the manfiest from the file specified by the |manifest_name|. | 1317 | manifest_name = os.path.basename(smart_sync_manifest_path) |
898 | 1318 | try: | |
899 | It unloads the manifest if |manifest_name| is None. | 1319 | with open(smart_sync_manifest_path, "w") as f: |
900 | 1320 | f.write(manifest_str) | |
901 | Args: | 1321 | except IOError as e: |
902 | manifest_name: Manifest file to be reloaded. | 1322 | print( |
903 | manifest: The manifest to use. | 1323 | "error: cannot write manifest to %s:\n%s" |
904 | """ | 1324 | % (smart_sync_manifest_path, e), |
905 | if manifest_name: | 1325 | file=sys.stderr, |
906 | # Override calls Unload already | 1326 | ) |
907 | manifest.Override(manifest_name) | 1327 | sys.exit(1) |
908 | else: | 1328 | self._ReloadManifest(manifest_name, manifest) |
909 | manifest.Unload() | 1329 | else: |
910 | 1330 | print( | |
911 | def UpdateProjectList(self, opt, manifest): | 1331 | "error: manifest server RPC call failed: %s" % manifest_str, |
912 | """Update the cached projects list for |manifest| | 1332 | file=sys.stderr, |
913 | 1333 | ) | |
914 | In a multi-manifest checkout, each manifest has its own project.list. | 1334 | sys.exit(1) |
1335 | except (socket.error, IOError, xmlrpc.client.Fault) as e: | ||
1336 | print( | ||
1337 | "error: cannot connect to manifest server %s:\n%s" | ||
1338 | % (manifest.manifest_server, e), | ||
1339 | file=sys.stderr, | ||
1340 | ) | ||
1341 | sys.exit(1) | ||
1342 | except xmlrpc.client.ProtocolError as e: | ||
1343 | print( | ||
1344 | "error: cannot connect to manifest server %s:\n%d %s" | ||
1345 | % (manifest.manifest_server, e.errcode, e.errmsg), | ||
1346 | file=sys.stderr, | ||
1347 | ) | ||
1348 | sys.exit(1) | ||
1349 | |||
1350 | return manifest_name | ||
1351 | |||
1352 | def _UpdateAllManifestProjects(self, opt, mp, manifest_name): | ||
1353 | """Fetch & update the local manifest project. | ||
1354 | |||
1355 | After syncing the manifest project, if the manifest has any sub | ||
1356 | manifests, those are recursively processed. | ||
1357 | |||
1358 | Args: | ||
1359 | opt: Program options returned from optparse. See _Options(). | ||
1360 | mp: the manifestProject to query. | ||
1361 | manifest_name: Manifest file to be reloaded. | ||
1362 | """ | ||
1363 | if not mp.standalone_manifest_url: | ||
1364 | self._UpdateManifestProject(opt, mp, manifest_name) | ||
1365 | |||
1366 | if mp.manifest.submanifests: | ||
1367 | for submanifest in mp.manifest.submanifests.values(): | ||
1368 | child = submanifest.repo_client.manifest | ||
1369 | child.manifestProject.SyncWithPossibleInit( | ||
1370 | submanifest, | ||
1371 | current_branch_only=self._GetCurrentBranchOnly(opt, child), | ||
1372 | verbose=opt.verbose, | ||
1373 | tags=opt.tags, | ||
1374 | git_event_log=self.git_event_log, | ||
1375 | ) | ||
1376 | self._UpdateAllManifestProjects( | ||
1377 | opt, child.manifestProject, None | ||
1378 | ) | ||
1379 | |||
1380 | def _UpdateManifestProject(self, opt, mp, manifest_name): | ||
1381 | """Fetch & update the local manifest project. | ||
1382 | |||
1383 | Args: | ||
1384 | opt: Program options returned from optparse. See _Options(). | ||
1385 | mp: the manifestProject to query. | ||
1386 | manifest_name: Manifest file to be reloaded. | ||
1387 | """ | ||
1388 | if not opt.local_only: | ||
1389 | start = time.time() | ||
1390 | success = mp.Sync_NetworkHalf( | ||
1391 | quiet=opt.quiet, | ||
1392 | verbose=opt.verbose, | ||
1393 | current_branch_only=self._GetCurrentBranchOnly( | ||
1394 | opt, mp.manifest | ||
1395 | ), | ||
1396 | force_sync=opt.force_sync, | ||
1397 | tags=opt.tags, | ||
1398 | optimized_fetch=opt.optimized_fetch, | ||
1399 | retry_fetches=opt.retry_fetches, | ||
1400 | submodules=mp.manifest.HasSubmodules, | ||
1401 | clone_filter=mp.manifest.CloneFilter, | ||
1402 | partial_clone_exclude=mp.manifest.PartialCloneExclude, | ||
1403 | ) | ||
1404 | finish = time.time() | ||
1405 | self.event_log.AddSync( | ||
1406 | mp, event_log.TASK_SYNC_NETWORK, start, finish, success | ||
1407 | ) | ||
1408 | |||
1409 | if mp.HasChanges: | ||
1410 | syncbuf = SyncBuffer(mp.config) | ||
1411 | start = time.time() | ||
1412 | mp.Sync_LocalHalf(syncbuf, submodules=mp.manifest.HasSubmodules) | ||
1413 | clean = syncbuf.Finish() | ||
1414 | self.event_log.AddSync( | ||
1415 | mp, event_log.TASK_SYNC_LOCAL, start, time.time(), clean | ||
1416 | ) | ||
1417 | if not clean: | ||
1418 | sys.exit(1) | ||
1419 | self._ReloadManifest(manifest_name, mp.manifest) | ||
1420 | |||
1421 | def ValidateOptions(self, opt, args): | ||
1422 | if opt.force_broken: | ||
1423 | print( | ||
1424 | "warning: -f/--force-broken is now the default behavior, and " | ||
1425 | "the options are deprecated", | ||
1426 | file=sys.stderr, | ||
1427 | ) | ||
1428 | if opt.network_only and opt.detach_head: | ||
1429 | self.OptionParser.error("cannot combine -n and -d") | ||
1430 | if opt.network_only and opt.local_only: | ||
1431 | self.OptionParser.error("cannot combine -n and -l") | ||
1432 | if opt.manifest_name and opt.smart_sync: | ||
1433 | self.OptionParser.error("cannot combine -m and -s") | ||
1434 | if opt.manifest_name and opt.smart_tag: | ||
1435 | self.OptionParser.error("cannot combine -m and -t") | ||
1436 | if opt.manifest_server_username or opt.manifest_server_password: | ||
1437 | if not (opt.smart_sync or opt.smart_tag): | ||
1438 | self.OptionParser.error( | ||
1439 | "-u and -p may only be combined with -s or -t" | ||
1440 | ) | ||
1441 | if None in [ | ||
1442 | opt.manifest_server_username, | ||
1443 | opt.manifest_server_password, | ||
1444 | ]: | ||
1445 | self.OptionParser.error("both -u and -p must be given") | ||
1446 | |||
1447 | if opt.prune is None: | ||
1448 | opt.prune = True | ||
1449 | |||
1450 | if opt.auto_gc is None and _AUTO_GC: | ||
1451 | print( | ||
1452 | f"Will run `git gc --auto` because {_REPO_AUTO_GC} is set.", | ||
1453 | f"{_REPO_AUTO_GC} is deprecated and will be removed in a ", | ||
1454 | "future release. Use `--auto-gc` instead.", | ||
1455 | file=sys.stderr, | ||
1456 | ) | ||
1457 | opt.auto_gc = True | ||
1458 | |||
1459 | def _ValidateOptionsWithManifest(self, opt, mp): | ||
1460 | """Like ValidateOptions, but after we've updated the manifest. | ||
1461 | |||
1462 | Needed to handle sync-xxx option defaults in the manifest. | ||
1463 | |||
1464 | Args: | ||
1465 | opt: The options to process. | ||
1466 | mp: The manifest project to pull defaults from. | ||
1467 | """ | ||
1468 | if not opt.jobs: | ||
1469 | # If the user hasn't made a choice, use the manifest value. | ||
1470 | opt.jobs = mp.manifest.default.sync_j | ||
1471 | if opt.jobs: | ||
1472 | # If --jobs has a non-default value, propagate it as the default for | ||
1473 | # --jobs-xxx flags too. | ||
1474 | if not opt.jobs_network: | ||
1475 | opt.jobs_network = opt.jobs | ||
1476 | if not opt.jobs_checkout: | ||
1477 | opt.jobs_checkout = opt.jobs | ||
1478 | else: | ||
1479 | # Neither user nor manifest have made a choice, so setup defaults. | ||
1480 | if not opt.jobs_network: | ||
1481 | opt.jobs_network = 1 | ||
1482 | if not opt.jobs_checkout: | ||
1483 | opt.jobs_checkout = DEFAULT_LOCAL_JOBS | ||
1484 | opt.jobs = os.cpu_count() | ||
1485 | |||
1486 | # Try to stay under user rlimit settings. | ||
1487 | # | ||
1488 | # Since each worker requires at 3 file descriptors to run `git fetch`, | ||
1489 | # use that to scale down the number of jobs. Unfortunately there isn't | ||
1490 | # an easy way to determine this reliably as systems change, but it was | ||
1491 | # last measured by hand in 2011. | ||
1492 | soft_limit, _ = _rlimit_nofile() | ||
1493 | jobs_soft_limit = max(1, (soft_limit - 5) // 3) | ||
1494 | opt.jobs = min(opt.jobs, jobs_soft_limit) | ||
1495 | opt.jobs_network = min(opt.jobs_network, jobs_soft_limit) | ||
1496 | opt.jobs_checkout = min(opt.jobs_checkout, jobs_soft_limit) | ||
1497 | |||
1498 | def Execute(self, opt, args): | ||
1499 | manifest = self.outer_manifest | ||
1500 | if not opt.outer_manifest: | ||
1501 | manifest = self.manifest | ||
1502 | |||
1503 | if opt.manifest_name: | ||
1504 | manifest.Override(opt.manifest_name) | ||
1505 | |||
1506 | manifest_name = opt.manifest_name | ||
1507 | smart_sync_manifest_path = os.path.join( | ||
1508 | manifest.manifestProject.worktree, "smart_sync_override.xml" | ||
1509 | ) | ||
915 | 1510 | ||
916 | Args: | 1511 | if opt.clone_bundle is None: |
917 | opt: Program options returned from optparse. See _Options(). | 1512 | opt.clone_bundle = manifest.CloneBundle |
918 | manifest: The manifest to use. | ||
919 | 1513 | ||
920 | Returns: | 1514 | if opt.smart_sync or opt.smart_tag: |
921 | 0: success | 1515 | manifest_name = self._SmartSyncSetup( |
922 | 1: failure | 1516 | opt, smart_sync_manifest_path, manifest |
923 | """ | 1517 | ) |
924 | new_project_paths = [] | ||
925 | for project in self.GetProjects(None, missing_ok=True, manifest=manifest, | ||
926 | all_manifests=False): | ||
927 | if project.relpath: | ||
928 | new_project_paths.append(project.relpath) | ||
929 | file_name = 'project.list' | ||
930 | file_path = os.path.join(manifest.subdir, file_name) | ||
931 | old_project_paths = [] | ||
932 | |||
933 | if os.path.exists(file_path): | ||
934 | with open(file_path, 'r') as fd: | ||
935 | old_project_paths = fd.read().split('\n') | ||
936 | # In reversed order, so subfolders are deleted before parent folder. | ||
937 | for path in sorted(old_project_paths, reverse=True): | ||
938 | if not path: | ||
939 | continue | ||
940 | if path not in new_project_paths: | ||
941 | # If the path has already been deleted, we don't need to do it | ||
942 | gitdir = os.path.join(manifest.topdir, path, '.git') | ||
943 | if os.path.exists(gitdir): | ||
944 | project = Project( | ||
945 | manifest=manifest, | ||
946 | name=path, | ||
947 | remote=RemoteSpec('origin'), | ||
948 | gitdir=gitdir, | ||
949 | objdir=gitdir, | ||
950 | use_git_worktrees=os.path.isfile(gitdir), | ||
951 | worktree=os.path.join(manifest.topdir, path), | ||
952 | relpath=path, | ||
953 | revisionExpr='HEAD', | ||
954 | revisionId=None, | ||
955 | groups=None) | ||
956 | if not project.DeleteWorktree( | ||
957 | quiet=opt.quiet, | ||
958 | force=opt.force_remove_dirty): | ||
959 | return 1 | ||
960 | |||
961 | new_project_paths.sort() | ||
962 | with open(file_path, 'w') as fd: | ||
963 | fd.write('\n'.join(new_project_paths)) | ||
964 | fd.write('\n') | ||
965 | return 0 | ||
966 | |||
967 | def UpdateCopyLinkfileList(self, manifest): | ||
968 | """Save all dests of copyfile and linkfile, and update them if needed. | ||
969 | |||
970 | Returns: | ||
971 | Whether update was successful. | ||
972 | """ | ||
973 | new_paths = {} | ||
974 | new_linkfile_paths = [] | ||
975 | new_copyfile_paths = [] | ||
976 | for project in self.GetProjects(None, missing_ok=True, | ||
977 | manifest=manifest, all_manifests=False): | ||
978 | new_linkfile_paths.extend(x.dest for x in project.linkfiles) | ||
979 | new_copyfile_paths.extend(x.dest for x in project.copyfiles) | ||
980 | |||
981 | new_paths = { | ||
982 | 'linkfile': new_linkfile_paths, | ||
983 | 'copyfile': new_copyfile_paths, | ||
984 | } | ||
985 | |||
986 | copylinkfile_name = 'copy-link-files.json' | ||
987 | copylinkfile_path = os.path.join(manifest.subdir, copylinkfile_name) | ||
988 | old_copylinkfile_paths = {} | ||
989 | |||
990 | if os.path.exists(copylinkfile_path): | ||
991 | with open(copylinkfile_path, 'rb') as fp: | ||
992 | try: | ||
993 | old_copylinkfile_paths = json.load(fp) | ||
994 | except Exception: | ||
995 | print('error: %s is not a json formatted file.' % | ||
996 | copylinkfile_path, file=sys.stderr) | ||
997 | platform_utils.remove(copylinkfile_path) | ||
998 | return False | ||
999 | |||
1000 | need_remove_files = [] | ||
1001 | need_remove_files.extend( | ||
1002 | set(old_copylinkfile_paths.get('linkfile', [])) - | ||
1003 | set(new_linkfile_paths)) | ||
1004 | need_remove_files.extend( | ||
1005 | set(old_copylinkfile_paths.get('copyfile', [])) - | ||
1006 | set(new_copyfile_paths)) | ||
1007 | |||
1008 | for need_remove_file in need_remove_files: | ||
1009 | # Try to remove the updated copyfile or linkfile. | ||
1010 | # So, if the file is not exist, nothing need to do. | ||
1011 | platform_utils.remove(need_remove_file, missing_ok=True) | ||
1012 | |||
1013 | # Create copy-link-files.json, save dest path of "copyfile" and "linkfile". | ||
1014 | with open(copylinkfile_path, 'w', encoding='utf-8') as fp: | ||
1015 | json.dump(new_paths, fp) | ||
1016 | return True | ||
1017 | |||
1018 | def _SmartSyncSetup(self, opt, smart_sync_manifest_path, manifest): | ||
1019 | if not manifest.manifest_server: | ||
1020 | print('error: cannot smart sync: no manifest server defined in ' | ||
1021 | 'manifest', file=sys.stderr) | ||
1022 | sys.exit(1) | ||
1023 | |||
1024 | manifest_server = manifest.manifest_server | ||
1025 | if not opt.quiet: | ||
1026 | print('Using manifest server %s' % manifest_server) | ||
1027 | |||
1028 | if '@' not in manifest_server: | ||
1029 | username = None | ||
1030 | password = None | ||
1031 | if opt.manifest_server_username and opt.manifest_server_password: | ||
1032 | username = opt.manifest_server_username | ||
1033 | password = opt.manifest_server_password | ||
1034 | else: | ||
1035 | try: | ||
1036 | info = netrc.netrc() | ||
1037 | except IOError: | ||
1038 | # .netrc file does not exist or could not be opened | ||
1039 | pass | ||
1040 | else: | 1518 | else: |
1041 | try: | 1519 | if os.path.isfile(smart_sync_manifest_path): |
1042 | parse_result = urllib.parse.urlparse(manifest_server) | 1520 | try: |
1043 | if parse_result.hostname: | 1521 | platform_utils.remove(smart_sync_manifest_path) |
1044 | auth = info.authenticators(parse_result.hostname) | 1522 | except OSError as e: |
1045 | if auth: | 1523 | print( |
1046 | username, _account, password = auth | 1524 | "error: failed to remove existing smart sync override " |
1047 | else: | 1525 | "manifest: %s" % e, |
1048 | print('No credentials found for %s in .netrc' | 1526 | file=sys.stderr, |
1049 | % parse_result.hostname, file=sys.stderr) | 1527 | ) |
1050 | except netrc.NetrcParseError as e: | 1528 | |
1051 | print('Error parsing .netrc file: %s' % e, file=sys.stderr) | 1529 | err_event = multiprocessing.Event() |
1052 | 1530 | ||
1053 | if (username and password): | 1531 | rp = manifest.repoProject |
1054 | manifest_server = manifest_server.replace('://', '://%s:%s@' % | 1532 | rp.PreSync() |
1055 | (username, password), | 1533 | cb = rp.CurrentBranch |
1056 | 1) | 1534 | if cb: |
1057 | 1535 | base = rp.GetBranch(cb).merge | |
1058 | transport = PersistentTransport(manifest_server) | 1536 | if not base or not base.startswith("refs/heads/"): |
1059 | if manifest_server.startswith('persistent-'): | 1537 | print( |
1060 | manifest_server = manifest_server[len('persistent-'):] | 1538 | "warning: repo is not tracking a remote branch, so it will " |
1061 | 1539 | "not receive updates; run `repo init --repo-rev=stable` to " | |
1062 | try: | 1540 | "fix.", |
1063 | server = xmlrpc.client.Server(manifest_server, transport=transport) | 1541 | file=sys.stderr, |
1064 | if opt.smart_sync: | 1542 | ) |
1065 | branch = self._GetBranch(manifest.manifestProject) | 1543 | |
1066 | 1544 | for m in self.ManifestList(opt): | |
1067 | if 'SYNC_TARGET' in os.environ: | 1545 | if not m.manifestProject.standalone_manifest_url: |
1068 | target = os.environ['SYNC_TARGET'] | 1546 | m.manifestProject.PreSync() |
1069 | [success, manifest_str] = server.GetApprovedManifest(branch, target) | 1547 | |
1070 | elif ('TARGET_PRODUCT' in os.environ and | 1548 | if opt.repo_upgraded: |
1071 | 'TARGET_BUILD_VARIANT' in os.environ): | 1549 | _PostRepoUpgrade(manifest, quiet=opt.quiet) |
1072 | target = '%s-%s' % (os.environ['TARGET_PRODUCT'], | 1550 | |
1073 | os.environ['TARGET_BUILD_VARIANT']) | 1551 | mp = manifest.manifestProject |
1074 | [success, manifest_str] = server.GetApprovedManifest(branch, target) | 1552 | if opt.mp_update: |
1553 | self._UpdateAllManifestProjects(opt, mp, manifest_name) | ||
1075 | else: | 1554 | else: |
1076 | [success, manifest_str] = server.GetApprovedManifest(branch) | 1555 | print("Skipping update of local manifest project.") |
1077 | else: | ||
1078 | assert(opt.smart_tag) | ||
1079 | [success, manifest_str] = server.GetManifest(opt.smart_tag) | ||
1080 | 1556 | ||
1081 | if success: | 1557 | # Now that the manifests are up-to-date, setup options whose defaults |
1082 | manifest_name = os.path.basename(smart_sync_manifest_path) | 1558 | # might be in the manifest. |
1083 | try: | 1559 | self._ValidateOptionsWithManifest(opt, mp) |
1084 | with open(smart_sync_manifest_path, 'w') as f: | 1560 | |
1085 | f.write(manifest_str) | 1561 | superproject_logging_data = {} |
1086 | except IOError as e: | 1562 | self._UpdateProjectsRevisionId( |
1087 | print('error: cannot write manifest to %s:\n%s' | 1563 | opt, args, superproject_logging_data, manifest |
1088 | % (smart_sync_manifest_path, e), | ||
1089 | file=sys.stderr) | ||
1090 | sys.exit(1) | ||
1091 | self._ReloadManifest(manifest_name, manifest) | ||
1092 | else: | ||
1093 | print('error: manifest server RPC call failed: %s' % | ||
1094 | manifest_str, file=sys.stderr) | ||
1095 | sys.exit(1) | ||
1096 | except (socket.error, IOError, xmlrpc.client.Fault) as e: | ||
1097 | print('error: cannot connect to manifest server %s:\n%s' | ||
1098 | % (manifest.manifest_server, e), file=sys.stderr) | ||
1099 | sys.exit(1) | ||
1100 | except xmlrpc.client.ProtocolError as e: | ||
1101 | print('error: cannot connect to manifest server %s:\n%d %s' | ||
1102 | % (manifest.manifest_server, e.errcode, e.errmsg), | ||
1103 | file=sys.stderr) | ||
1104 | sys.exit(1) | ||
1105 | |||
1106 | return manifest_name | ||
1107 | |||
1108 | def _UpdateAllManifestProjects(self, opt, mp, manifest_name): | ||
1109 | """Fetch & update the local manifest project. | ||
1110 | |||
1111 | After syncing the manifest project, if the manifest has any sub manifests, | ||
1112 | those are recursively processed. | ||
1113 | |||
1114 | Args: | ||
1115 | opt: Program options returned from optparse. See _Options(). | ||
1116 | mp: the manifestProject to query. | ||
1117 | manifest_name: Manifest file to be reloaded. | ||
1118 | """ | ||
1119 | if not mp.standalone_manifest_url: | ||
1120 | self._UpdateManifestProject(opt, mp, manifest_name) | ||
1121 | |||
1122 | if mp.manifest.submanifests: | ||
1123 | for submanifest in mp.manifest.submanifests.values(): | ||
1124 | child = submanifest.repo_client.manifest | ||
1125 | child.manifestProject.SyncWithPossibleInit( | ||
1126 | submanifest, | ||
1127 | current_branch_only=self._GetCurrentBranchOnly(opt, child), | ||
1128 | verbose=opt.verbose, | ||
1129 | tags=opt.tags, | ||
1130 | git_event_log=self.git_event_log, | ||
1131 | ) | 1564 | ) |
1132 | self._UpdateAllManifestProjects(opt, child.manifestProject, None) | ||
1133 | 1565 | ||
1134 | def _UpdateManifestProject(self, opt, mp, manifest_name): | 1566 | if self.gitc_manifest: |
1135 | """Fetch & update the local manifest project. | 1567 | gitc_manifest_projects = self.GetProjects(args, missing_ok=True) |
1568 | gitc_projects = [] | ||
1569 | opened_projects = [] | ||
1570 | for project in gitc_manifest_projects: | ||
1571 | if ( | ||
1572 | project.relpath in self.gitc_manifest.paths | ||
1573 | and self.gitc_manifest.paths[project.relpath].old_revision | ||
1574 | ): | ||
1575 | opened_projects.append(project.relpath) | ||
1576 | else: | ||
1577 | gitc_projects.append(project.relpath) | ||
1578 | |||
1579 | if not args: | ||
1580 | gitc_projects = None | ||
1581 | |||
1582 | if gitc_projects != [] and not opt.local_only: | ||
1583 | print( | ||
1584 | "Updating GITC client: %s" | ||
1585 | % self.gitc_manifest.gitc_client_name | ||
1586 | ) | ||
1587 | manifest = GitcManifest( | ||
1588 | self.repodir, self.gitc_manifest.gitc_client_name | ||
1589 | ) | ||
1590 | if manifest_name: | ||
1591 | manifest.Override(manifest_name) | ||
1592 | else: | ||
1593 | manifest.Override(manifest.manifestFile) | ||
1594 | gitc_utils.generate_gitc_manifest( | ||
1595 | self.gitc_manifest, manifest, gitc_projects | ||
1596 | ) | ||
1597 | print("GITC client successfully synced.") | ||
1598 | |||
1599 | # The opened projects need to be synced as normal, therefore we | ||
1600 | # generate a new args list to represent the opened projects. | ||
1601 | # TODO: make this more reliable -- if there's a project name/path | ||
1602 | # overlap, this may choose the wrong project. | ||
1603 | args = [ | ||
1604 | os.path.relpath(manifest.paths[path].worktree, os.getcwd()) | ||
1605 | for path in opened_projects | ||
1606 | ] | ||
1607 | if not args: | ||
1608 | return | ||
1609 | |||
1610 | all_projects = self.GetProjects( | ||
1611 | args, | ||
1612 | missing_ok=True, | ||
1613 | submodules_ok=opt.fetch_submodules, | ||
1614 | manifest=manifest, | ||
1615 | all_manifests=not opt.this_manifest_only, | ||
1616 | ) | ||
1136 | 1617 | ||
1137 | Args: | 1618 | err_network_sync = False |
1138 | opt: Program options returned from optparse. See _Options(). | 1619 | err_update_projects = False |
1139 | mp: the manifestProject to query. | 1620 | err_update_linkfiles = False |
1140 | manifest_name: Manifest file to be reloaded. | 1621 | |
1141 | """ | 1622 | self._fetch_times = _FetchTimes(manifest) |
1142 | if not opt.local_only: | 1623 | if not opt.local_only: |
1143 | start = time.time() | 1624 | with multiprocessing.Manager() as manager: |
1144 | success = mp.Sync_NetworkHalf(quiet=opt.quiet, verbose=opt.verbose, | 1625 | with ssh.ProxyManager(manager) as ssh_proxy: |
1145 | current_branch_only=self._GetCurrentBranchOnly(opt, mp.manifest), | 1626 | # Initialize the socket dir once in the parent. |
1146 | force_sync=opt.force_sync, | 1627 | ssh_proxy.sock() |
1147 | tags=opt.tags, | 1628 | result = self._FetchMain( |
1148 | optimized_fetch=opt.optimized_fetch, | 1629 | opt, args, all_projects, err_event, ssh_proxy, manifest |
1149 | retry_fetches=opt.retry_fetches, | 1630 | ) |
1150 | submodules=mp.manifest.HasSubmodules, | 1631 | all_projects = result.all_projects |
1151 | clone_filter=mp.manifest.CloneFilter, | 1632 | |
1152 | partial_clone_exclude=mp.manifest.PartialCloneExclude) | 1633 | if opt.network_only: |
1153 | finish = time.time() | 1634 | return |
1154 | self.event_log.AddSync(mp, event_log.TASK_SYNC_NETWORK, | 1635 | |
1155 | start, finish, success) | 1636 | # If we saw an error, exit with code 1 so that other scripts can |
1156 | 1637 | # check. | |
1157 | if mp.HasChanges: | 1638 | if err_event.is_set(): |
1158 | syncbuf = SyncBuffer(mp.config) | 1639 | err_network_sync = True |
1159 | start = time.time() | 1640 | if opt.fail_fast: |
1160 | mp.Sync_LocalHalf(syncbuf, submodules=mp.manifest.HasSubmodules) | 1641 | print( |
1161 | clean = syncbuf.Finish() | 1642 | "\nerror: Exited sync due to fetch errors.\n" |
1162 | self.event_log.AddSync(mp, event_log.TASK_SYNC_LOCAL, | 1643 | "Local checkouts *not* updated. Resolve network issues " |
1163 | start, time.time(), clean) | 1644 | "& retry.\n" |
1164 | if not clean: | 1645 | "`repo sync -l` will update some local checkouts.", |
1165 | sys.exit(1) | 1646 | file=sys.stderr, |
1166 | self._ReloadManifest(manifest_name, mp.manifest) | 1647 | ) |
1167 | 1648 | sys.exit(1) | |
1168 | def ValidateOptions(self, opt, args): | 1649 | |
1169 | if opt.force_broken: | 1650 | for m in self.ManifestList(opt): |
1170 | print('warning: -f/--force-broken is now the default behavior, and the ' | 1651 | if m.IsMirror or m.IsArchive: |
1171 | 'options are deprecated', file=sys.stderr) | 1652 | # Bail out now, we have no working tree. |
1172 | if opt.network_only and opt.detach_head: | 1653 | continue |
1173 | self.OptionParser.error('cannot combine -n and -d') | 1654 | |
1174 | if opt.network_only and opt.local_only: | 1655 | if self.UpdateProjectList(opt, m): |
1175 | self.OptionParser.error('cannot combine -n and -l') | 1656 | err_event.set() |
1176 | if opt.manifest_name and opt.smart_sync: | 1657 | err_update_projects = True |
1177 | self.OptionParser.error('cannot combine -m and -s') | 1658 | if opt.fail_fast: |
1178 | if opt.manifest_name and opt.smart_tag: | 1659 | print( |
1179 | self.OptionParser.error('cannot combine -m and -t') | 1660 | "\nerror: Local checkouts *not* updated.", |
1180 | if opt.manifest_server_username or opt.manifest_server_password: | 1661 | file=sys.stderr, |
1181 | if not (opt.smart_sync or opt.smart_tag): | 1662 | ) |
1182 | self.OptionParser.error('-u and -p may only be combined with -s or -t') | 1663 | sys.exit(1) |
1183 | if None in [opt.manifest_server_username, opt.manifest_server_password]: | 1664 | |
1184 | self.OptionParser.error('both -u and -p must be given') | 1665 | err_update_linkfiles = not self.UpdateCopyLinkfileList(m) |
1185 | 1666 | if err_update_linkfiles: | |
1186 | if opt.prune is None: | 1667 | err_event.set() |
1187 | opt.prune = True | 1668 | if opt.fail_fast: |
1188 | 1669 | print( | |
1189 | if opt.auto_gc is None and _AUTO_GC: | 1670 | "\nerror: Local update copyfile or linkfile failed.", |
1190 | print(f"Will run `git gc --auto` because {_REPO_AUTO_GC} is set.", | 1671 | file=sys.stderr, |
1191 | f'{_REPO_AUTO_GC} is deprecated and will be removed in a future', | 1672 | ) |
1192 | 'release. Use `--auto-gc` instead.', file=sys.stderr) | 1673 | sys.exit(1) |
1193 | opt.auto_gc = True | 1674 | |
1194 | 1675 | err_results = [] | |
1195 | def _ValidateOptionsWithManifest(self, opt, mp): | 1676 | # NB: We don't exit here because this is the last step. |
1196 | """Like ValidateOptions, but after we've updated the manifest. | 1677 | err_checkout = not self._Checkout(all_projects, opt, err_results) |
1197 | 1678 | if err_checkout: | |
1198 | Needed to handle sync-xxx option defaults in the manifest. | 1679 | err_event.set() |
1199 | 1680 | ||
1200 | Args: | 1681 | printed_notices = set() |
1201 | opt: The options to process. | 1682 | # If there's a notice that's supposed to print at the end of the sync, |
1202 | mp: The manifest project to pull defaults from. | 1683 | # print it now... But avoid printing duplicate messages, and preserve |
1203 | """ | 1684 | # order. |
1204 | if not opt.jobs: | 1685 | for m in sorted(self.ManifestList(opt), key=lambda x: x.path_prefix): |
1205 | # If the user hasn't made a choice, use the manifest value. | 1686 | if m.notice and m.notice not in printed_notices: |
1206 | opt.jobs = mp.manifest.default.sync_j | 1687 | print(m.notice) |
1207 | if opt.jobs: | 1688 | printed_notices.add(m.notice) |
1208 | # If --jobs has a non-default value, propagate it as the default for | 1689 | |
1209 | # --jobs-xxx flags too. | 1690 | # If we saw an error, exit with code 1 so that other scripts can check. |
1210 | if not opt.jobs_network: | 1691 | if err_event.is_set(): |
1211 | opt.jobs_network = opt.jobs | 1692 | print("\nerror: Unable to fully sync the tree.", file=sys.stderr) |
1212 | if not opt.jobs_checkout: | 1693 | if err_network_sync: |
1213 | opt.jobs_checkout = opt.jobs | 1694 | print( |
1214 | else: | 1695 | "error: Downloading network changes failed.", |
1215 | # Neither user nor manifest have made a choice, so setup defaults. | 1696 | file=sys.stderr, |
1216 | if not opt.jobs_network: | 1697 | ) |
1217 | opt.jobs_network = 1 | 1698 | if err_update_projects: |
1218 | if not opt.jobs_checkout: | 1699 | print( |
1219 | opt.jobs_checkout = DEFAULT_LOCAL_JOBS | 1700 | "error: Updating local project lists failed.", |
1220 | opt.jobs = os.cpu_count() | 1701 | file=sys.stderr, |
1221 | 1702 | ) | |
1222 | # Try to stay under user rlimit settings. | 1703 | if err_update_linkfiles: |
1223 | # | 1704 | print( |
1224 | # Since each worker requires at 3 file descriptors to run `git fetch`, use | 1705 | "error: Updating copyfiles or linkfiles failed.", |
1225 | # that to scale down the number of jobs. Unfortunately there isn't an easy | 1706 | file=sys.stderr, |
1226 | # way to determine this reliably as systems change, but it was last measured | 1707 | ) |
1227 | # by hand in 2011. | 1708 | if err_checkout: |
1228 | soft_limit, _ = _rlimit_nofile() | 1709 | print( |
1229 | jobs_soft_limit = max(1, (soft_limit - 5) // 3) | 1710 | "error: Checking out local projects failed.", |
1230 | opt.jobs = min(opt.jobs, jobs_soft_limit) | 1711 | file=sys.stderr, |
1231 | opt.jobs_network = min(opt.jobs_network, jobs_soft_limit) | 1712 | ) |
1232 | opt.jobs_checkout = min(opt.jobs_checkout, jobs_soft_limit) | 1713 | if err_results: |
1233 | 1714 | print( | |
1234 | def Execute(self, opt, args): | 1715 | "Failing repos:\n%s" % "\n".join(err_results), |
1235 | manifest = self.outer_manifest | 1716 | file=sys.stderr, |
1236 | if not opt.outer_manifest: | 1717 | ) |
1237 | manifest = self.manifest | 1718 | print( |
1238 | 1719 | 'Try re-running with "-j1 --fail-fast" to exit at the first ' | |
1239 | if opt.manifest_name: | 1720 | "error.", |
1240 | manifest.Override(opt.manifest_name) | 1721 | file=sys.stderr, |
1241 | 1722 | ) | |
1242 | manifest_name = opt.manifest_name | 1723 | sys.exit(1) |
1243 | smart_sync_manifest_path = os.path.join( | 1724 | |
1244 | manifest.manifestProject.worktree, 'smart_sync_override.xml') | 1725 | # Log the previous sync analysis state from the config. |
1245 | 1726 | self.git_event_log.LogDataConfigEvents( | |
1246 | if opt.clone_bundle is None: | 1727 | mp.config.GetSyncAnalysisStateData(), "previous_sync_state" |
1247 | opt.clone_bundle = manifest.CloneBundle | 1728 | ) |
1248 | |||
1249 | if opt.smart_sync or opt.smart_tag: | ||
1250 | manifest_name = self._SmartSyncSetup(opt, smart_sync_manifest_path, manifest) | ||
1251 | else: | ||
1252 | if os.path.isfile(smart_sync_manifest_path): | ||
1253 | try: | ||
1254 | platform_utils.remove(smart_sync_manifest_path) | ||
1255 | except OSError as e: | ||
1256 | print('error: failed to remove existing smart sync override manifest: %s' % | ||
1257 | e, file=sys.stderr) | ||
1258 | |||
1259 | err_event = multiprocessing.Event() | ||
1260 | |||
1261 | rp = manifest.repoProject | ||
1262 | rp.PreSync() | ||
1263 | cb = rp.CurrentBranch | ||
1264 | if cb: | ||
1265 | base = rp.GetBranch(cb).merge | ||
1266 | if not base or not base.startswith('refs/heads/'): | ||
1267 | print('warning: repo is not tracking a remote branch, so it will not ' | ||
1268 | 'receive updates; run `repo init --repo-rev=stable` to fix.', | ||
1269 | file=sys.stderr) | ||
1270 | |||
1271 | for m in self.ManifestList(opt): | ||
1272 | if not m.manifestProject.standalone_manifest_url: | ||
1273 | m.manifestProject.PreSync() | ||
1274 | |||
1275 | if opt.repo_upgraded: | ||
1276 | _PostRepoUpgrade(manifest, quiet=opt.quiet) | ||
1277 | |||
1278 | mp = manifest.manifestProject | ||
1279 | if opt.mp_update: | ||
1280 | self._UpdateAllManifestProjects(opt, mp, manifest_name) | ||
1281 | else: | ||
1282 | print('Skipping update of local manifest project.') | ||
1283 | |||
1284 | # Now that the manifests are up-to-date, setup options whose defaults might | ||
1285 | # be in the manifest. | ||
1286 | self._ValidateOptionsWithManifest(opt, mp) | ||
1287 | |||
1288 | superproject_logging_data = {} | ||
1289 | self._UpdateProjectsRevisionId(opt, args, superproject_logging_data, | ||
1290 | manifest) | ||
1291 | |||
1292 | if self.gitc_manifest: | ||
1293 | gitc_manifest_projects = self.GetProjects(args, missing_ok=True) | ||
1294 | gitc_projects = [] | ||
1295 | opened_projects = [] | ||
1296 | for project in gitc_manifest_projects: | ||
1297 | if project.relpath in self.gitc_manifest.paths and \ | ||
1298 | self.gitc_manifest.paths[project.relpath].old_revision: | ||
1299 | opened_projects.append(project.relpath) | ||
1300 | else: | ||
1301 | gitc_projects.append(project.relpath) | ||
1302 | 1729 | ||
1303 | if not args: | 1730 | # Update and log with the new sync analysis state. |
1304 | gitc_projects = None | 1731 | mp.config.UpdateSyncAnalysisState(opt, superproject_logging_data) |
1732 | self.git_event_log.LogDataConfigEvents( | ||
1733 | mp.config.GetSyncAnalysisStateData(), "current_sync_state" | ||
1734 | ) | ||
1305 | 1735 | ||
1306 | if gitc_projects != [] and not opt.local_only: | 1736 | if not opt.quiet: |
1307 | print('Updating GITC client: %s' % self.gitc_manifest.gitc_client_name) | 1737 | print("repo sync has finished successfully.") |
1308 | manifest = GitcManifest(self.repodir, self.gitc_manifest.gitc_client_name) | ||
1309 | if manifest_name: | ||
1310 | manifest.Override(manifest_name) | ||
1311 | else: | ||
1312 | manifest.Override(manifest.manifestFile) | ||
1313 | gitc_utils.generate_gitc_manifest(self.gitc_manifest, | ||
1314 | manifest, | ||
1315 | gitc_projects) | ||
1316 | print('GITC client successfully synced.') | ||
1317 | |||
1318 | # The opened projects need to be synced as normal, therefore we | ||
1319 | # generate a new args list to represent the opened projects. | ||
1320 | # TODO: make this more reliable -- if there's a project name/path overlap, | ||
1321 | # this may choose the wrong project. | ||
1322 | args = [os.path.relpath(manifest.paths[path].worktree, os.getcwd()) | ||
1323 | for path in opened_projects] | ||
1324 | if not args: | ||
1325 | return | ||
1326 | |||
1327 | all_projects = self.GetProjects(args, | ||
1328 | missing_ok=True, | ||
1329 | submodules_ok=opt.fetch_submodules, | ||
1330 | manifest=manifest, | ||
1331 | all_manifests=not opt.this_manifest_only) | ||
1332 | |||
1333 | err_network_sync = False | ||
1334 | err_update_projects = False | ||
1335 | err_update_linkfiles = False | ||
1336 | |||
1337 | self._fetch_times = _FetchTimes(manifest) | ||
1338 | if not opt.local_only: | ||
1339 | with multiprocessing.Manager() as manager: | ||
1340 | with ssh.ProxyManager(manager) as ssh_proxy: | ||
1341 | # Initialize the socket dir once in the parent. | ||
1342 | ssh_proxy.sock() | ||
1343 | result = self._FetchMain(opt, args, all_projects, err_event, | ||
1344 | ssh_proxy, manifest) | ||
1345 | all_projects = result.all_projects | ||
1346 | |||
1347 | if opt.network_only: | ||
1348 | return | ||
1349 | |||
1350 | # If we saw an error, exit with code 1 so that other scripts can check. | ||
1351 | if err_event.is_set(): | ||
1352 | err_network_sync = True | ||
1353 | if opt.fail_fast: | ||
1354 | print('\nerror: Exited sync due to fetch errors.\n' | ||
1355 | 'Local checkouts *not* updated. Resolve network issues & ' | ||
1356 | 'retry.\n' | ||
1357 | '`repo sync -l` will update some local checkouts.', | ||
1358 | file=sys.stderr) | ||
1359 | sys.exit(1) | ||
1360 | |||
1361 | for m in self.ManifestList(opt): | ||
1362 | if m.IsMirror or m.IsArchive: | ||
1363 | # bail out now, we have no working tree | ||
1364 | continue | ||
1365 | |||
1366 | if self.UpdateProjectList(opt, m): | ||
1367 | err_event.set() | ||
1368 | err_update_projects = True | ||
1369 | if opt.fail_fast: | ||
1370 | print('\nerror: Local checkouts *not* updated.', file=sys.stderr) | ||
1371 | sys.exit(1) | ||
1372 | |||
1373 | err_update_linkfiles = not self.UpdateCopyLinkfileList(m) | ||
1374 | if err_update_linkfiles: | ||
1375 | err_event.set() | ||
1376 | if opt.fail_fast: | ||
1377 | print('\nerror: Local update copyfile or linkfile failed.', file=sys.stderr) | ||
1378 | sys.exit(1) | ||
1379 | |||
1380 | err_results = [] | ||
1381 | # NB: We don't exit here because this is the last step. | ||
1382 | err_checkout = not self._Checkout(all_projects, opt, err_results) | ||
1383 | if err_checkout: | ||
1384 | err_event.set() | ||
1385 | |||
1386 | printed_notices = set() | ||
1387 | # If there's a notice that's supposed to print at the end of the sync, | ||
1388 | # print it now... But avoid printing duplicate messages, and preserve | ||
1389 | # order. | ||
1390 | for m in sorted(self.ManifestList(opt), key=lambda x: x.path_prefix): | ||
1391 | if m.notice and m.notice not in printed_notices: | ||
1392 | print(m.notice) | ||
1393 | printed_notices.add(m.notice) | ||
1394 | |||
1395 | # If we saw an error, exit with code 1 so that other scripts can check. | ||
1396 | if err_event.is_set(): | ||
1397 | print('\nerror: Unable to fully sync the tree.', file=sys.stderr) | ||
1398 | if err_network_sync: | ||
1399 | print('error: Downloading network changes failed.', file=sys.stderr) | ||
1400 | if err_update_projects: | ||
1401 | print('error: Updating local project lists failed.', file=sys.stderr) | ||
1402 | if err_update_linkfiles: | ||
1403 | print('error: Updating copyfiles or linkfiles failed.', file=sys.stderr) | ||
1404 | if err_checkout: | ||
1405 | print('error: Checking out local projects failed.', file=sys.stderr) | ||
1406 | if err_results: | ||
1407 | print('Failing repos:\n%s' % '\n'.join(err_results), file=sys.stderr) | ||
1408 | print('Try re-running with "-j1 --fail-fast" to exit at the first error.', | ||
1409 | file=sys.stderr) | ||
1410 | sys.exit(1) | ||
1411 | |||
1412 | # Log the previous sync analysis state from the config. | ||
1413 | self.git_event_log.LogDataConfigEvents(mp.config.GetSyncAnalysisStateData(), | ||
1414 | 'previous_sync_state') | ||
1415 | |||
1416 | # Update and log with the new sync analysis state. | ||
1417 | mp.config.UpdateSyncAnalysisState(opt, superproject_logging_data) | ||
1418 | self.git_event_log.LogDataConfigEvents(mp.config.GetSyncAnalysisStateData(), | ||
1419 | 'current_sync_state') | ||
1420 | |||
1421 | if not opt.quiet: | ||
1422 | print('repo sync has finished successfully.') | ||
1423 | 1738 | ||
1424 | 1739 | ||
1425 | def _PostRepoUpgrade(manifest, quiet=False): | 1740 | def _PostRepoUpgrade(manifest, quiet=False): |
1426 | # Link the docs for the internal .repo/ layout for people | 1741 | # Link the docs for the internal .repo/ layout for people. |
1427 | link = os.path.join(manifest.repodir, 'internal-fs-layout.md') | 1742 | link = os.path.join(manifest.repodir, "internal-fs-layout.md") |
1428 | if not platform_utils.islink(link): | 1743 | if not platform_utils.islink(link): |
1429 | target = os.path.join('repo', 'docs', 'internal-fs-layout.md') | 1744 | target = os.path.join("repo", "docs", "internal-fs-layout.md") |
1430 | try: | 1745 | try: |
1431 | platform_utils.symlink(target, link) | 1746 | platform_utils.symlink(target, link) |
1432 | except Exception: | 1747 | except Exception: |
1433 | pass | 1748 | pass |
1434 | 1749 | ||
1435 | wrapper = Wrapper() | 1750 | wrapper = Wrapper() |
1436 | if wrapper.NeedSetupGnuPG(): | 1751 | if wrapper.NeedSetupGnuPG(): |
1437 | wrapper.SetupGnuPG(quiet) | 1752 | wrapper.SetupGnuPG(quiet) |
1438 | for project in manifest.projects: | 1753 | for project in manifest.projects: |
1439 | if project.Exists: | 1754 | if project.Exists: |
1440 | project.PostRepoUpgrade() | 1755 | project.PostRepoUpgrade() |
1441 | 1756 | ||
1442 | 1757 | ||
1443 | def _PostRepoFetch(rp, repo_verify=True, verbose=False): | 1758 | def _PostRepoFetch(rp, repo_verify=True, verbose=False): |
1444 | if rp.HasChanges: | 1759 | if rp.HasChanges: |
1445 | print('info: A new version of repo is available', file=sys.stderr) | 1760 | print("info: A new version of repo is available", file=sys.stderr) |
1446 | wrapper = Wrapper() | 1761 | wrapper = Wrapper() |
1447 | try: | 1762 | try: |
1448 | rev = rp.bare_git.describe(rp.GetRevisionId()) | 1763 | rev = rp.bare_git.describe(rp.GetRevisionId()) |
1449 | except GitError: | 1764 | except GitError: |
1450 | rev = None | 1765 | rev = None |
1451 | _, new_rev = wrapper.check_repo_rev(rp.gitdir, rev, repo_verify=repo_verify) | 1766 | _, new_rev = wrapper.check_repo_rev( |
1452 | # See if we're held back due to missing signed tag. | 1767 | rp.gitdir, rev, repo_verify=repo_verify |
1453 | current_revid = rp.bare_git.rev_parse('HEAD') | 1768 | ) |
1454 | new_revid = rp.bare_git.rev_parse('--verify', new_rev) | 1769 | # See if we're held back due to missing signed tag. |
1455 | if current_revid != new_revid: | 1770 | current_revid = rp.bare_git.rev_parse("HEAD") |
1456 | # We want to switch to the new rev, but also not trash any uncommitted | 1771 | new_revid = rp.bare_git.rev_parse("--verify", new_rev) |
1457 | # changes. This helps with local testing/hacking. | 1772 | if current_revid != new_revid: |
1458 | # If a local change has been made, we will throw that away. | 1773 | # We want to switch to the new rev, but also not trash any |
1459 | # We also have to make sure this will switch to an older commit if that's | 1774 | # uncommitted changes. This helps with local testing/hacking. |
1460 | # the latest tag in order to support release rollback. | 1775 | # If a local change has been made, we will throw that away. |
1461 | try: | 1776 | # We also have to make sure this will switch to an older commit if |
1462 | rp.work_git.reset('--keep', new_rev) | 1777 | # that's the latest tag in order to support release rollback. |
1463 | except GitError as e: | 1778 | try: |
1464 | sys.exit(str(e)) | 1779 | rp.work_git.reset("--keep", new_rev) |
1465 | print('info: Restarting repo with latest version', file=sys.stderr) | 1780 | except GitError as e: |
1466 | raise RepoChangedException(['--repo-upgraded']) | 1781 | sys.exit(str(e)) |
1782 | print("info: Restarting repo with latest version", file=sys.stderr) | ||
1783 | raise RepoChangedException(["--repo-upgraded"]) | ||
1784 | else: | ||
1785 | print( | ||
1786 | "warning: Skipped upgrade to unverified version", | ||
1787 | file=sys.stderr, | ||
1788 | ) | ||
1467 | else: | 1789 | else: |
1468 | print('warning: Skipped upgrade to unverified version', file=sys.stderr) | 1790 | if verbose: |
1469 | else: | 1791 | print( |
1470 | if verbose: | 1792 | "repo version %s is current" % rp.work_git.describe(HEAD), |
1471 | print('repo version %s is current' % rp.work_git.describe(HEAD), | 1793 | file=sys.stderr, |
1472 | file=sys.stderr) | 1794 | ) |
1473 | 1795 | ||
1474 | 1796 | ||
1475 | class _FetchTimes(object): | 1797 | class _FetchTimes(object): |
1476 | _ALPHA = 0.5 | 1798 | _ALPHA = 0.5 |
1477 | 1799 | ||
1478 | def __init__(self, manifest): | 1800 | def __init__(self, manifest): |
1479 | self._path = os.path.join(manifest.repodir, '.repo_fetchtimes.json') | 1801 | self._path = os.path.join(manifest.repodir, ".repo_fetchtimes.json") |
1480 | self._times = None | 1802 | self._times = None |
1481 | self._seen = set() | 1803 | self._seen = set() |
1482 | 1804 | ||
1483 | def Get(self, project): | 1805 | def Get(self, project): |
1484 | self._Load() | 1806 | self._Load() |
1485 | return self._times.get(project.name, _ONE_DAY_S) | 1807 | return self._times.get(project.name, _ONE_DAY_S) |
1486 | 1808 | ||
1487 | def Set(self, project, t): | 1809 | def Set(self, project, t): |
1488 | self._Load() | 1810 | self._Load() |
1489 | name = project.name | 1811 | name = project.name |
1490 | old = self._times.get(name, t) | 1812 | old = self._times.get(name, t) |
1491 | self._seen.add(name) | 1813 | self._seen.add(name) |
1492 | a = self._ALPHA | 1814 | a = self._ALPHA |
1493 | self._times[name] = (a * t) + ((1 - a) * old) | 1815 | self._times[name] = (a * t) + ((1 - a) * old) |
1494 | 1816 | ||
1495 | def _Load(self): | 1817 | def _Load(self): |
1496 | if self._times is None: | 1818 | if self._times is None: |
1497 | try: | 1819 | try: |
1498 | with open(self._path) as f: | 1820 | with open(self._path) as f: |
1499 | self._times = json.load(f) | 1821 | self._times = json.load(f) |
1500 | except (IOError, ValueError): | 1822 | except (IOError, ValueError): |
1501 | platform_utils.remove(self._path, missing_ok=True) | 1823 | platform_utils.remove(self._path, missing_ok=True) |
1502 | self._times = {} | 1824 | self._times = {} |
1503 | 1825 | ||
1504 | def Save(self): | 1826 | def Save(self): |
1505 | if self._times is None: | 1827 | if self._times is None: |
1506 | return | 1828 | return |
1507 | 1829 | ||
1508 | to_delete = [] | 1830 | to_delete = [] |
1509 | for name in self._times: | 1831 | for name in self._times: |
1510 | if name not in self._seen: | 1832 | if name not in self._seen: |
1511 | to_delete.append(name) | 1833 | to_delete.append(name) |
1512 | for name in to_delete: | 1834 | for name in to_delete: |
1513 | del self._times[name] | 1835 | del self._times[name] |
1514 | 1836 | ||
1515 | try: | 1837 | try: |
1516 | with open(self._path, 'w') as f: | 1838 | with open(self._path, "w") as f: |
1517 | json.dump(self._times, f, indent=2) | 1839 | json.dump(self._times, f, indent=2) |
1518 | except (IOError, TypeError): | 1840 | except (IOError, TypeError): |
1519 | platform_utils.remove(self._path, missing_ok=True) | 1841 | platform_utils.remove(self._path, missing_ok=True) |
1842 | |||
1520 | 1843 | ||
1521 | # This is a replacement for xmlrpc.client.Transport using urllib2 | 1844 | # This is a replacement for xmlrpc.client.Transport using urllib2 |
1522 | # and supporting persistent-http[s]. It cannot change hosts from | 1845 | # and supporting persistent-http[s]. It cannot change hosts from |
@@ -1525,98 +1848,105 @@ class _FetchTimes(object): | |||
1525 | 1848 | ||
1526 | 1849 | ||
1527 | class PersistentTransport(xmlrpc.client.Transport): | 1850 | class PersistentTransport(xmlrpc.client.Transport): |
1528 | def __init__(self, orig_host): | 1851 | def __init__(self, orig_host): |
1529 | self.orig_host = orig_host | 1852 | self.orig_host = orig_host |
1530 | 1853 | ||
1531 | def request(self, host, handler, request_body, verbose=False): | 1854 | def request(self, host, handler, request_body, verbose=False): |
1532 | with GetUrlCookieFile(self.orig_host, not verbose) as (cookiefile, proxy): | 1855 | with GetUrlCookieFile(self.orig_host, not verbose) as ( |
1533 | # Python doesn't understand cookies with the #HttpOnly_ prefix | 1856 | cookiefile, |
1534 | # Since we're only using them for HTTP, copy the file temporarily, | 1857 | proxy, |
1535 | # stripping those prefixes away. | 1858 | ): |
1536 | if cookiefile: | 1859 | # Python doesn't understand cookies with the #HttpOnly_ prefix |
1537 | tmpcookiefile = tempfile.NamedTemporaryFile(mode='w') | 1860 | # Since we're only using them for HTTP, copy the file temporarily, |
1538 | tmpcookiefile.write("# HTTP Cookie File") | 1861 | # stripping those prefixes away. |
1539 | try: | 1862 | if cookiefile: |
1540 | with open(cookiefile) as f: | 1863 | tmpcookiefile = tempfile.NamedTemporaryFile(mode="w") |
1541 | for line in f: | 1864 | tmpcookiefile.write("# HTTP Cookie File") |
1542 | if line.startswith("#HttpOnly_"): | 1865 | try: |
1543 | line = line[len("#HttpOnly_"):] | 1866 | with open(cookiefile) as f: |
1544 | tmpcookiefile.write(line) | 1867 | for line in f: |
1545 | tmpcookiefile.flush() | 1868 | if line.startswith("#HttpOnly_"): |
1546 | 1869 | line = line[len("#HttpOnly_") :] | |
1547 | cookiejar = cookielib.MozillaCookieJar(tmpcookiefile.name) | 1870 | tmpcookiefile.write(line) |
1548 | try: | 1871 | tmpcookiefile.flush() |
1549 | cookiejar.load() | 1872 | |
1550 | except cookielib.LoadError: | 1873 | cookiejar = cookielib.MozillaCookieJar(tmpcookiefile.name) |
1551 | cookiejar = cookielib.CookieJar() | 1874 | try: |
1552 | finally: | 1875 | cookiejar.load() |
1553 | tmpcookiefile.close() | 1876 | except cookielib.LoadError: |
1554 | else: | 1877 | cookiejar = cookielib.CookieJar() |
1555 | cookiejar = cookielib.CookieJar() | 1878 | finally: |
1556 | 1879 | tmpcookiefile.close() | |
1557 | proxyhandler = urllib.request.ProxyHandler | 1880 | else: |
1558 | if proxy: | 1881 | cookiejar = cookielib.CookieJar() |
1559 | proxyhandler = urllib.request.ProxyHandler({ | 1882 | |
1560 | "http": proxy, | 1883 | proxyhandler = urllib.request.ProxyHandler |
1561 | "https": proxy}) | 1884 | if proxy: |
1562 | 1885 | proxyhandler = urllib.request.ProxyHandler( | |
1563 | opener = urllib.request.build_opener( | 1886 | {"http": proxy, "https": proxy} |
1564 | urllib.request.HTTPCookieProcessor(cookiejar), | 1887 | ) |
1565 | proxyhandler) | 1888 | |
1566 | 1889 | opener = urllib.request.build_opener( | |
1567 | url = urllib.parse.urljoin(self.orig_host, handler) | 1890 | urllib.request.HTTPCookieProcessor(cookiejar), proxyhandler |
1568 | parse_results = urllib.parse.urlparse(url) | 1891 | ) |
1569 | 1892 | ||
1570 | scheme = parse_results.scheme | 1893 | url = urllib.parse.urljoin(self.orig_host, handler) |
1571 | if scheme == 'persistent-http': | 1894 | parse_results = urllib.parse.urlparse(url) |
1572 | scheme = 'http' | 1895 | |
1573 | if scheme == 'persistent-https': | 1896 | scheme = parse_results.scheme |
1574 | # If we're proxying through persistent-https, use http. The | 1897 | if scheme == "persistent-http": |
1575 | # proxy itself will do the https. | 1898 | scheme = "http" |
1576 | if proxy: | 1899 | if scheme == "persistent-https": |
1577 | scheme = 'http' | 1900 | # If we're proxying through persistent-https, use http. The |
1578 | else: | 1901 | # proxy itself will do the https. |
1579 | scheme = 'https' | 1902 | if proxy: |
1580 | 1903 | scheme = "http" | |
1581 | # Parse out any authentication information using the base class | 1904 | else: |
1582 | host, extra_headers, _ = self.get_host_info(parse_results.netloc) | 1905 | scheme = "https" |
1583 | 1906 | ||
1584 | url = urllib.parse.urlunparse(( | 1907 | # Parse out any authentication information using the base class. |
1585 | scheme, | 1908 | host, extra_headers, _ = self.get_host_info(parse_results.netloc) |
1586 | host, | 1909 | |
1587 | parse_results.path, | 1910 | url = urllib.parse.urlunparse( |
1588 | parse_results.params, | 1911 | ( |
1589 | parse_results.query, | 1912 | scheme, |
1590 | parse_results.fragment)) | 1913 | host, |
1591 | 1914 | parse_results.path, | |
1592 | request = urllib.request.Request(url, request_body) | 1915 | parse_results.params, |
1593 | if extra_headers is not None: | 1916 | parse_results.query, |
1594 | for (name, header) in extra_headers: | 1917 | parse_results.fragment, |
1595 | request.add_header(name, header) | 1918 | ) |
1596 | request.add_header('Content-Type', 'text/xml') | 1919 | ) |
1597 | try: | 1920 | |
1598 | response = opener.open(request) | 1921 | request = urllib.request.Request(url, request_body) |
1599 | except urllib.error.HTTPError as e: | 1922 | if extra_headers is not None: |
1600 | if e.code == 501: | 1923 | for name, header in extra_headers: |
1601 | # We may have been redirected through a login process | 1924 | request.add_header(name, header) |
1602 | # but our POST turned into a GET. Retry. | 1925 | request.add_header("Content-Type", "text/xml") |
1603 | response = opener.open(request) | 1926 | try: |
1604 | else: | 1927 | response = opener.open(request) |
1605 | raise | 1928 | except urllib.error.HTTPError as e: |
1606 | 1929 | if e.code == 501: | |
1607 | p, u = xmlrpc.client.getparser() | 1930 | # We may have been redirected through a login process |
1608 | # Response should be fairly small, so read it all at once. | 1931 | # but our POST turned into a GET. Retry. |
1609 | # This way we can show it to the user in case of error (e.g. HTML). | 1932 | response = opener.open(request) |
1610 | data = response.read() | 1933 | else: |
1611 | try: | 1934 | raise |
1612 | p.feed(data) | 1935 | |
1613 | except xml.parsers.expat.ExpatError as e: | 1936 | p, u = xmlrpc.client.getparser() |
1614 | raise IOError( | 1937 | # Response should be fairly small, so read it all at once. |
1615 | f'Parsing the manifest failed: {e}\n' | 1938 | # This way we can show it to the user in case of error (e.g. HTML). |
1616 | f'Please report this to your manifest server admin.\n' | 1939 | data = response.read() |
1617 | f'Here is the full response:\n{data.decode("utf-8")}') | 1940 | try: |
1618 | p.close() | 1941 | p.feed(data) |
1619 | return u.close() | 1942 | except xml.parsers.expat.ExpatError as e: |
1620 | 1943 | raise IOError( | |
1621 | def close(self): | 1944 | f"Parsing the manifest failed: {e}\n" |
1622 | pass | 1945 | f"Please report this to your manifest server admin.\n" |
1946 | f'Here is the full response:\n{data.decode("utf-8")}' | ||
1947 | ) | ||
1948 | p.close() | ||
1949 | return u.close() | ||
1950 | |||
1951 | def close(self): | ||
1952 | pass | ||