diff options
Diffstat (limited to 'subcmds/sync.py')
-rw-r--r-- | subcmds/sync.py | 49 |
1 files changed, 19 insertions, 30 deletions
diff --git a/subcmds/sync.py b/subcmds/sync.py index 21166af5..4763fadc 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py | |||
@@ -51,7 +51,7 @@ import git_superproject | |||
51 | import gitc_utils | 51 | import gitc_utils |
52 | from project import Project | 52 | from project import Project |
53 | from project import RemoteSpec | 53 | from project import RemoteSpec |
54 | from command import Command, MirrorSafeCommand, WORKER_BATCH_SIZE | 54 | from command import Command, MirrorSafeCommand |
55 | from error import RepoChangedException, GitError, ManifestParseError | 55 | from error import RepoChangedException, GitError, ManifestParseError |
56 | import platform_utils | 56 | import platform_utils |
57 | from project import SyncBuffer | 57 | from project import SyncBuffer |
@@ -428,11 +428,12 @@ later is required to fix a server side protocol bug. | |||
428 | 428 | ||
429 | return (ret, fetched) | 429 | return (ret, fetched) |
430 | 430 | ||
431 | def _CheckoutOne(self, opt, project): | 431 | def _CheckoutOne(self, detach_head, force_sync, project): |
432 | """Checkout work tree for one project | 432 | """Checkout work tree for one project |
433 | 433 | ||
434 | Args: | 434 | Args: |
435 | opt: Program options returned from optparse. See _Options(). | 435 | detach_head: Whether to leave a detached HEAD. |
436 | force_sync: Force checking out of the repo. | ||
436 | project: Project object for the project to checkout. | 437 | project: Project object for the project to checkout. |
437 | 438 | ||
438 | Returns: | 439 | Returns: |
@@ -440,10 +441,10 @@ later is required to fix a server side protocol bug. | |||
440 | """ | 441 | """ |
441 | start = time.time() | 442 | start = time.time() |
442 | syncbuf = SyncBuffer(self.manifest.manifestProject.config, | 443 | syncbuf = SyncBuffer(self.manifest.manifestProject.config, |
443 | detach_head=opt.detach_head) | 444 | detach_head=detach_head) |
444 | success = False | 445 | success = False |
445 | try: | 446 | try: |
446 | project.Sync_LocalHalf(syncbuf, force_sync=opt.force_sync) | 447 | project.Sync_LocalHalf(syncbuf, force_sync=force_sync) |
447 | success = syncbuf.Finish() | 448 | success = syncbuf.Finish() |
448 | except Exception as e: | 449 | except Exception as e: |
449 | print('error: Cannot checkout %s: %s: %s' % | 450 | print('error: Cannot checkout %s: %s: %s' % |
@@ -464,44 +465,32 @@ later is required to fix a server side protocol bug. | |||
464 | opt: Program options returned from optparse. See _Options(). | 465 | opt: Program options returned from optparse. See _Options(). |
465 | err_results: A list of strings, paths to git repos where checkout failed. | 466 | err_results: A list of strings, paths to git repos where checkout failed. |
466 | """ | 467 | """ |
467 | ret = True | ||
468 | jobs = opt.jobs_checkout if opt.jobs_checkout else self.jobs | ||
469 | |||
470 | # Only checkout projects with worktrees. | 468 | # Only checkout projects with worktrees. |
471 | all_projects = [x for x in all_projects if x.worktree] | 469 | all_projects = [x for x in all_projects if x.worktree] |
472 | 470 | ||
473 | pm = Progress('Checking out', len(all_projects), quiet=opt.quiet) | 471 | def _ProcessResults(pool, pm, results): |
474 | 472 | ret = True | |
475 | def _ProcessResults(results): | ||
476 | for (success, project, start, finish) in results: | 473 | for (success, project, start, finish) in results: |
477 | self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL, | 474 | self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL, |
478 | start, finish, success) | 475 | start, finish, success) |
479 | # Check for any errors before running any more tasks. | 476 | # Check for any errors before running any more tasks. |
480 | # ...we'll let existing jobs finish, though. | 477 | # ...we'll let existing jobs finish, though. |
481 | if not success: | 478 | if not success: |
479 | ret = False | ||
482 | err_results.append(project.relpath) | 480 | err_results.append(project.relpath) |
483 | if opt.fail_fast: | 481 | if opt.fail_fast: |
484 | return False | 482 | if pool: |
483 | pool.close() | ||
484 | return ret | ||
485 | pm.update(msg=project.name) | 485 | pm.update(msg=project.name) |
486 | return True | 486 | return ret |
487 | |||
488 | # NB: Multiprocessing is heavy, so don't spin it up for one job. | ||
489 | if len(all_projects) == 1 or jobs == 1: | ||
490 | if not _ProcessResults(self._CheckoutOne(opt, x) for x in all_projects): | ||
491 | ret = False | ||
492 | else: | ||
493 | with multiprocessing.Pool(jobs) as pool: | ||
494 | results = pool.imap_unordered( | ||
495 | functools.partial(self._CheckoutOne, opt), | ||
496 | all_projects, | ||
497 | chunksize=WORKER_BATCH_SIZE) | ||
498 | if not _ProcessResults(results): | ||
499 | ret = False | ||
500 | pool.close() | ||
501 | |||
502 | pm.end() | ||
503 | 487 | ||
504 | return ret and not err_results | 488 | return self.ExecuteInParallel( |
489 | opt.jobs_checkout if opt.jobs_checkout else self.jobs, | ||
490 | functools.partial(self._CheckoutOne, opt.detach_head, opt.force_sync), | ||
491 | all_projects, | ||
492 | callback=_ProcessResults, | ||
493 | output=Progress('Checking out', len(all_projects), quiet=opt.quiet)) and not err_results | ||
505 | 494 | ||
506 | def _GCProjects(self, projects, opt, err_event): | 495 | def _GCProjects(self, projects, opt, err_event): |
507 | pm = Progress('Garbage collecting', len(projects), delay=False, quiet=opt.quiet) | 496 | pm = Progress('Garbage collecting', len(projects), delay=False, quiet=opt.quiet) |