summaryrefslogtreecommitdiffstats
path: root/subcmds/sync.py
diff options
context:
space:
mode:
Diffstat (limited to 'subcmds/sync.py')
-rw-r--r--subcmds/sync.py49
1 files changed, 19 insertions, 30 deletions
diff --git a/subcmds/sync.py b/subcmds/sync.py
index 21166af5..4763fadc 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -51,7 +51,7 @@ import git_superproject
51import gitc_utils 51import gitc_utils
52from project import Project 52from project import Project
53from project import RemoteSpec 53from project import RemoteSpec
54from command import Command, MirrorSafeCommand, WORKER_BATCH_SIZE 54from command import Command, MirrorSafeCommand
55from error import RepoChangedException, GitError, ManifestParseError 55from error import RepoChangedException, GitError, ManifestParseError
56import platform_utils 56import platform_utils
57from project import SyncBuffer 57from project import SyncBuffer
@@ -428,11 +428,12 @@ later is required to fix a server side protocol bug.
428 428
429 return (ret, fetched) 429 return (ret, fetched)
430 430
431 def _CheckoutOne(self, opt, project): 431 def _CheckoutOne(self, detach_head, force_sync, project):
432 """Checkout work tree for one project 432 """Checkout work tree for one project
433 433
434 Args: 434 Args:
435 opt: Program options returned from optparse. See _Options(). 435 detach_head: Whether to leave a detached HEAD.
436 force_sync: Force checking out of the repo.
436 project: Project object for the project to checkout. 437 project: Project object for the project to checkout.
437 438
438 Returns: 439 Returns:
@@ -440,10 +441,10 @@ later is required to fix a server side protocol bug.
440 """ 441 """
441 start = time.time() 442 start = time.time()
442 syncbuf = SyncBuffer(self.manifest.manifestProject.config, 443 syncbuf = SyncBuffer(self.manifest.manifestProject.config,
443 detach_head=opt.detach_head) 444 detach_head=detach_head)
444 success = False 445 success = False
445 try: 446 try:
446 project.Sync_LocalHalf(syncbuf, force_sync=opt.force_sync) 447 project.Sync_LocalHalf(syncbuf, force_sync=force_sync)
447 success = syncbuf.Finish() 448 success = syncbuf.Finish()
448 except Exception as e: 449 except Exception as e:
449 print('error: Cannot checkout %s: %s: %s' % 450 print('error: Cannot checkout %s: %s: %s' %
@@ -464,44 +465,32 @@ later is required to fix a server side protocol bug.
464 opt: Program options returned from optparse. See _Options(). 465 opt: Program options returned from optparse. See _Options().
465 err_results: A list of strings, paths to git repos where checkout failed. 466 err_results: A list of strings, paths to git repos where checkout failed.
466 """ 467 """
467 ret = True
468 jobs = opt.jobs_checkout if opt.jobs_checkout else self.jobs
469
470 # Only checkout projects with worktrees. 468 # Only checkout projects with worktrees.
471 all_projects = [x for x in all_projects if x.worktree] 469 all_projects = [x for x in all_projects if x.worktree]
472 470
473 pm = Progress('Checking out', len(all_projects), quiet=opt.quiet) 471 def _ProcessResults(pool, pm, results):
474 472 ret = True
475 def _ProcessResults(results):
476 for (success, project, start, finish) in results: 473 for (success, project, start, finish) in results:
477 self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL, 474 self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL,
478 start, finish, success) 475 start, finish, success)
479 # Check for any errors before running any more tasks. 476 # Check for any errors before running any more tasks.
480 # ...we'll let existing jobs finish, though. 477 # ...we'll let existing jobs finish, though.
481 if not success: 478 if not success:
479 ret = False
482 err_results.append(project.relpath) 480 err_results.append(project.relpath)
483 if opt.fail_fast: 481 if opt.fail_fast:
484 return False 482 if pool:
483 pool.close()
484 return ret
485 pm.update(msg=project.name) 485 pm.update(msg=project.name)
486 return True 486 return ret
487
488 # NB: Multiprocessing is heavy, so don't spin it up for one job.
489 if len(all_projects) == 1 or jobs == 1:
490 if not _ProcessResults(self._CheckoutOne(opt, x) for x in all_projects):
491 ret = False
492 else:
493 with multiprocessing.Pool(jobs) as pool:
494 results = pool.imap_unordered(
495 functools.partial(self._CheckoutOne, opt),
496 all_projects,
497 chunksize=WORKER_BATCH_SIZE)
498 if not _ProcessResults(results):
499 ret = False
500 pool.close()
501
502 pm.end()
503 487
504 return ret and not err_results 488 return self.ExecuteInParallel(
489 opt.jobs_checkout if opt.jobs_checkout else self.jobs,
490 functools.partial(self._CheckoutOne, opt.detach_head, opt.force_sync),
491 all_projects,
492 callback=_ProcessResults,
493 output=Progress('Checking out', len(all_projects), quiet=opt.quiet)) and not err_results
505 494
506 def _GCProjects(self, projects, opt, err_event): 495 def _GCProjects(self, projects, opt, err_event):
507 pm = Progress('Garbage collecting', len(projects), delay=False, quiet=opt.quiet) 496 pm = Progress('Garbage collecting', len(projects), delay=False, quiet=opt.quiet)