From b5d075d04f1e555f85aad27e74f16073a50b2ae6 Mon Sep 17 00:00:00 2001 From: Mike Frysinger Date: Mon, 1 Mar 2021 00:56:38 -0500 Subject: command: add a helper for the parallel execution boilerplate Now that we have a bunch of subcommands doing parallel execution, a common pattern arises that we can factor out for most of them. We leave forall alone as it's a bit too complicated atm to cut over. Change-Id: I3617a4f7c66142bcd1ab030cb4cca698a65010ac Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/301942 Tested-by: Mike Frysinger Reviewed-by: Chris Mcdonald --- subcmds/branches.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'subcmds/branches.py') diff --git a/subcmds/branches.py b/subcmds/branches.py index d5ea580c..2dc102bb 100644 --- a/subcmds/branches.py +++ b/subcmds/branches.py @@ -13,10 +13,10 @@ # limitations under the License. import itertools -import multiprocessing import sys + from color import Coloring -from command import Command, DEFAULT_LOCAL_JOBS, WORKER_BATCH_SIZE +from command import Command, DEFAULT_LOCAL_JOBS class BranchColoring(Coloring): @@ -102,15 +102,19 @@ is shown, then the branch appears in all projects. out = BranchColoring(self.manifest.manifestProject.config) all_branches = {} project_cnt = len(projects) - with multiprocessing.Pool(processes=opt.jobs) as pool: - project_branches = pool.imap_unordered( - expand_project_to_branches, projects, chunksize=WORKER_BATCH_SIZE) - for name, b in itertools.chain.from_iterable(project_branches): + def _ProcessResults(_pool, _output, results): + for name, b in itertools.chain.from_iterable(results): if name not in all_branches: all_branches[name] = BranchInfo(name) all_branches[name].add(b) + self.ExecuteInParallel( + opt.jobs, + expand_project_to_branches, + projects, + callback=_ProcessResults) + names = sorted(all_branches) if not names: -- cgit v1.2.3-54-g00ecf