From b5d075d04f1e555f85aad27e74f16073a50b2ae6 Mon Sep 17 00:00:00 2001 From: Mike Frysinger Date: Mon, 1 Mar 2021 00:56:38 -0500 Subject: command: add a helper for the parallel execution boilerplate Now that we have a bunch of subcommands doing parallel execution, a common pattern arises that we can factor out for most of them. We leave forall alone as it's a bit too complicated atm to cut over. Change-Id: I3617a4f7c66142bcd1ab030cb4cca698a65010ac Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/301942 Tested-by: Mike Frysinger Reviewed-by: Chris Mcdonald --- subcmds/grep.py | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) (limited to 'subcmds/grep.py') diff --git a/subcmds/grep.py b/subcmds/grep.py index 9a4a8a36..6cb1445a 100644 --- a/subcmds/grep.py +++ b/subcmds/grep.py @@ -13,11 +13,10 @@ # limitations under the License. import functools -import multiprocessing import sys from color import Coloring -from command import DEFAULT_LOCAL_JOBS, PagedCommand, WORKER_BATCH_SIZE +from command import DEFAULT_LOCAL_JOBS, PagedCommand from error import GitError from git_command import GitCommand @@ -173,7 +172,7 @@ contain a line that matches both expressions: return (project, p.Wait(), p.stdout, p.stderr) @staticmethod - def _ProcessResults(out, full_name, have_rev, results): + def _ProcessResults(full_name, have_rev, _pool, out, results): git_failed = False bad_rev = False have_match = False @@ -256,18 +255,13 @@ contain a line that matches both expressions: cmd_argv.extend(opt.revision) cmd_argv.append('--') - process_results = functools.partial( - self._ProcessResults, out, full_name, have_rev) - # NB: Multiprocessing is heavy, so don't spin it up for one job. - if len(projects) == 1 or opt.jobs == 1: - git_failed, bad_rev, have_match = process_results( - self._ExecuteOne(cmd_argv, x) for x in projects) - else: - with multiprocessing.Pool(opt.jobs) as pool: - results = pool.imap( - functools.partial(self._ExecuteOne, cmd_argv), projects, - chunksize=WORKER_BATCH_SIZE) - git_failed, bad_rev, have_match = process_results(results) + git_failed, bad_rev, have_match = self.ExecuteInParallel( + opt.jobs, + functools.partial(self._ExecuteOne, cmd_argv), + projects, + callback=functools.partial(self._ProcessResults, full_name, have_rev), + output=out, + ordered=True) if git_failed: sys.exit(1) -- cgit v1.2.3-54-g00ecf