From 7c871163c8803e812998e5b2296e3bbb30b1367f Mon Sep 17 00:00:00 2001 From: Mike Frysinger Date: Tue, 16 Feb 2021 01:45:39 -0500 Subject: status: improve parallel execution stability The status command runs a bunch of jobs in parallel, and each one is responsible for writing to stdout directly. When running many noisy jobs in parallel, output can get intermingled. Pass down a StringIO buffer for writing to so we can return the entire output as a string so the main job can handle displaying it. This fixes interleaved output as well as making the output stable: we always display results in the same project order now. By switching from map to imap, this ends up not really adding any overhead. Bug: https://crbug.com/gerrit/12231 Change-Id: Ic18b07c8074c046ff36e306eb8d392fb34fb6eca Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/297242 Tested-by: Mike Frysinger Reviewed-by: Chris Mcdonald --- subcmds/status.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) (limited to 'subcmds/status.py') diff --git a/subcmds/status.py b/subcmds/status.py index f0f2e034..6c8e22e5 100644 --- a/subcmds/status.py +++ b/subcmds/status.py @@ -14,10 +14,11 @@ import functools import glob +import io import multiprocessing import os -from command import DEFAULT_LOCAL_JOBS, PagedCommand +from command import DEFAULT_LOCAL_JOBS, PagedCommand, WORKER_BATCH_SIZE from color import Coloring import platform_utils @@ -99,7 +100,9 @@ the following meanings: Returns: The status of the project. """ - return project.PrintWorkTreeStatus(quiet=quiet) + buf = io.StringIO() + ret = project.PrintWorkTreeStatus(quiet=quiet, output_redir=buf) + return (ret, buf.getvalue()) def _FindOrphans(self, dirs, proj_dirs, proj_dirs_parents, outstring): """find 'dirs' that are present in 'proj_dirs_parents' but not in 'proj_dirs'""" @@ -128,8 +131,13 @@ the following meanings: counter += 1 else: with multiprocessing.Pool(opt.jobs) as pool: - states = pool.map(functools.partial(self._StatusHelper, opt.quiet), all_projects) - counter += states.count('CLEAN') + states = pool.imap(functools.partial(self._StatusHelper, opt.quiet), + all_projects, chunksize=WORKER_BATCH_SIZE) + for (state, output) in states: + if output: + print(output, end='') + if state == 'CLEAN': + counter += 1 if not opt.quiet and len(all_projects) == counter: print('nothing to commit (working directory clean)') -- cgit v1.2.3-54-g00ecf