From 8da4861b3860c505e39341b4135c21f67569e4d8 Mon Sep 17 00:00:00 2001 From: Kuang-che Wu Date: Tue, 22 Oct 2024 21:04:41 +0800 Subject: subcmds: reduce multiprocessing serialization overhead Follow the same approach as 39ffd9977e to reduce serialization overhead. Below benchmarks are tested with 2.7k projects on my workstation (warm cache). git tracing is disabled for benchmark. (seconds) | v2.48 | v2.48 | this CL | this CL | | -j32 | | -j32 ----------------------------------------------------------- with clean tree state: branches (none) | 5.6 | 5.9 | 1.0 | 0.9 status (clean) | 21.3 | 9.4 | 19.4 | 4.7 diff (none) | 7.6 | 7.2 | 5.7 | 2.2 prune (none) | 5.7 | 6.1 | 1.3 | 1.2 abandon (none) | 19.4 | 18.6 | 0.9 | 0.8 upload (none) | 19.7 | 18.7 | 0.9 | 0.8 forall -c true | 7.5 | 7.6 | 0.6 | 0.6 forall -c "git log -1" | 11.3 | 11.1 | 0.6 | 0.6 with branches: start BRANCH --all | 21.9 | 20.3 | 13.6 | 2.6 checkout BRANCH | 29.1 | 27.8 | 1.1 | 1.0 branches (2) | 28.0 | 28.6 | 1.5 | 1.3 abandon BRANCH | 29.2 | 27.5 | 9.7 | 2.2 Bug: b/371638995 Change-Id: I53989a3d1e43063587b3f52f852b1c2c56b49412 Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/440221 Reviewed-by: Josip Sokcevic Tested-by: Kuang-che Wu Commit-Queue: Kuang-che Wu --- subcmds/branches.py | 49 ++++++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 23 deletions(-) (limited to 'subcmds/branches.py') diff --git a/subcmds/branches.py b/subcmds/branches.py index 59b5cb28..08c6389c 100644 --- a/subcmds/branches.py +++ b/subcmds/branches.py @@ -98,6 +98,22 @@ is shown, then the branch appears in all projects. """ PARALLEL_JOBS = DEFAULT_LOCAL_JOBS + @classmethod + def _ExpandProjectToBranches(cls, project_idx): + """Expands a project into a list of branch names & associated info. + + Args: + project_idx: project.Project index + + Returns: + List[Tuple[str, git_config.Branch, int]] + """ + branches = [] + project = cls.get_parallel_context()["projects"][project_idx] + for name, b in project.GetBranches().items(): + branches.append((name, b, project_idx)) + return branches + def Execute(self, opt, args): projects = self.GetProjects( args, all_manifests=not opt.this_manifest_only @@ -107,17 +123,20 @@ is shown, then the branch appears in all projects. project_cnt = len(projects) def _ProcessResults(_pool, _output, results): - for name, b in itertools.chain.from_iterable(results): + for name, b, project_idx in itertools.chain.from_iterable(results): + b.project = projects[project_idx] if name not in all_branches: all_branches[name] = BranchInfo(name) all_branches[name].add(b) - self.ExecuteInParallel( - opt.jobs, - expand_project_to_branches, - projects, - callback=_ProcessResults, - ) + with self.ParallelContext(): + self.get_parallel_context()["projects"] = projects + self.ExecuteInParallel( + opt.jobs, + self._ExpandProjectToBranches, + range(len(projects)), + callback=_ProcessResults, + ) names = sorted(all_branches) @@ -191,19 +210,3 @@ is shown, then the branch appears in all projects. else: out.write(" in all projects") out.nl() - - -def expand_project_to_branches(project): - """Expands a project into a list of branch names & associated information. - - Args: - project: project.Project - - Returns: - List[Tuple[str, git_config.Branch]] - """ - branches = [] - for name, b in project.GetBranches().items(): - b.project = project - branches.append((name, b)) - return branches -- cgit v1.2.3-54-g00ecf