From 1fd7bc24386dbba3a9454bb49c702a642f00e34c Mon Sep 17 00:00:00 2001 From: Raman Tenneti Date: Thu, 4 Feb 2021 14:39:38 -0800 Subject: sync: superproject performance changes. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After updating all project’s revsionIds with the SHAs from superproject, write the updated manifest into superproject_override.xml file. Reload that file for future Reloads. This file is created in exp-superproject directory. Moved most of the code that is superproject specific into git_superproject.py and wrote test code. If git pull fails, did a git clone of the superproject. We saw performance gains for consecutive repo sync's. The time to sync went down from around 120 secs to 40 secs when repo sync is executed consecutively. Tested the code with the following commands. $ ./run_tests -v tests/test_git_superproject.py $ ./run_tests -v Tested the sync code by copying all the repo changes into my Android AOSP checkout and doing a repo sync --use-superproject twice. First run $ time repo sync --use-superproject ... real 21m3.745s user 97m59.380s sys 19m11.286s After two consecutive sync runs $ time repo sync -c -j8 --use-superproject real 0m39.626s user 0m29.937s sys 0m38.155s Bug: https://crbug.com/gerrit/13709 Bug: https://crbug.com/gerrit/13707 Tested-by: Raman Tenneti Change-Id: Id79a0d7c4d20babd65e9bd485196c6f8fbe9de5e Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/296082 Reviewed-by: Ian Kasprzak Tested-by: Raman Tenneti --- subcmds/sync.py | 81 ++++++++++++++++++++++++++++++++------------------------- 1 file changed, 45 insertions(+), 36 deletions(-) (limited to 'subcmds/sync.py') diff --git a/subcmds/sync.py b/subcmds/sync.py index 225e565a..c0f605a8 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -56,7 +56,7 @@ import gitc_utils from project import Project from project import RemoteSpec from command import Command, MirrorSafeCommand -from error import BUG_REPORT_URL, RepoChangedException, GitError, ManifestParseError +from error import RepoChangedException, GitError, ManifestParseError import platform_utils from project import SyncBuffer from progress import Progress @@ -271,6 +271,47 @@ later is required to fix a server side protocol bug. dest='repo_upgraded', action='store_true', help=SUPPRESS_HELP) + def _UpdateProjectsRevisionId(self, opt, args): + """Update revisionId of every project with the SHA from superproject. + + This function updates each project's revisionId with SHA from superproject. + It writes the updated manifest into a file and reloads the manifest from it. + + Args: + opt: Program options returned from optparse. See _Options(). + args: Arguments to pass to GetProjects. See the GetProjects + docstring for details. + + Returns: + Returns path to the overriding manifest file. + """ + if not self.manifest.superproject: + print('error: superproject tag is not defined in manifest.xml', + file=sys.stderr) + sys.exit(1) + print('WARNING: --use-superproject is experimental and not ' + 'for general use', file=sys.stderr) + + superproject_url = self.manifest.superproject['remote'].url + if not superproject_url: + print('error: superproject URL is not defined in manifest.xml', + file=sys.stderr) + sys.exit(1) + + superproject = git_superproject.Superproject(self.manifest.repodir) + all_projects = self.GetProjects(args, + missing_ok=True, + submodules_ok=opt.fetch_submodules) + manifest_path = superproject.UpdateProjectsRevisionId(self.manifest, + all_projects, + url=superproject_url) + if not manifest_path: + print('error: Update of revsionId from superproject has failed', + file=sys.stderr) + sys.exit(1) + self._ReloadManifest(manifest_path) + return manifest_path + def _FetchProjectList(self, opt, projects, sem, *args, **kwargs): """Main function of the fetch threads. @@ -859,6 +900,9 @@ later is required to fix a server side protocol bug. else: self._UpdateManifestProject(opt, mp, manifest_name) + if opt.use_superproject: + manifest_name = self._UpdateProjectsRevisionId(opt, args) + if self.gitc_manifest: gitc_manifest_projects = self.GetProjects(args, missing_ok=True) @@ -898,41 +942,6 @@ later is required to fix a server side protocol bug. missing_ok=True, submodules_ok=opt.fetch_submodules) - if opt.use_superproject: - if not self.manifest.superproject: - print('error: superproject tag is not defined in manifest.xml', - file=sys.stderr) - sys.exit(1) - print('WARNING: --use-superproject is experimental and not ' - 'for general use', file=sys.stderr) - superproject_url = self.manifest.superproject['remote'].url - if not superproject_url: - print('error: superproject URL is not defined in manifest.xml', - file=sys.stderr) - sys.exit(1) - superproject = git_superproject.Superproject(self.manifest.repodir) - try: - superproject_shas = superproject.GetAllProjectsSHAs(url=superproject_url) - except Exception as e: - print('error: Cannot get project SHAs for %s: %s: %s' % - (superproject_url, type(e).__name__, str(e)), - file=sys.stderr) - sys.exit(1) - projects_missing_shas = [] - for project in all_projects: - path = project.relpath - if not path: - continue - sha = superproject_shas.get(path) - if sha: - project.SetRevisionId(sha) - else: - projects_missing_shas.append(path) - if projects_missing_shas: - print('error: please file a bug using %s to report missing shas for: %s' % - (BUG_REPORT_URL, projects_missing_shas), file=sys.stderr) - sys.exit(1) - err_network_sync = False err_update_projects = False err_checkout = False -- cgit v1.2.3-54-g00ecf