sync: superproject performance changes.

After updating all project’s revsionIds with the SHAs from superproject,
write the updated manifest into superproject_override.xml file. Reload
that file for future Reloads. This file is created in exp-superproject
directory.

Moved most of the code that is superproject specific into
git_superproject.py and wrote test code.

If git pull fails, did a git clone of the superproject.

We saw performance gains for consecutive repo sync's. The time to sync
went down from around 120 secs to 40 secs when repo sync is executed
consecutively.

Tested the code with the following commands.

$ ./run_tests -v tests/test_git_superproject.py
$ ./run_tests -v

Tested the sync code by copying all the repo changes into my Android
AOSP checkout and doing a repo sync --use-superproject twice.

First run
$ time repo sync --use-superproject
...
real	21m3.745s
user	97m59.380s
sys	19m11.286s

After two consecutive sync runs
$ time repo sync -c -j8 --use-superproject
real	0m39.626s
user	0m29.937s
sys	0m38.155s

Bug: https://crbug.com/gerrit/13709
Bug: https://crbug.com/gerrit/13707
Tested-by: Raman Tenneti <rtenneti@google.com>

Change-Id: Id79a0d7c4d20babd65e9bd485196c6f8fbe9de5e
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/296082
Reviewed-by: Ian Kasprzak <iankaz@google.com>
Tested-by: Raman Tenneti <rtenneti@google.com>

1 files changed, 85 insertions, 15 deletions

diff --git a/git_superproject.py b/git_superproject.py
index e2045cfd..465d1f87 100644
--- a/git_superproject.py
+++ b/git_superproject.py
@@ -25,8 +25,9 @@ Examples:
 import os
 import sys
 
-from error import GitError
+from error import BUG_REPORT_URL, GitError
 from git_command import GitCommand
+import platform_utils
 
 
 class Superproject(object):
@@ -46,6 +47,9 @@ class Superproject(object):
     self._repodir = os.path.abspath(repodir)
     self._superproject_dir = superproject_dir
     self._superproject_path = os.path.join(self._repodir, superproject_dir)
+    self._manifest_path = os.path.join(self._superproject_path,
+                                       'superproject_override.xml')
+    self._work_git = os.path.join(self._superproject_path, 'superproject')
 
   @property
   def project_shas(self):
@@ -57,7 +61,7 @@ class Superproject(object):
 
     Args:
       url: superproject's url to be passed to git clone.
-      branch: the branchname to be passed as argument to git clone.
+      branch: The branchname to be passed as argument to git clone.
 
     Returns:
       True if 'git clone <url> <branch>' is successful, or False.
@@ -86,13 +90,12 @@ class Superproject(object):
     Returns:
       True if 'git pull <branch>' is successful, or False.
     """
-    git_dir = os.path.join(self._superproject_path, 'superproject')
-    if not os.path.exists(git_dir):
-      raise GitError('git pull. Missing drectory: %s' % git_dir)
+    if not os.path.exists(self._work_git):
+      raise GitError('git pull missing drectory: %s' % self._work_git)
     cmd = ['pull']
     p = GitCommand(None,
                    cmd,
-                   cwd=git_dir,
+                   cwd=self._work_git,
                    capture_stdout=True,
                    capture_stderr=True)
     retval = p.Wait()
@@ -110,14 +113,13 @@ class Superproject(object):
     Returns:
       data: data returned from 'git ls-tree -r HEAD' instead of None.
     """
-    git_dir = os.path.join(self._superproject_path, 'superproject')
-    if not os.path.exists(git_dir):
-      raise GitError('git ls-tree. Missing drectory: %s' % git_dir)
+    if not os.path.exists(self._work_git):
+      raise GitError('git ls-tree. Missing drectory: %s' % self._work_git)
     data = None
     cmd = ['ls-tree', '-z', '-r', 'HEAD']
     p = GitCommand(None,
                    cmd,
-                   cwd=git_dir,
+                   cwd=self._work_git,
                    capture_stdout=True,
                    capture_stderr=True)
     retval = p.Wait()
@@ -130,22 +132,26 @@ class Superproject(object):
           retval, p.stderr), file=sys.stderr)
     return data
 
-  def GetAllProjectsSHAs(self, url, branch=None):
+  def _GetAllProjectsSHAs(self, url, branch=None):
     """Get SHAs for all projects from superproject and save them in _project_shas.
 
     Args:
-      url: superproject's url to be passed to git clone.
-      branch: the branchname to be passed as argument to git clone.
+      url: superproject's url to be passed to git clone or pull.
+      branch: The branchname to be passed as argument to git clone or pull.
 
     Returns:
       A dictionary with the projects/SHAs instead of None.
     """
     if not url:
       raise ValueError('url argument is not supplied.')
+    do_clone = True
     if os.path.exists(self._superproject_path):
       if not self._Pull():
-        raise GitError('git pull failed for url: %s' % url)
-    else:
+        # If pull fails due to a corrupted git directory, then do a git clone.
+        platform_utils.rmtree(self._superproject_path)
+      else:
+        do_clone = False
+    if do_clone:
       if not self._Clone(url, branch):
         raise GitError('git clone failed for url: %s' % url)
 
@@ -168,3 +174,67 @@ class Superproject(object):
 
     self._project_shas = shas
     return shas
+
+  def _WriteManfiestFile(self, manifest):
+    """Writes manifest to a file.
+
+    Args:
+      manifest: A Manifest object that is to be written to a file.
+
+    Returns:
+      manifest_path: Path name of the file into which manifest is written instead of None.
+    """
+    if not os.path.exists(self._superproject_path):
+      print('error: missing superproject directory %s' %
+            self._superproject_path,
+            file=sys.stderr)
+      return None
+    manifest_str = manifest.ToXml().toxml()
+    manifest_path = self._manifest_path
+    try:
+      with open(manifest_path, 'w', encoding='utf-8') as fp:
+        fp.write(manifest_str)
+    except IOError as e:
+      print('error: cannot write manifest to %s:\n%s'
+            % (manifest_path, e),
+            file=sys.stderr)
+      return None
+    return manifest_path
+
+  def UpdateProjectsRevisionId(self, manifest, projects, url, branch=None):
+    """Update revisionId of every project in projects with the SHA.
+
+    Args:
+      manifest: A Manifest object that is to be written to a file.
+      projects: List of projects whose revisionId needs to be updated.
+      url: superproject's url to be passed to git clone or fetch.
+      branch: The branchname to be passed as argument to git clone or pull.
+
+    Returns:
+      manifest_path: Path name of the overriding manfiest file instead of None.
+    """
+    try:
+      shas = self._GetAllProjectsSHAs(url=url, branch=branch)
+    except Exception as e:
+      print('error: Cannot get project SHAs for %s: %s: %s' %
+            (url, type(e).__name__, str(e)),
+            file=sys.stderr)
+      return None
+
+    projects_missing_shas = []
+    for project in projects:
+      path = project.relpath
+      if not path:
+        continue
+      sha = shas.get(path)
+      if sha:
+        project.SetRevisionId(sha)
+      else:
+        projects_missing_shas.append(path)
+    if projects_missing_shas:
+      print('error: please file a bug using %s to report missing shas for: %s' %
+            (BUG_REPORT_URL, projects_missing_shas), file=sys.stderr)
+      return None
+
+    manifest_path = self._WriteManfiestFile(manifest)
+    return manifest_path