init: added --use-superproject option to clone superproject.v2.12.2

Added --no-use-superproject to repo and init.py to disable use of
manifest superprojects.

Replaced the term "sha" with "commit id".

Added _GetBranch method to Superproject object.

Moved shared code between init and sync into SyncSuperproject function.
This function either does git clone or git fetch. If git fetch fails
it does git clone.

Changed Superproject constructor to accept manifest, repodir and branch
to avoid passing them to multiple functions as argument.

Changed functions that were raising exceptions to return either True
or False.

Saved the --use-superproject option in config as repo.superproject.
Updated internal-fs-layout.md document.

Updated the tests to work with the new API changes in Superproject.

Performance for the first time sync has improved from 20 minutes to
around 15 minutes.

Tested the code with the following commands.

$ ./run_tests -v

Tested the sync code by using repo_dev alias and pointing to this CL.

$ repo init took around 20 seconds longer because of cloning of superproject.

$ time repo_dev init -u sso://android.git.corp.google.com/platform/manifest -b master --partial-clone --clone-filter=blob:limit=10M --repo-rev=main --use-superproject
...
real	0m35.919s
user	0m21.947s
sys	0m8.977s

First run
$ time repo sync --use-superproject
...
real	16m41.982s
user	100m6.916s
sys	19m18.753s

No difference in repo sync time after the first run.

Bug: [google internal] b/179090734
Bug: https://crbug.com/gerrit/13709
Bug: https://crbug.com/gerrit/13707

Change-Id: I12df92112f46e001dfbc6f12cd633c3a15cf924b
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/296382
Reviewed-by: Mike Frysinger <vapier@google.com>
Tested-by: Raman Tenneti <rtenneti@google.com>

1 files changed, 64 insertions, 51 deletions

diff --git a/tests/test_git_superproject.py b/tests/test_git_superproject.py
index fc9101dd..d2ee9f4f 100644
--- a/tests/test_git_superproject.py
+++ b/tests/test_git_superproject.py
@@ -19,7 +19,6 @@ import tempfile
 import unittest
 from unittest import mock
 
-from error import GitError
 import git_superproject
 import manifest_xml
 import platform_utils
@@ -32,7 +31,6 @@ class SuperprojectTestCase(unittest.TestCase):
     """Set up superproject every time."""
     self.tempdir = tempfile.mkdtemp(prefix='repo_tests')
     self.repodir = os.path.join(self.tempdir, '.repo')
-    self._superproject = git_superproject.Superproject(self.repodir)
     self.manifest_file = os.path.join(
         self.repodir, manifest_xml.MANIFEST_FILE_NAME)
     os.mkdir(self.repodir)
@@ -45,6 +43,16 @@ class SuperprojectTestCase(unittest.TestCase):
         url = https://localhost:0/manifest
 """)
 
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="refs/heads/main" />
+  <superproject name="superproject"/>
+  <project path="art" name="platform/art" />
+</manifest>
+""")
+    self._superproject = git_superproject.Superproject(manifest, self.repodir)
+
   def tearDown(self):
     """Tear down superproject every time."""
     platform_utils.rmtree(self.tempdir)
@@ -55,37 +63,53 @@ class SuperprojectTestCase(unittest.TestCase):
       fp.write(data)
     return manifest_xml.XmlManifest(self.repodir, self.manifest_file)
 
-  def test_superproject_get_project_shas_no_url(self):
+  def test_superproject_get_superproject_no_superproject(self):
     """Test with no url."""
-    with self.assertRaises(ValueError):
-      self._superproject._GetAllProjectsSHAs(url=None)
+    manifest = self.getXmlManifest("""
+<manifest>
+</manifest>
+""")
+    superproject = git_superproject.Superproject(manifest, self.repodir)
+    self.assertFalse(superproject.Sync())
 
-  def test_superproject_get_project_shas_invalid_url(self):
+  def test_superproject_get_superproject_invalid_url(self):
     """Test with an invalid url."""
-    with self.assertRaises(GitError):
-      self._superproject._GetAllProjectsSHAs(url='localhost')
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="test-remote" fetch="localhost" />
+  <default remote="test-remote" revision="refs/heads/main" />
+  <superproject name="superproject"/>
+</manifest>
+""")
+    superproject = git_superproject.Superproject(manifest, self.repodir)
+    self.assertFalse(superproject.Sync())
 
-  def test_superproject_get_project_shas_invalid_branch(self):
+  def test_superproject_get_superproject_invalid_branch(self):
     """Test with an invalid branch."""
-    with self.assertRaises(GitError):
-      self._superproject._GetAllProjectsSHAs(
-          url='sso://android/platform/superproject',
-          branch='junk')
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="test-remote" fetch="localhost" />
+  <default remote="test-remote" revision="refs/heads/main" />
+  <superproject name="superproject"/>
+</manifest>
+""")
+    superproject = git_superproject.Superproject(manifest, self.repodir)
+    with mock.patch.object(self._superproject, '_GetBranch', return_value='junk'):
+      self.assertFalse(superproject.Sync())
 
-  def test_superproject_get_project_shas_mock_clone(self):
+  def test_superproject_get_superproject_mock_clone(self):
     """Test with _Clone failing."""
-    with self.assertRaises(GitError):
-      with mock.patch.object(self._superproject, '_Clone', return_value=False):
-        self._superproject._GetAllProjectsSHAs(url='localhost')
-
-  def test_superproject_get_project_shas_mock_fetch(self):
-    """Test with _Fetch failing."""
-    with self.assertRaises(GitError):
-      with mock.patch.object(self._superproject, '_Clone', return_value=True):
-        with mock.patch.object(self._superproject, '_Fetch', return_value=False):
-          self._superproject._GetAllProjectsSHAs(url='localhost')
-
-  def test_superproject_get_project_shas_mock_ls_tree(self):
+    with mock.patch.object(self._superproject, '_Clone', return_value=False):
+      self.assertFalse(self._superproject.Sync())
+
+  def test_superproject_get_superproject_mock_fetch(self):
+    """Test with _Fetch failing and _clone being called."""
+    with mock.patch.object(self._superproject, '_Clone', return_value=True):
+      os.mkdir(self._superproject._superproject_path)
+      with mock.patch.object(self._superproject, '_Fetch', return_value=False):
+        self.assertTrue(self._superproject.Sync())
+
+  def test_superproject_get_all_project_commit_ids_mock_ls_tree(self):
     """Test with LsTree being a mock."""
     data = ('120000 blob 158258bdf146f159218e2b90f8b699c4d85b5804\tAndroid.bp\x00'
             '160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00'
@@ -94,8 +118,8 @@ class SuperprojectTestCase(unittest.TestCase):
             '160000 commit ade9b7a0d874e25fff4bf2552488825c6f111928\tbuild/bazel\x00')
     with mock.patch.object(self._superproject, '_Clone', return_value=True):
       with mock.patch.object(self._superproject, '_LsTree', return_value=data):
-        shas = self._superproject._GetAllProjectsSHAs(url='localhost', branch='junk')
-        self.assertEqual(shas, {
+        commit_ids = self._superproject._GetAllProjectsCommitIds()
+        self.assertEqual(commit_ids, {
             'art': '2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea',
             'bootable/recovery': 'e9d25da64d8d365dbba7c8ee00fe8c4473fe9a06',
             'build/bazel': 'ade9b7a0d874e25fff4bf2552488825c6f111928'
@@ -103,19 +127,12 @@ class SuperprojectTestCase(unittest.TestCase):
 
   def test_superproject_write_manifest_file(self):
     """Test with writing manifest to a file after setting revisionId."""
-    manifest = self.getXmlManifest("""
-<manifest>
-  <remote name="default-remote" fetch="http://localhost" />
-  <default remote="default-remote" revision="refs/heads/main" />
-  <project name="test-name"/>
-</manifest>
-""")
-    self.assertEqual(len(manifest.projects), 1)
-    project = manifest.projects[0]
+    self.assertEqual(len(self._superproject._manifest.projects), 1)
+    project = self._superproject._manifest.projects[0]
     project.SetRevisionId('ABCDEF')
     # Create temporary directory so that it can write the file.
     os.mkdir(self._superproject._superproject_path)
-    manifest_path = self._superproject._WriteManfiestFile(manifest)
+    manifest_path = self._superproject._WriteManfiestFile()
     self.assertIsNotNone(manifest_path)
     with open(manifest_path, 'r') as fp:
       manifest_xml = fp.read()
@@ -124,29 +141,24 @@ class SuperprojectTestCase(unittest.TestCase):
         '<?xml version="1.0" ?><manifest>' +
         '<remote name="default-remote" fetch="http://localhost"/>' +
         '<default remote="default-remote" revision="refs/heads/main"/>' +
-        '<project name="test-name" revision="ABCDEF"/>' +
+        '<project name="platform/art" path="art" revision="ABCDEF"/>' +
+        '<superproject name="superproject"/>' +
         '</manifest>')
 
   def test_superproject_update_project_revision_id(self):
     """Test with LsTree being a mock."""
-    manifest = self.getXmlManifest("""
-<manifest>
-  <remote name="default-remote" fetch="http://localhost" />
-  <default remote="default-remote" revision="refs/heads/main" />
-  <project path="art" name="platform/art" />
-</manifest>
-""")
-    self.assertEqual(len(manifest.projects), 1)
-    projects = manifest.projects
+    self.assertEqual(len(self._superproject._manifest.projects), 1)
+    projects = self._superproject._manifest.projects
     data = ('160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00'
             '160000 commit e9d25da64d8d365dbba7c8ee00fe8c4473fe9a06\tbootable/recovery\x00')
     with mock.patch.object(self._superproject, '_Clone', return_value=True):
       with mock.patch.object(self._superproject, '_Fetch', return_value=True):
-        with mock.patch.object(self._superproject, '_LsTree', return_value=data):
+        with mock.patch.object(self._superproject,
+                               '_LsTree',
+                               return_value=data):
           # Create temporary directory so that it can write the file.
           os.mkdir(self._superproject._superproject_path)
-          manifest_path = self._superproject.UpdateProjectsRevisionId(
-              manifest, projects, url='localhost')
+          manifest_path = self._superproject.UpdateProjectsRevisionId(projects)
           self.assertIsNotNone(manifest_path)
           with open(manifest_path, 'r') as fp:
             manifest_xml = fp.read()
@@ -157,6 +169,7 @@ class SuperprojectTestCase(unittest.TestCase):
               '<default remote="default-remote" revision="refs/heads/main"/>' +
               '<project name="platform/art" path="art" ' +
               'revision="2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea"/>' +
+              '<superproject name="superproject"/>' +
               '</manifest>')