Add envar to replace shallow clones with partial

An investigation go/git-repo-shallow shows a number of problems
when doing a shallow git fetch/clone. This change introduces an
environment variable REPO_ALLOW_SHALLOW. When this environment variable
is set to 1 during a repo init or repo sync all shallow git fetch
commands are replaced with partial fetch commands. Any shallow
repository needing update is unshallowed. This behavior continues until
a subsequent repo sync command is run with REPO_ALLOW_SHALLOW set to 1.

Bug: b/274340522
Change-Id: I1c3188270629359e52449788897d9d4988ebf280
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/374754
Reviewed-by: Josip Sokcevic <sokcevic@google.com>
Tested-by: Jason Chang <jasonnc@google.com>

4 files changed, 55 insertions, 0 deletions

diff --git a/manifest_xml.py b/manifest_xml.py
index 14b03a30..555bf736 100644
--- a/manifest_xml.py
+++ b/manifest_xml.py
@@ -982,6 +982,12 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
         return None
 
     @property
+    def CloneFilterForDepth(self):
+        if self.manifestProject.clone_filter_for_depth:
+            return self.manifestProject.clone_filter_for_depth
+        return None
+
+    @property
     def PartialCloneExclude(self):
         exclude = self.manifest.manifestProject.partial_clone_exclude or ""
         return set(x.strip() for x in exclude.split(","))
diff --git a/project.py b/project.py
index ff018393..83f3eff9 100644
--- a/project.py
+++ b/project.py
@@ -1186,6 +1186,7 @@ class Project(object):
         ssh_proxy=None,
         clone_filter=None,
         partial_clone_exclude=set(),
+        clone_filter_for_depth=None,
     ):
         """Perform only the network IO portion of the sync process.
         Local working directory/branch state is not affected.
@@ -1295,6 +1296,10 @@ class Project(object):
         else:
             depth = self.manifest.manifestProject.depth
 
+        if depth and clone_filter_for_depth:
+            depth = None
+            clone_filter = clone_filter_for_depth
+
         # See if we can skip the network fetch entirely.
         remote_fetched = False
         if not (
@@ -3885,6 +3890,11 @@ class ManifestProject(MetaProject):
         return self.config.GetString("repo.partialcloneexclude")
 
     @property
+    def clone_filter_for_depth(self):
+        """Replace shallow clone with partial clone."""
+        return self.config.GetString("repo.clonefilterfordepth")
+
+    @property
     def manifest_platform(self):
         """The --platform argument from `repo init`."""
         return self.config.GetString("manifest.platform")
@@ -3961,6 +3971,7 @@ class ManifestProject(MetaProject):
             manifest_name=spec.manifestName,
             this_manifest_only=True,
             outer_manifest=False,
+            clone_filter_for_depth=mp.clone_filter_for_depth,
         )
 
     def Sync(
@@ -3991,6 +4002,7 @@ class ManifestProject(MetaProject):
         tags="",
         this_manifest_only=False,
         outer_manifest=True,
+        clone_filter_for_depth=None,
     ):
         """Sync the manifest and all submanifests.
 
@@ -4035,6 +4047,8 @@ class ManifestProject(MetaProject):
                 current sub manifest.
             outer_manifest: a boolean, whether to start at the outermost
                 manifest.
+            clone_filter_for_depth: a string, when specified replaces shallow
+                clones with partial.
 
         Returns:
             a boolean, whether the sync was successful.
@@ -4297,6 +4311,9 @@ class ManifestProject(MetaProject):
                     file=sys.stderr,
                 )
 
+        if clone_filter_for_depth is not None:
+            self.ConfigureCloneFilterForDepth(clone_filter_for_depth)
+
         if use_superproject is not None:
             self.config.SetBoolean("repo.superproject", use_superproject)
 
@@ -4311,6 +4328,7 @@ class ManifestProject(MetaProject):
                 submodules=submodules,
                 clone_filter=clone_filter,
                 partial_clone_exclude=self.manifest.PartialCloneExclude,
+                clone_filter_for_depth=self.manifest.CloneFilterForDepth,
             ).success
             if not success:
                 r = self.GetRemote()
@@ -4415,6 +4433,18 @@ class ManifestProject(MetaProject):
 
         return True
 
+    def ConfigureCloneFilterForDepth(self, clone_filter_for_depth):
+        """Configure clone filter to replace shallow clones.
+
+        Args:
+            clone_filter_for_depth: a string or None, e.g. 'blob:none' will
+            disable shallow clones and replace with partial clone. None will
+            enable shallow clones.
+        """
+        self.config.SetString(
+            "repo.clonefilterfordepth", clone_filter_for_depth
+        )
+
     def _ConfigureDepth(self, depth):
         """Configure the depth we'll sync down.
 
diff --git a/subcmds/init.py b/subcmds/init.py
index 9946466d..6d7fd857 100644
--- a/subcmds/init.py
+++ b/subcmds/init.py
@@ -20,6 +20,8 @@ from command import InteractiveCommand, MirrorSafeCommand
 from git_command import git_require, MIN_GIT_VERSION_SOFT, MIN_GIT_VERSION_HARD
 from wrapper import Wrapper
 
+_REPO_ALLOW_SHALLOW = os.environ.get("REPO_ALLOW_SHALLOW")
+
 
 class Init(InteractiveCommand, MirrorSafeCommand):
     COMMON = True
@@ -125,6 +127,9 @@ to update the working directory files.
         # manifest project is special and is created when instantiating the
         # manifest which happens before we parse options.
         self.manifest.manifestProject.clone_depth = opt.manifest_depth
+        clone_filter_for_depth = (
+            "blob:none" if (_REPO_ALLOW_SHALLOW == "0") else None
+        )
         if not self.manifest.manifestProject.Sync(
             manifest_url=opt.manifest_url,
             manifest_branch=opt.manifest_branch,
@@ -140,6 +145,7 @@ to update the working directory files.
             partial_clone=opt.partial_clone,
             clone_filter=opt.clone_filter,
             partial_clone_exclude=opt.partial_clone_exclude,
+            clone_filter_for_depth=clone_filter_for_depth,
             clone_bundle=opt.clone_bundle,
             git_lfs=opt.git_lfs,
             use_superproject=opt.use_superproject,
diff --git a/subcmds/sync.py b/subcmds/sync.py
index a44ed5b4..9ae8a4ce 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -79,6 +79,8 @@ _ONE_DAY_S = 24 * 60 * 60
 _REPO_AUTO_GC = "REPO_AUTO_GC"
 _AUTO_GC = os.environ.get(_REPO_AUTO_GC) == "1"
 
+_REPO_ALLOW_SHALLOW = os.environ.get("REPO_ALLOW_SHALLOW")
+
 
 class _FetchOneResult(NamedTuple):
     """_FetchOne return value.
@@ -638,6 +640,7 @@ later is required to fix a server side protocol bug.
                 ssh_proxy=self.ssh_proxy,
                 clone_filter=project.manifest.CloneFilter,
                 partial_clone_exclude=project.manifest.PartialCloneExclude,
+                clone_filter_for_depth=project.manifest.CloneFilterForDepth,
             )
             success = sync_result.success
             remote_fetched = sync_result.remote_fetched
@@ -1440,6 +1443,7 @@ later is required to fix a server side protocol bug.
                 submodules=mp.manifest.HasSubmodules,
                 clone_filter=mp.manifest.CloneFilter,
                 partial_clone_exclude=mp.manifest.PartialCloneExclude,
+                clone_filter_for_depth=mp.manifest.CloneFilterForDepth,
             )
             finish = time.time()
             self.event_log.AddSync(
@@ -1589,6 +1593,15 @@ later is required to fix a server side protocol bug.
             _PostRepoUpgrade(manifest, quiet=opt.quiet)
 
         mp = manifest.manifestProject
+
+        if _REPO_ALLOW_SHALLOW is not None:
+            if _REPO_ALLOW_SHALLOW == "1":
+                mp.ConfigureCloneFilterForDepth(None)
+            elif (
+                _REPO_ALLOW_SHALLOW == "0" and mp.clone_filter_for_depth is None
+            ):
+                mp.ConfigureCloneFilterForDepth("blob:none")
+
         if opt.mp_update:
             self._UpdateAllManifestProjects(opt, mp, manifest_name)
         else: