1 files changed, 214 insertions, 79 deletions

diff --git a/git_superproject.py b/git_superproject.py
index 89320971..4ca84a58 100644
--- a/git_superproject.py
+++ b/git_superproject.py
@@ -19,21 +19,52 @@ https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects
 
 Examples:
   superproject = Superproject()
-  project_commit_ids = superproject.UpdateProjectsRevisionId(projects)
+  UpdateProjectsResult = superproject.UpdateProjectsRevisionId(projects)
 """
 
 import hashlib
+import functools
 import os
 import sys
+import time
+from typing import NamedTuple
 
-from error import BUG_REPORT_URL
-from git_command import GitCommand
+from git_command import git_require, GitCommand
+from git_config import RepoConfig
 from git_refs import R_HEADS
+from manifest_xml import LOCAL_MANIFEST_GROUP_PREFIX
 
 _SUPERPROJECT_GIT_NAME = 'superproject.git'
 _SUPERPROJECT_MANIFEST_NAME = 'superproject_override.xml'
 
 
+class SyncResult(NamedTuple):
+  """Return the status of sync and whether caller should exit."""
+
+  # Whether the superproject sync was successful.
+  success: bool
+  # Whether the caller should exit.
+  fatal: bool
+
+
+class CommitIdsResult(NamedTuple):
+  """Return the commit ids and whether caller should exit."""
+
+  # A dictionary with the projects/commit ids on success, otherwise None.
+  commit_ids: dict
+  # Whether the caller should exit.
+  fatal: bool
+
+
+class UpdateProjectsResult(NamedTuple):
+  """Return the overriding manifest file and whether caller should exit."""
+
+  # Path name of the overriding manifest file if successful, otherwise None.
+  manifest_path: str
+  # Whether the caller should exit.
+  fatal: bool
+
+
 class Superproject(object):
   """Get commit ids from superproject.
 
@@ -41,21 +72,25 @@ class Superproject(object):
   lookup of commit ids for all projects. It contains _project_commit_ids which
   is a dictionary with project/commit id entries.
   """
-  def __init__(self, manifest, repodir, superproject_dir='exp-superproject',
-               quiet=False):
+  def __init__(self, manifest, repodir, git_event_log,
+               superproject_dir='exp-superproject', quiet=False, print_messages=False):
     """Initializes superproject.
 
     Args:
       manifest: A Manifest object that is to be written to a file.
       repodir: Path to the .repo/ dir for holding all internal checkout state.
           It must be in the top directory of the repo client checkout.
+      git_event_log: A git trace2 event log to log events.
       superproject_dir: Relative path under |repodir| to checkout superproject.
       quiet:  If True then only print the progress messages.
+      print_messages: if True then print error/warning messages.
     """
     self._project_commit_ids = None
     self._manifest = manifest
+    self._git_event_log = git_event_log
     self._quiet = quiet
-    self._branch = self._GetBranch()
+    self._print_messages = print_messages
+    self._branch = manifest.branch
     self._repodir = os.path.abspath(repodir)
     self._superproject_dir = superproject_dir
     self._superproject_path = os.path.join(self._repodir, superproject_dir)
@@ -63,8 +98,12 @@ class Superproject(object):
                                        _SUPERPROJECT_MANIFEST_NAME)
     git_name = ''
     if self._manifest.superproject:
-      remote_name = self._manifest.superproject['remote'].name
-      git_name = hashlib.md5(remote_name.encode('utf8')).hexdigest() + '-'
+      remote = self._manifest.superproject['remote']
+      git_name = hashlib.md5(remote.name.encode('utf8')).hexdigest() + '-'
+      self._branch = self._manifest.superproject['revision']
+      self._remote_url = remote.url
+    else:
+      self._remote_url = None
     self._work_git_name = git_name + _SUPERPROJECT_GIT_NAME
     self._work_git = os.path.join(self._superproject_path, self._work_git_name)
 
@@ -73,16 +112,28 @@ class Superproject(object):
     """Returns a dictionary of projects and their commit ids."""
     return self._project_commit_ids
 
-  def _GetBranch(self):
-    """Returns the branch name for getting the approved manifest."""
-    p = self._manifest.manifestProject
-    b = p.GetBranch(p.CurrentBranch)
-    if not b:
-      return None
-    branch = b.merge
-    if branch and branch.startswith(R_HEADS):
-      branch = branch[len(R_HEADS):]
-    return branch
+  @property
+  def manifest_path(self):
+    """Returns the manifest path if the path exists or None."""
+    return self._manifest_path if os.path.exists(self._manifest_path) else None
+
+  def _LogMessage(self, message):
+    """Logs message to stderr and _git_event_log."""
+    if self._print_messages:
+      print(message, file=sys.stderr)
+    self._git_event_log.ErrorEvent(message, f'{message}')
+
+  def _LogMessagePrefix(self):
+    """Returns the prefix string to be logged in each log message"""
+    return f'repo superproject branch: {self._branch} url: {self._remote_url}'
+
+  def _LogError(self, message):
+    """Logs error message to stderr and _git_event_log."""
+    self._LogMessage(f'{self._LogMessagePrefix()} error: {message}')
+
+  def _LogWarning(self, message):
+    """Logs warning message to stderr and _git_event_log."""
+    self._LogMessage(f'{self._LogMessagePrefix()} warning: {message}')
 
   def _Init(self):
     """Sets up a local Git repository to get a copy of a superproject.
@@ -103,25 +154,25 @@ class Superproject(object):
                    capture_stderr=True)
     retval = p.Wait()
     if retval:
-      print('repo: error: git init call failed with return code: %r, stderr: %r' %
-            (retval, p.stderr), file=sys.stderr)
+      self._LogWarning(f'git init call failed, command: git {cmd}, '
+                       f'return code: {retval}, stderr: {p.stderr}')
       return False
     return True
 
-  def _Fetch(self, url):
-    """Fetches a local copy of a superproject for the manifest based on url.
-
-    Args:
-      url: superproject's url.
+  def _Fetch(self):
+    """Fetches a local copy of a superproject for the manifest based on |_remote_url|.
 
     Returns:
       True if fetch is successful, or False.
     """
     if not os.path.exists(self._work_git):
-      print('git fetch missing drectory: %s' % self._work_git,
-            file=sys.stderr)
+      self._LogWarning(f'git fetch missing directory: {self._work_git}')
       return False
-    cmd = ['fetch', url, '--depth', '1', '--force', '--no-tags', '--filter', 'blob:none']
+    if not git_require((2, 28, 0)):
+      self._LogWarning('superproject requires a git version 2.28 or later')
+      return False
+    cmd = ['fetch', self._remote_url, '--depth', '1', '--force', '--no-tags',
+           '--filter', 'blob:none']
     if self._branch:
       cmd += [self._branch + ':' + self._branch]
     p = GitCommand(None,
@@ -131,8 +182,8 @@ class Superproject(object):
                    capture_stderr=True)
     retval = p.Wait()
     if retval:
-      print('repo: error: git fetch call failed with return code: %r, stderr: %r' %
-            (retval, p.stderr), file=sys.stderr)
+      self._LogWarning(f'git fetch call failed, command: git {cmd}, '
+                       f'return code: {retval}, stderr: {p.stderr}')
       return False
     return True
 
@@ -145,8 +196,7 @@ class Superproject(object):
       data: data returned from 'git ls-tree ...' instead of None.
     """
     if not os.path.exists(self._work_git):
-      print('git ls-tree missing drectory: %s' % self._work_git,
-            file=sys.stderr)
+      self._LogWarning(f'git ls-tree missing directory: {self._work_git}')
       return None
     data = None
     branch = 'HEAD' if not self._branch else self._branch
@@ -161,52 +211,52 @@ class Superproject(object):
     if retval == 0:
       data = p.stdout
     else:
-      print('repo: error: git ls-tree call failed with return code: %r, stderr: %r' % (
-          retval, p.stderr), file=sys.stderr)
+      self._LogWarning(f'git ls-tree call failed, command: git {cmd}, '
+                       f'return code: {retval}, stderr: {p.stderr}')
     return data
 
   def Sync(self):
     """Gets a local copy of a superproject for the manifest.
 
     Returns:
-      True if sync of superproject is successful, or False.
+      SyncResult
     """
-    print('WARNING: --use-superproject is experimental and not '
-          'for general use', file=sys.stderr)
-
     if not self._manifest.superproject:
-      print('error: superproject tag is not defined in manifest',
-            file=sys.stderr)
-      return False
-
-    url = self._manifest.superproject['remote'].url
-    if not url:
-      print('error: superproject URL is not defined in manifest',
-            file=sys.stderr)
-      return False
+      self._LogWarning(f'superproject tag is not defined in manifest: '
+                       f'{self._manifest.manifestFile}')
+      return SyncResult(False, False)
+
+    print('NOTICE: --use-superproject is in beta; report any issues to the '
+          'address described in `repo version`', file=sys.stderr)
+    should_exit = True
+    if not self._remote_url:
+      self._LogWarning(f'superproject URL is not defined in manifest: '
+                       f'{self._manifest.manifestFile}')
+      return SyncResult(False, should_exit)
 
     if not self._Init():
-      return False
-    if not self._Fetch(url):
-      return False
+      return SyncResult(False, should_exit)
+    if not self._Fetch():
+      return SyncResult(False, should_exit)
     if not self._quiet:
       print('%s: Initial setup for superproject completed.' % self._work_git)
-    return True
+    return SyncResult(True, False)
 
   def _GetAllProjectsCommitIds(self):
     """Get commit ids for all projects from superproject and save them in _project_commit_ids.
 
     Returns:
-      A dictionary with the projects/commit ids on success, otherwise None.
+      CommitIdsResult
     """
-    if not self.Sync():
-      return None
+    sync_result = self.Sync()
+    if not sync_result.success:
+      return CommitIdsResult(None, sync_result.fatal)
 
     data = self._LsTree()
     if not data:
-      print('error: git ls-tree failed to return data for superproject',
-            file=sys.stderr)
-      return None
+      self._LogWarning(f'git ls-tree failed to return data for manifest: '
+                       f'{self._manifest.manifestFile}')
+      return CommitIdsResult(None, True)
 
     # Parse lines like the following to select lines starting with '160000' and
     # build a dictionary with project path (last element) and its commit id (3rd element).
@@ -222,18 +272,16 @@ class Superproject(object):
         commit_ids[ls_data[3]] = ls_data[2]
 
     self._project_commit_ids = commit_ids
-    return commit_ids
+    return CommitIdsResult(commit_ids, False)
 
-  def _WriteManfiestFile(self):
+  def _WriteManifestFile(self):
     """Writes manifest to a file.
 
     Returns:
       manifest_path: Path name of the file into which manifest is written instead of None.
     """
     if not os.path.exists(self._superproject_path):
-      print('error: missing superproject directory %s' %
-            self._superproject_path,
-            file=sys.stderr)
+      self._LogWarning(f'missing superproject directory: {self._superproject_path}')
       return None
     manifest_str = self._manifest.ToXml(groups=self._manifest.GetGroupsStr()).toxml()
     manifest_path = self._manifest_path
@@ -241,12 +289,30 @@ class Superproject(object):
       with open(manifest_path, 'w', encoding='utf-8') as fp:
         fp.write(manifest_str)
     except IOError as e:
-      print('error: cannot write manifest to %s:\n%s'
-            % (manifest_path, e),
-            file=sys.stderr)
+      self._LogError(f'cannot write manifest to : {manifest_path} {e}')
       return None
     return manifest_path
 
+  def _SkipUpdatingProjectRevisionId(self, project):
+    """Checks if a project's revision id needs to be updated or not.
+
+    Revision id for projects from local manifest will not be updated.
+
+    Args:
+      project: project whose revision id is being updated.
+
+    Returns:
+      True if a project's revision id should not be updated, or False,
+    """
+    path = project.relpath
+    if not path:
+      return True
+    # Skip the project with revisionId.
+    if project.revisionId:
+      return True
+    # Skip the project if it comes from the local manifest.
+    return any(s.startswith(LOCAL_MANIFEST_GROUP_PREFIX) for s in project.groups)
+
   def UpdateProjectsRevisionId(self, projects):
     """Update revisionId of every project in projects with the commit id.
 
@@ -254,27 +320,96 @@ class Superproject(object):
       projects: List of projects whose revisionId needs to be updated.
 
     Returns:
-      manifest_path: Path name of the overriding manfiest file instead of None.
+      UpdateProjectsResult
     """
-    commit_ids = self._GetAllProjectsCommitIds()
+    commit_ids_result = self._GetAllProjectsCommitIds()
+    commit_ids = commit_ids_result.commit_ids
     if not commit_ids:
-      print('error: Cannot get project commit ids from manifest', file=sys.stderr)
-      return None
+      return UpdateProjectsResult(None, commit_ids_result.fatal)
 
     projects_missing_commit_ids = []
     for project in projects:
-      path = project.relpath
-      if not path:
+      if self._SkipUpdatingProjectRevisionId(project):
         continue
+      path = project.relpath
       commit_id = commit_ids.get(path)
-      if commit_id:
-        project.SetRevisionId(commit_id)
-      else:
+      if not commit_id:
         projects_missing_commit_ids.append(path)
+
+    # If superproject doesn't have a commit id for a project, then report an
+    # error event and continue as if do not use superproject is specified.
     if projects_missing_commit_ids:
-      print('error: please file a bug using %s to report missing commit_ids for: %s' %
-            (BUG_REPORT_URL, projects_missing_commit_ids), file=sys.stderr)
-      return None
+      self._LogWarning(f'please file a bug using {self._manifest.contactinfo.bugurl} '
+                       f'to report missing commit_ids for: {projects_missing_commit_ids}')
+      return UpdateProjectsResult(None, False)
 
-    manifest_path = self._WriteManfiestFile()
-    return manifest_path
+    for project in projects:
+      if not self._SkipUpdatingProjectRevisionId(project):
+        project.SetRevisionId(commit_ids.get(project.relpath))
+
+    manifest_path = self._WriteManifestFile()
+    return UpdateProjectsResult(manifest_path, False)
+
+
+@functools.lru_cache(maxsize=None)
+def _UseSuperprojectFromConfiguration():
+  """Returns the user choice of whether to use superproject."""
+  user_cfg = RepoConfig.ForUser()
+  time_now = int(time.time())
+
+  user_value = user_cfg.GetBoolean('repo.superprojectChoice')
+  if user_value is not None:
+    user_expiration = user_cfg.GetInt('repo.superprojectChoiceExpire')
+    if user_expiration is None or user_expiration <= 0 or user_expiration >= time_now:
+      # TODO(b/190688390) - Remove prompt when we are comfortable with the new
+      # default value.
+      if user_value:
+        print(('You are currently enrolled in Git submodules experiment '
+               '(go/android-submodules-quickstart).  Use --no-use-superproject '
+               'to override.\n'), file=sys.stderr)
+      else:
+        print(('You are not currently enrolled in Git submodules experiment '
+               '(go/android-submodules-quickstart).  Use --use-superproject '
+               'to override.\n'), file=sys.stderr)
+      return user_value
+
+  # We don't have an unexpired choice, ask for one.
+  system_cfg = RepoConfig.ForSystem()
+  system_value = system_cfg.GetBoolean('repo.superprojectChoice')
+  if system_value:
+    # The system configuration is proposing that we should enable the
+    # use of superproject. Treat the user as enrolled for two weeks.
+    #
+    # TODO(b/190688390) - Remove prompt when we are comfortable with the new
+    # default value.
+    userchoice = True
+    time_choiceexpire = time_now + (86400 * 14)
+    user_cfg.SetString('repo.superprojectChoiceExpire', str(time_choiceexpire))
+    user_cfg.SetBoolean('repo.superprojectChoice', userchoice)
+    print('You are automatically enrolled in Git submodules experiment '
+          '(go/android-submodules-quickstart) for another two weeks.\n',
+          file=sys.stderr)
+    return True
+
+  # For all other cases, we would not use superproject by default.
+  return False
+
+
+def PrintMessages(opt, manifest):
+  """Returns a boolean if error/warning messages are to be printed."""
+  return opt.use_superproject is not None or manifest.superproject
+
+
+def UseSuperproject(opt, manifest):
+  """Returns a boolean if use-superproject option is enabled."""
+
+  if opt.use_superproject is not None:
+    return opt.use_superproject
+  else:
+    client_value = manifest.manifestProject.config.GetBoolean('repo.superproject')
+    if client_value is not None:
+      return client_value
+    else:
+      if not manifest.superproject:
+        return False
+      return _UseSuperprojectFromConfiguration()