From ea2e330e43c182dc16b0111ebc69ee5a71ee4ce1 Mon Sep 17 00:00:00 2001 From: Gavin Mak Date: Sat, 11 Mar 2023 06:46:20 +0000 Subject: Format codebase with black and check formatting in CQ Apply rules set by https://gerrit-review.googlesource.com/c/git-repo/+/362954/ across the codebase and fix any lingering errors caught by flake8. Also check black formatting in run_tests (and CQ). Bug: b/267675342 Change-Id: I972d77649dac351150dcfeb1cd1ad0ea2efc1956 Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/363474 Reviewed-by: Mike Frysinger Tested-by: Gavin Mak Commit-Queue: Gavin Mak --- project.py | 7874 ++++++++++++++++++++++++++++++++---------------------------- 1 file changed, 4169 insertions(+), 3705 deletions(-) (limited to 'project.py') diff --git a/project.py b/project.py index 3ccfd140..887fe83a 100644 --- a/project.py +++ b/project.py @@ -32,8 +32,13 @@ import urllib.parse from color import Coloring import fetch from git_command import GitCommand, git_require -from git_config import GitConfig, IsId, GetSchemeFromUrl, GetUrlCookieFile, \ - ID_RE +from git_config import ( + GitConfig, + IsId, + GetSchemeFromUrl, + GetUrlCookieFile, + ID_RE, +) import git_superproject from git_trace2_event_log import EventLog from error import GitError, UploadError, DownloadError @@ -47,12 +52,13 @@ from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M, R_WORKTREE_M class SyncNetworkHalfResult(NamedTuple): - """Sync_NetworkHalf return value.""" - # True if successful. - success: bool - # Did we query the remote? False when optimized_fetch is True and we have the - # commit already present. - remote_fetched: bool + """Sync_NetworkHalf return value.""" + + # True if successful. + success: bool + # Did we query the remote? False when optimized_fetch is True and we have + # the commit already present. + remote_fetched: bool # Maximum sleep time allowed during retries. @@ -62,3904 +68,4362 @@ RETRY_JITTER_PERCENT = 0.1 # Whether to use alternates. Switching back and forth is *NOT* supported. # TODO(vapier): Remove knob once behavior is verified. -_ALTERNATES = os.environ.get('REPO_USE_ALTERNATES') == '1' +_ALTERNATES = os.environ.get("REPO_USE_ALTERNATES") == "1" def _lwrite(path, content): - lock = '%s.lock' % path + lock = "%s.lock" % path - # Maintain Unix line endings on all OS's to match git behavior. - with open(lock, 'w', newline='\n') as fd: - fd.write(content) + # Maintain Unix line endings on all OS's to match git behavior. + with open(lock, "w", newline="\n") as fd: + fd.write(content) - try: - platform_utils.rename(lock, path) - except OSError: - platform_utils.remove(lock) - raise + try: + platform_utils.rename(lock, path) + except OSError: + platform_utils.remove(lock) + raise def _error(fmt, *args): - msg = fmt % args - print('error: %s' % msg, file=sys.stderr) + msg = fmt % args + print("error: %s" % msg, file=sys.stderr) def _warn(fmt, *args): - msg = fmt % args - print('warn: %s' % msg, file=sys.stderr) + msg = fmt % args + print("warn: %s" % msg, file=sys.stderr) def not_rev(r): - return '^' + r + return "^" + r def sq(r): - return "'" + r.replace("'", "'\''") + "'" + return "'" + r.replace("'", "'''") + "'" _project_hook_list = None def _ProjectHooks(): - """List the hooks present in the 'hooks' directory. + """List the hooks present in the 'hooks' directory. - These hooks are project hooks and are copied to the '.git/hooks' directory - of all subprojects. + These hooks are project hooks and are copied to the '.git/hooks' directory + of all subprojects. - This function caches the list of hooks (based on the contents of the - 'repo/hooks' directory) on the first call. + This function caches the list of hooks (based on the contents of the + 'repo/hooks' directory) on the first call. - Returns: - A list of absolute paths to all of the files in the hooks directory. - """ - global _project_hook_list - if _project_hook_list is None: - d = platform_utils.realpath(os.path.abspath(os.path.dirname(__file__))) - d = os.path.join(d, 'hooks') - _project_hook_list = [os.path.join(d, x) for x in platform_utils.listdir(d)] - return _project_hook_list + Returns: + A list of absolute paths to all of the files in the hooks directory. + """ + global _project_hook_list + if _project_hook_list is None: + d = platform_utils.realpath(os.path.abspath(os.path.dirname(__file__))) + d = os.path.join(d, "hooks") + _project_hook_list = [ + os.path.join(d, x) for x in platform_utils.listdir(d) + ] + return _project_hook_list class DownloadedChange(object): - _commit_cache = None - - def __init__(self, project, base, change_id, ps_id, commit): - self.project = project - self.base = base - self.change_id = change_id - self.ps_id = ps_id - self.commit = commit - - @property - def commits(self): - if self._commit_cache is None: - self._commit_cache = self.project.bare_git.rev_list('--abbrev=8', - '--abbrev-commit', - '--pretty=oneline', - '--reverse', - '--date-order', - not_rev(self.base), - self.commit, - '--') - return self._commit_cache + _commit_cache = None + + def __init__(self, project, base, change_id, ps_id, commit): + self.project = project + self.base = base + self.change_id = change_id + self.ps_id = ps_id + self.commit = commit + + @property + def commits(self): + if self._commit_cache is None: + self._commit_cache = self.project.bare_git.rev_list( + "--abbrev=8", + "--abbrev-commit", + "--pretty=oneline", + "--reverse", + "--date-order", + not_rev(self.base), + self.commit, + "--", + ) + return self._commit_cache class ReviewableBranch(object): - _commit_cache = None - _base_exists = None - - def __init__(self, project, branch, base): - self.project = project - self.branch = branch - self.base = base - - @property - def name(self): - return self.branch.name - - @property - def commits(self): - if self._commit_cache is None: - args = ('--abbrev=8', '--abbrev-commit', '--pretty=oneline', '--reverse', - '--date-order', not_rev(self.base), R_HEADS + self.name, '--') - try: - self._commit_cache = self.project.bare_git.rev_list(*args) - except GitError: - # We weren't able to probe the commits for this branch. Was it tracking - # a branch that no longer exists? If so, return no commits. Otherwise, - # rethrow the error as we don't know what's going on. - if self.base_exists: - raise - - self._commit_cache = [] - - return self._commit_cache - - @property - def unabbrev_commits(self): - r = dict() - for commit in self.project.bare_git.rev_list(not_rev(self.base), - R_HEADS + self.name, - '--'): - r[commit[0:8]] = commit - return r - - @property - def date(self): - return self.project.bare_git.log('--pretty=format:%cd', - '-n', '1', - R_HEADS + self.name, - '--') - - @property - def base_exists(self): - """Whether the branch we're tracking exists. - - Normally it should, but sometimes branches we track can get deleted. - """ - if self._base_exists is None: - try: - self.project.bare_git.rev_parse('--verify', not_rev(self.base)) - # If we're still here, the base branch exists. - self._base_exists = True - except GitError: - # If we failed to verify, the base branch doesn't exist. - self._base_exists = False - - return self._base_exists - - def UploadForReview(self, people, - dryrun=False, - auto_topic=False, - hashtags=(), - labels=(), - private=False, - notify=None, - wip=False, - ready=False, - dest_branch=None, - validate_certs=True, - push_options=None): - self.project.UploadForReview(branch=self.name, - people=people, - dryrun=dryrun, - auto_topic=auto_topic, - hashtags=hashtags, - labels=labels, - private=private, - notify=notify, - wip=wip, - ready=ready, - dest_branch=dest_branch, - validate_certs=validate_certs, - push_options=push_options) - - def GetPublishedRefs(self): - refs = {} - output = self.project.bare_git.ls_remote( - self.branch.remote.SshReviewUrl(self.project.UserEmail), - 'refs/changes/*') - for line in output.split('\n'): - try: - (sha, ref) = line.split() - refs[sha] = ref - except ValueError: - pass - - return refs + _commit_cache = None + _base_exists = None + + def __init__(self, project, branch, base): + self.project = project + self.branch = branch + self.base = base + + @property + def name(self): + return self.branch.name + + @property + def commits(self): + if self._commit_cache is None: + args = ( + "--abbrev=8", + "--abbrev-commit", + "--pretty=oneline", + "--reverse", + "--date-order", + not_rev(self.base), + R_HEADS + self.name, + "--", + ) + try: + self._commit_cache = self.project.bare_git.rev_list(*args) + except GitError: + # We weren't able to probe the commits for this branch. Was it + # tracking a branch that no longer exists? If so, return no + # commits. Otherwise, rethrow the error as we don't know what's + # going on. + if self.base_exists: + raise + + self._commit_cache = [] + + return self._commit_cache + + @property + def unabbrev_commits(self): + r = dict() + for commit in self.project.bare_git.rev_list( + not_rev(self.base), R_HEADS + self.name, "--" + ): + r[commit[0:8]] = commit + return r + + @property + def date(self): + return self.project.bare_git.log( + "--pretty=format:%cd", "-n", "1", R_HEADS + self.name, "--" + ) + @property + def base_exists(self): + """Whether the branch we're tracking exists. -class StatusColoring(Coloring): + Normally it should, but sometimes branches we track can get deleted. + """ + if self._base_exists is None: + try: + self.project.bare_git.rev_parse("--verify", not_rev(self.base)) + # If we're still here, the base branch exists. + self._base_exists = True + except GitError: + # If we failed to verify, the base branch doesn't exist. + self._base_exists = False + + return self._base_exists + + def UploadForReview( + self, + people, + dryrun=False, + auto_topic=False, + hashtags=(), + labels=(), + private=False, + notify=None, + wip=False, + ready=False, + dest_branch=None, + validate_certs=True, + push_options=None, + ): + self.project.UploadForReview( + branch=self.name, + people=people, + dryrun=dryrun, + auto_topic=auto_topic, + hashtags=hashtags, + labels=labels, + private=private, + notify=notify, + wip=wip, + ready=ready, + dest_branch=dest_branch, + validate_certs=validate_certs, + push_options=push_options, + ) - def __init__(self, config): - super().__init__(config, 'status') - self.project = self.printer('header', attr='bold') - self.branch = self.printer('header', attr='bold') - self.nobranch = self.printer('nobranch', fg='red') - self.important = self.printer('important', fg='red') + def GetPublishedRefs(self): + refs = {} + output = self.project.bare_git.ls_remote( + self.branch.remote.SshReviewUrl(self.project.UserEmail), + "refs/changes/*", + ) + for line in output.split("\n"): + try: + (sha, ref) = line.split() + refs[sha] = ref + except ValueError: + pass - self.added = self.printer('added', fg='green') - self.changed = self.printer('changed', fg='red') - self.untracked = self.printer('untracked', fg='red') + return refs -class DiffColoring(Coloring): +class StatusColoring(Coloring): + def __init__(self, config): + super().__init__(config, "status") + self.project = self.printer("header", attr="bold") + self.branch = self.printer("header", attr="bold") + self.nobranch = self.printer("nobranch", fg="red") + self.important = self.printer("important", fg="red") - def __init__(self, config): - super().__init__(config, 'diff') - self.project = self.printer('header', attr='bold') - self.fail = self.printer('fail', fg='red') + self.added = self.printer("added", fg="green") + self.changed = self.printer("changed", fg="red") + self.untracked = self.printer("untracked", fg="red") -class Annotation(object): +class DiffColoring(Coloring): + def __init__(self, config): + super().__init__(config, "diff") + self.project = self.printer("header", attr="bold") + self.fail = self.printer("fail", fg="red") + - def __init__(self, name, value, keep): - self.name = name - self.value = value - self.keep = keep +class Annotation(object): + def __init__(self, name, value, keep): + self.name = name + self.value = value + self.keep = keep - def __eq__(self, other): - if not isinstance(other, Annotation): - return False - return self.__dict__ == other.__dict__ + def __eq__(self, other): + if not isinstance(other, Annotation): + return False + return self.__dict__ == other.__dict__ - def __lt__(self, other): - # This exists just so that lists of Annotation objects can be sorted, for - # use in comparisons. - if not isinstance(other, Annotation): - raise ValueError('comparison is not between two Annotation objects') - if self.name == other.name: - if self.value == other.value: - return self.keep < other.keep - return self.value < other.value - return self.name < other.name + def __lt__(self, other): + # This exists just so that lists of Annotation objects can be sorted, + # for use in comparisons. + if not isinstance(other, Annotation): + raise ValueError("comparison is not between two Annotation objects") + if self.name == other.name: + if self.value == other.value: + return self.keep < other.keep + return self.value < other.value + return self.name < other.name def _SafeExpandPath(base, subpath, skipfinal=False): - """Make sure |subpath| is completely safe under |base|. - - We make sure no intermediate symlinks are traversed, and that the final path - is not a special file (e.g. not a socket or fifo). - - NB: We rely on a number of paths already being filtered out while parsing the - manifest. See the validation logic in manifest_xml.py for more details. - """ - # Split up the path by its components. We can't use os.path.sep exclusively - # as some platforms (like Windows) will convert / to \ and that bypasses all - # our constructed logic here. Especially since manifest authors only use - # / in their paths. - resep = re.compile(r'[/%s]' % re.escape(os.path.sep)) - components = resep.split(subpath) - if skipfinal: - # Whether the caller handles the final component itself. - finalpart = components.pop() - - path = base - for part in components: - if part in {'.', '..'}: - raise ManifestInvalidPathError( - '%s: "%s" not allowed in paths' % (subpath, part)) - - path = os.path.join(path, part) - if platform_utils.islink(path): - raise ManifestInvalidPathError( - '%s: traversing symlinks not allow' % (path,)) - - if os.path.exists(path): - if not os.path.isfile(path) and not platform_utils.isdir(path): - raise ManifestInvalidPathError( - '%s: only regular files & directories allowed' % (path,)) - - if skipfinal: - path = os.path.join(path, finalpart) - - return path + """Make sure |subpath| is completely safe under |base|. + + We make sure no intermediate symlinks are traversed, and that the final path + is not a special file (e.g. not a socket or fifo). + + NB: We rely on a number of paths already being filtered out while parsing + the manifest. See the validation logic in manifest_xml.py for more details. + """ + # Split up the path by its components. We can't use os.path.sep exclusively + # as some platforms (like Windows) will convert / to \ and that bypasses all + # our constructed logic here. Especially since manifest authors only use + # / in their paths. + resep = re.compile(r"[/%s]" % re.escape(os.path.sep)) + components = resep.split(subpath) + if skipfinal: + # Whether the caller handles the final component itself. + finalpart = components.pop() + + path = base + for part in components: + if part in {".", ".."}: + raise ManifestInvalidPathError( + '%s: "%s" not allowed in paths' % (subpath, part) + ) + + path = os.path.join(path, part) + if platform_utils.islink(path): + raise ManifestInvalidPathError( + "%s: traversing symlinks not allow" % (path,) + ) + + if os.path.exists(path): + if not os.path.isfile(path) and not platform_utils.isdir(path): + raise ManifestInvalidPathError( + "%s: only regular files & directories allowed" % (path,) + ) + + if skipfinal: + path = os.path.join(path, finalpart) + + return path class _CopyFile(object): - """Container for manifest element.""" + """Container for manifest element.""" - def __init__(self, git_worktree, src, topdir, dest): - """Register a request. + def __init__(self, git_worktree, src, topdir, dest): + """Register a request. - Args: - git_worktree: Absolute path to the git project checkout. - src: Relative path under |git_worktree| of file to read. - topdir: Absolute path to the top of the repo client checkout. - dest: Relative path under |topdir| of file to write. - """ - self.git_worktree = git_worktree - self.topdir = topdir - self.src = src - self.dest = dest - - def _Copy(self): - src = _SafeExpandPath(self.git_worktree, self.src) - dest = _SafeExpandPath(self.topdir, self.dest) - - if platform_utils.isdir(src): - raise ManifestInvalidPathError( - '%s: copying from directory not supported' % (self.src,)) - if platform_utils.isdir(dest): - raise ManifestInvalidPathError( - '%s: copying to directory not allowed' % (self.dest,)) - - # copy file if it does not exist or is out of date - if not os.path.exists(dest) or not filecmp.cmp(src, dest): - try: - # remove existing file first, since it might be read-only - if os.path.exists(dest): - platform_utils.remove(dest) - else: - dest_dir = os.path.dirname(dest) - if not platform_utils.isdir(dest_dir): - os.makedirs(dest_dir) - shutil.copy(src, dest) - # make the file read-only - mode = os.stat(dest)[stat.ST_MODE] - mode = mode & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH) - os.chmod(dest, mode) - except IOError: - _error('Cannot copy file %s to %s', src, dest) + Args: + git_worktree: Absolute path to the git project checkout. + src: Relative path under |git_worktree| of file to read. + topdir: Absolute path to the top of the repo client checkout. + dest: Relative path under |topdir| of file to write. + """ + self.git_worktree = git_worktree + self.topdir = topdir + self.src = src + self.dest = dest + + def _Copy(self): + src = _SafeExpandPath(self.git_worktree, self.src) + dest = _SafeExpandPath(self.topdir, self.dest) + + if platform_utils.isdir(src): + raise ManifestInvalidPathError( + "%s: copying from directory not supported" % (self.src,) + ) + if platform_utils.isdir(dest): + raise ManifestInvalidPathError( + "%s: copying to directory not allowed" % (self.dest,) + ) + + # Copy file if it does not exist or is out of date. + if not os.path.exists(dest) or not filecmp.cmp(src, dest): + try: + # Remove existing file first, since it might be read-only. + if os.path.exists(dest): + platform_utils.remove(dest) + else: + dest_dir = os.path.dirname(dest) + if not platform_utils.isdir(dest_dir): + os.makedirs(dest_dir) + shutil.copy(src, dest) + # Make the file read-only. + mode = os.stat(dest)[stat.ST_MODE] + mode = mode & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH) + os.chmod(dest, mode) + except IOError: + _error("Cannot copy file %s to %s", src, dest) class _LinkFile(object): - """Container for manifest element.""" + """Container for manifest element.""" - def __init__(self, git_worktree, src, topdir, dest): - """Register a request. + def __init__(self, git_worktree, src, topdir, dest): + """Register a request. - Args: - git_worktree: Absolute path to the git project checkout. - src: Target of symlink relative to path under |git_worktree|. - topdir: Absolute path to the top of the repo client checkout. - dest: Relative path under |topdir| of symlink to create. - """ - self.git_worktree = git_worktree - self.topdir = topdir - self.src = src - self.dest = dest - - def __linkIt(self, relSrc, absDest): - # link file if it does not exist or is out of date - if not platform_utils.islink(absDest) or (platform_utils.readlink(absDest) != relSrc): - try: - # remove existing file first, since it might be read-only - if os.path.lexists(absDest): - platform_utils.remove(absDest) + Args: + git_worktree: Absolute path to the git project checkout. + src: Target of symlink relative to path under |git_worktree|. + topdir: Absolute path to the top of the repo client checkout. + dest: Relative path under |topdir| of symlink to create. + """ + self.git_worktree = git_worktree + self.topdir = topdir + self.src = src + self.dest = dest + + def __linkIt(self, relSrc, absDest): + # Link file if it does not exist or is out of date. + if not platform_utils.islink(absDest) or ( + platform_utils.readlink(absDest) != relSrc + ): + try: + # Remove existing file first, since it might be read-only. + if os.path.lexists(absDest): + platform_utils.remove(absDest) + else: + dest_dir = os.path.dirname(absDest) + if not platform_utils.isdir(dest_dir): + os.makedirs(dest_dir) + platform_utils.symlink(relSrc, absDest) + except IOError: + _error("Cannot link file %s to %s", relSrc, absDest) + + def _Link(self): + """Link the self.src & self.dest paths. + + Handles wild cards on the src linking all of the files in the source in + to the destination directory. + """ + # Some people use src="." to create stable links to projects. Let's + # allow that but reject all other uses of "." to keep things simple. + if self.src == ".": + src = self.git_worktree else: - dest_dir = os.path.dirname(absDest) - if not platform_utils.isdir(dest_dir): - os.makedirs(dest_dir) - platform_utils.symlink(relSrc, absDest) - except IOError: - _error('Cannot link file %s to %s', relSrc, absDest) - - def _Link(self): - """Link the self.src & self.dest paths. - - Handles wild cards on the src linking all of the files in the source in to - the destination directory. - """ - # Some people use src="." to create stable links to projects. Lets allow - # that but reject all other uses of "." to keep things simple. - if self.src == '.': - src = self.git_worktree - else: - src = _SafeExpandPath(self.git_worktree, self.src) - - if not glob.has_magic(src): - # Entity does not contain a wild card so just a simple one to one link operation. - dest = _SafeExpandPath(self.topdir, self.dest, skipfinal=True) - # dest & src are absolute paths at this point. Make sure the target of - # the symlink is relative in the context of the repo client checkout. - relpath = os.path.relpath(src, os.path.dirname(dest)) - self.__linkIt(relpath, dest) - else: - dest = _SafeExpandPath(self.topdir, self.dest) - # Entity contains a wild card. - if os.path.exists(dest) and not platform_utils.isdir(dest): - _error('Link error: src with wildcard, %s must be a directory', dest) - else: - for absSrcFile in glob.glob(src): - # Create a releative path from source dir to destination dir - absSrcDir = os.path.dirname(absSrcFile) - relSrcDir = os.path.relpath(absSrcDir, dest) - - # Get the source file name - srcFile = os.path.basename(absSrcFile) - - # Now form the final full paths to srcFile. They will be - # absolute for the desintaiton and relative for the srouce. - absDest = os.path.join(dest, srcFile) - relSrc = os.path.join(relSrcDir, srcFile) - self.__linkIt(relSrc, absDest) + src = _SafeExpandPath(self.git_worktree, self.src) + + if not glob.has_magic(src): + # Entity does not contain a wild card so just a simple one to one + # link operation. + dest = _SafeExpandPath(self.topdir, self.dest, skipfinal=True) + # dest & src are absolute paths at this point. Make sure the target + # of the symlink is relative in the context of the repo client + # checkout. + relpath = os.path.relpath(src, os.path.dirname(dest)) + self.__linkIt(relpath, dest) + else: + dest = _SafeExpandPath(self.topdir, self.dest) + # Entity contains a wild card. + if os.path.exists(dest) and not platform_utils.isdir(dest): + _error( + "Link error: src with wildcard, %s must be a directory", + dest, + ) + else: + for absSrcFile in glob.glob(src): + # Create a releative path from source dir to destination + # dir. + absSrcDir = os.path.dirname(absSrcFile) + relSrcDir = os.path.relpath(absSrcDir, dest) + + # Get the source file name. + srcFile = os.path.basename(absSrcFile) + + # Now form the final full paths to srcFile. They will be + # absolute for the desintaiton and relative for the source. + absDest = os.path.join(dest, srcFile) + relSrc = os.path.join(relSrcDir, srcFile) + self.__linkIt(relSrc, absDest) class RemoteSpec(object): - - def __init__(self, - name, - url=None, - pushUrl=None, - review=None, - revision=None, - orig_name=None, - fetchUrl=None): - self.name = name - self.url = url - self.pushUrl = pushUrl - self.review = review - self.revision = revision - self.orig_name = orig_name - self.fetchUrl = fetchUrl + def __init__( + self, + name, + url=None, + pushUrl=None, + review=None, + revision=None, + orig_name=None, + fetchUrl=None, + ): + self.name = name + self.url = url + self.pushUrl = pushUrl + self.review = review + self.revision = revision + self.orig_name = orig_name + self.fetchUrl = fetchUrl class Project(object): - # These objects can be shared between several working trees. - @property - def shareable_dirs(self): - """Return the shareable directories""" - if self.UseAlternates: - return ['hooks', 'rr-cache'] - else: - return ['hooks', 'objects', 'rr-cache'] - - def __init__(self, - manifest, - name, - remote, - gitdir, - objdir, - worktree, - relpath, - revisionExpr, - revisionId, - rebase=True, - groups=None, - sync_c=False, - sync_s=False, - sync_tags=True, - clone_depth=None, - upstream=None, - parent=None, - use_git_worktrees=False, - is_derived=False, - dest_branch=None, - optimized_fetch=False, - retry_fetches=0, - old_revision=None): - """Init a Project object. - - Args: - manifest: The XmlManifest object. - name: The `name` attribute of manifest.xml's project element. - remote: RemoteSpec object specifying its remote's properties. - gitdir: Absolute path of git directory. - objdir: Absolute path of directory to store git objects. - worktree: Absolute path of git working tree. - relpath: Relative path of git working tree to repo's top directory. - revisionExpr: The `revision` attribute of manifest.xml's project element. - revisionId: git commit id for checking out. - rebase: The `rebase` attribute of manifest.xml's project element. - groups: The `groups` attribute of manifest.xml's project element. - sync_c: The `sync-c` attribute of manifest.xml's project element. - sync_s: The `sync-s` attribute of manifest.xml's project element. - sync_tags: The `sync-tags` attribute of manifest.xml's project element. - upstream: The `upstream` attribute of manifest.xml's project element. - parent: The parent Project object. - use_git_worktrees: Whether to use `git worktree` for this project. - is_derived: False if the project was explicitly defined in the manifest; - True if the project is a discovered submodule. - dest_branch: The branch to which to push changes for review by default. - optimized_fetch: If True, when a project is set to a sha1 revision, only - fetch from the remote if the sha1 is not present locally. - retry_fetches: Retry remote fetches n times upon receiving transient error - with exponential backoff and jitter. - old_revision: saved git commit id for open GITC projects. - """ - self.client = self.manifest = manifest - self.name = name - self.remote = remote - self.UpdatePaths(relpath, worktree, gitdir, objdir) - self.SetRevision(revisionExpr, revisionId=revisionId) - - self.rebase = rebase - self.groups = groups - self.sync_c = sync_c - self.sync_s = sync_s - self.sync_tags = sync_tags - self.clone_depth = clone_depth - self.upstream = upstream - self.parent = parent - # NB: Do not use this setting in __init__ to change behavior so that the - # manifest.git checkout can inspect & change it after instantiating. See - # the XmlManifest init code for more info. - self.use_git_worktrees = use_git_worktrees - self.is_derived = is_derived - self.optimized_fetch = optimized_fetch - self.retry_fetches = max(0, retry_fetches) - self.subprojects = [] - - self.snapshots = {} - self.copyfiles = [] - self.linkfiles = [] - self.annotations = [] - self.dest_branch = dest_branch - self.old_revision = old_revision - - # This will be filled in if a project is later identified to be the - # project containing repo hooks. - self.enabled_repo_hooks = [] - - def RelPath(self, local=True): - """Return the path for the project relative to a manifest. - - Args: - local: a boolean, if True, the path is relative to the local - (sub)manifest. If false, the path is relative to the - outermost manifest. - """ - if local: - return self.relpath - return os.path.join(self.manifest.path_prefix, self.relpath) - - def SetRevision(self, revisionExpr, revisionId=None): - """Set revisionId based on revision expression and id""" - self.revisionExpr = revisionExpr - if revisionId is None and revisionExpr and IsId(revisionExpr): - self.revisionId = self.revisionExpr - else: - self.revisionId = revisionId - - def UpdatePaths(self, relpath, worktree, gitdir, objdir): - """Update paths used by this project""" - self.gitdir = gitdir.replace('\\', '/') - self.objdir = objdir.replace('\\', '/') - if worktree: - self.worktree = os.path.normpath(worktree).replace('\\', '/') - else: - self.worktree = None - self.relpath = relpath - - self.config = GitConfig.ForRepository(gitdir=self.gitdir, - defaults=self.manifest.globalConfig) - - if self.worktree: - self.work_git = self._GitGetByExec(self, bare=False, gitdir=self.gitdir) - else: - self.work_git = None - self.bare_git = self._GitGetByExec(self, bare=True, gitdir=self.gitdir) - self.bare_ref = GitRefs(self.gitdir) - self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=self.objdir) - - @property - def UseAlternates(self): - """Whether git alternates are in use. - - This will be removed once migration to alternates is complete. - """ - return _ALTERNATES or self.manifest.is_multimanifest + # These objects can be shared between several working trees. + @property + def shareable_dirs(self): + """Return the shareable directories""" + if self.UseAlternates: + return ["hooks", "rr-cache"] + else: + return ["hooks", "objects", "rr-cache"] + + def __init__( + self, + manifest, + name, + remote, + gitdir, + objdir, + worktree, + relpath, + revisionExpr, + revisionId, + rebase=True, + groups=None, + sync_c=False, + sync_s=False, + sync_tags=True, + clone_depth=None, + upstream=None, + parent=None, + use_git_worktrees=False, + is_derived=False, + dest_branch=None, + optimized_fetch=False, + retry_fetches=0, + old_revision=None, + ): + """Init a Project object. - @property - def Derived(self): - return self.is_derived + Args: + manifest: The XmlManifest object. + name: The `name` attribute of manifest.xml's project element. + remote: RemoteSpec object specifying its remote's properties. + gitdir: Absolute path of git directory. + objdir: Absolute path of directory to store git objects. + worktree: Absolute path of git working tree. + relpath: Relative path of git working tree to repo's top directory. + revisionExpr: The `revision` attribute of manifest.xml's project + element. + revisionId: git commit id for checking out. + rebase: The `rebase` attribute of manifest.xml's project element. + groups: The `groups` attribute of manifest.xml's project element. + sync_c: The `sync-c` attribute of manifest.xml's project element. + sync_s: The `sync-s` attribute of manifest.xml's project element. + sync_tags: The `sync-tags` attribute of manifest.xml's project + element. + upstream: The `upstream` attribute of manifest.xml's project + element. + parent: The parent Project object. + use_git_worktrees: Whether to use `git worktree` for this project. + is_derived: False if the project was explicitly defined in the + manifest; True if the project is a discovered submodule. + dest_branch: The branch to which to push changes for review by + default. + optimized_fetch: If True, when a project is set to a sha1 revision, + only fetch from the remote if the sha1 is not present locally. + retry_fetches: Retry remote fetches n times upon receiving transient + error with exponential backoff and jitter. + old_revision: saved git commit id for open GITC projects. + """ + self.client = self.manifest = manifest + self.name = name + self.remote = remote + self.UpdatePaths(relpath, worktree, gitdir, objdir) + self.SetRevision(revisionExpr, revisionId=revisionId) + + self.rebase = rebase + self.groups = groups + self.sync_c = sync_c + self.sync_s = sync_s + self.sync_tags = sync_tags + self.clone_depth = clone_depth + self.upstream = upstream + self.parent = parent + # NB: Do not use this setting in __init__ to change behavior so that the + # manifest.git checkout can inspect & change it after instantiating. + # See the XmlManifest init code for more info. + self.use_git_worktrees = use_git_worktrees + self.is_derived = is_derived + self.optimized_fetch = optimized_fetch + self.retry_fetches = max(0, retry_fetches) + self.subprojects = [] + + self.snapshots = {} + self.copyfiles = [] + self.linkfiles = [] + self.annotations = [] + self.dest_branch = dest_branch + self.old_revision = old_revision + + # This will be filled in if a project is later identified to be the + # project containing repo hooks. + self.enabled_repo_hooks = [] + + def RelPath(self, local=True): + """Return the path for the project relative to a manifest. - @property - def Exists(self): - return platform_utils.isdir(self.gitdir) and platform_utils.isdir(self.objdir) + Args: + local: a boolean, if True, the path is relative to the local + (sub)manifest. If false, the path is relative to the outermost + manifest. + """ + if local: + return self.relpath + return os.path.join(self.manifest.path_prefix, self.relpath) + + def SetRevision(self, revisionExpr, revisionId=None): + """Set revisionId based on revision expression and id""" + self.revisionExpr = revisionExpr + if revisionId is None and revisionExpr and IsId(revisionExpr): + self.revisionId = self.revisionExpr + else: + self.revisionId = revisionId + + def UpdatePaths(self, relpath, worktree, gitdir, objdir): + """Update paths used by this project""" + self.gitdir = gitdir.replace("\\", "/") + self.objdir = objdir.replace("\\", "/") + if worktree: + self.worktree = os.path.normpath(worktree).replace("\\", "/") + else: + self.worktree = None + self.relpath = relpath - @property - def CurrentBranch(self): - """Obtain the name of the currently checked out branch. + self.config = GitConfig.ForRepository( + gitdir=self.gitdir, defaults=self.manifest.globalConfig + ) - The branch name omits the 'refs/heads/' prefix. - None is returned if the project is on a detached HEAD, or if the work_git is - otheriwse inaccessible (e.g. an incomplete sync). - """ - try: - b = self.work_git.GetHead() - except NoManifestException: - # If the local checkout is in a bad state, don't barf. Let the callers - # process this like the head is unreadable. - return None - if b.startswith(R_HEADS): - return b[len(R_HEADS):] - return None - - def IsRebaseInProgress(self): - return (os.path.exists(self.work_git.GetDotgitPath('rebase-apply')) or - os.path.exists(self.work_git.GetDotgitPath('rebase-merge')) or - os.path.exists(os.path.join(self.worktree, '.dotest'))) - - def IsDirty(self, consider_untracked=True): - """Is the working directory modified in some way? - """ - self.work_git.update_index('-q', - '--unmerged', - '--ignore-missing', - '--refresh') - if self.work_git.DiffZ('diff-index', '-M', '--cached', HEAD): - return True - if self.work_git.DiffZ('diff-files'): - return True - if consider_untracked and self.UntrackedFiles(): - return True - return False - - _userident_name = None - _userident_email = None - - @property - def UserName(self): - """Obtain the user's personal name. - """ - if self._userident_name is None: - self._LoadUserIdentity() - return self._userident_name - - @property - def UserEmail(self): - """Obtain the user's email address. This is very likely - to be their Gerrit login. - """ - if self._userident_email is None: - self._LoadUserIdentity() - return self._userident_email - - def _LoadUserIdentity(self): - u = self.bare_git.var('GIT_COMMITTER_IDENT') - m = re.compile("^(.*) <([^>]*)> ").match(u) - if m: - self._userident_name = m.group(1) - self._userident_email = m.group(2) - else: - self._userident_name = '' - self._userident_email = '' - - def GetRemote(self, name=None): - """Get the configuration for a single remote. - - Defaults to the current project's remote. - """ - if name is None: - name = self.remote.name - return self.config.GetRemote(name) + if self.worktree: + self.work_git = self._GitGetByExec( + self, bare=False, gitdir=self.gitdir + ) + else: + self.work_git = None + self.bare_git = self._GitGetByExec(self, bare=True, gitdir=self.gitdir) + self.bare_ref = GitRefs(self.gitdir) + self.bare_objdir = self._GitGetByExec( + self, bare=True, gitdir=self.objdir + ) - def GetBranch(self, name): - """Get the configuration for a single branch. - """ - return self.config.GetBranch(name) + @property + def UseAlternates(self): + """Whether git alternates are in use. + + This will be removed once migration to alternates is complete. + """ + return _ALTERNATES or self.manifest.is_multimanifest + + @property + def Derived(self): + return self.is_derived + + @property + def Exists(self): + return platform_utils.isdir(self.gitdir) and platform_utils.isdir( + self.objdir + ) + + @property + def CurrentBranch(self): + """Obtain the name of the currently checked out branch. + + The branch name omits the 'refs/heads/' prefix. + None is returned if the project is on a detached HEAD, or if the + work_git is otheriwse inaccessible (e.g. an incomplete sync). + """ + try: + b = self.work_git.GetHead() + except NoManifestException: + # If the local checkout is in a bad state, don't barf. Let the + # callers process this like the head is unreadable. + return None + if b.startswith(R_HEADS): + return b[len(R_HEADS) :] + return None + + def IsRebaseInProgress(self): + return ( + os.path.exists(self.work_git.GetDotgitPath("rebase-apply")) + or os.path.exists(self.work_git.GetDotgitPath("rebase-merge")) + or os.path.exists(os.path.join(self.worktree, ".dotest")) + ) + + def IsDirty(self, consider_untracked=True): + """Is the working directory modified in some way?""" + self.work_git.update_index( + "-q", "--unmerged", "--ignore-missing", "--refresh" + ) + if self.work_git.DiffZ("diff-index", "-M", "--cached", HEAD): + return True + if self.work_git.DiffZ("diff-files"): + return True + if consider_untracked and self.UntrackedFiles(): + return True + return False + + _userident_name = None + _userident_email = None + + @property + def UserName(self): + """Obtain the user's personal name.""" + if self._userident_name is None: + self._LoadUserIdentity() + return self._userident_name + + @property + def UserEmail(self): + """Obtain the user's email address. This is very likely + to be their Gerrit login. + """ + if self._userident_email is None: + self._LoadUserIdentity() + return self._userident_email + + def _LoadUserIdentity(self): + u = self.bare_git.var("GIT_COMMITTER_IDENT") + m = re.compile("^(.*) <([^>]*)> ").match(u) + if m: + self._userident_name = m.group(1) + self._userident_email = m.group(2) + else: + self._userident_name = "" + self._userident_email = "" + + def GetRemote(self, name=None): + """Get the configuration for a single remote. + + Defaults to the current project's remote. + """ + if name is None: + name = self.remote.name + return self.config.GetRemote(name) + + def GetBranch(self, name): + """Get the configuration for a single branch.""" + return self.config.GetBranch(name) + + def GetBranches(self): + """Get all existing local branches.""" + current = self.CurrentBranch + all_refs = self._allrefs + heads = {} + + for name, ref_id in all_refs.items(): + if name.startswith(R_HEADS): + name = name[len(R_HEADS) :] + b = self.GetBranch(name) + b.current = name == current + b.published = None + b.revision = ref_id + heads[name] = b + + for name, ref_id in all_refs.items(): + if name.startswith(R_PUB): + name = name[len(R_PUB) :] + b = heads.get(name) + if b: + b.published = ref_id + + return heads + + def MatchesGroups(self, manifest_groups): + """Returns true if the manifest groups specified at init should cause + this project to be synced. + Prefixing a manifest group with "-" inverts the meaning of a group. + All projects are implicitly labelled with "all". + + labels are resolved in order. In the example case of + project_groups: "all,group1,group2" + manifest_groups: "-group1,group2" + the project will be matched. + + The special manifest group "default" will match any project that + does not have the special project group "notdefault" + """ + default_groups = self.manifest.default_groups or ["default"] + expanded_manifest_groups = manifest_groups or default_groups + expanded_project_groups = ["all"] + (self.groups or []) + if "notdefault" not in expanded_project_groups: + expanded_project_groups += ["default"] - def GetBranches(self): - """Get all existing local branches. - """ - current = self.CurrentBranch - all_refs = self._allrefs - heads = {} - - for name, ref_id in all_refs.items(): - if name.startswith(R_HEADS): - name = name[len(R_HEADS):] - b = self.GetBranch(name) - b.current = name == current - b.published = None - b.revision = ref_id - heads[name] = b - - for name, ref_id in all_refs.items(): - if name.startswith(R_PUB): - name = name[len(R_PUB):] - b = heads.get(name) - if b: - b.published = ref_id - - return heads - - def MatchesGroups(self, manifest_groups): - """Returns true if the manifest groups specified at init should cause - this project to be synced. - Prefixing a manifest group with "-" inverts the meaning of a group. - All projects are implicitly labelled with "all". - - labels are resolved in order. In the example case of - project_groups: "all,group1,group2" - manifest_groups: "-group1,group2" - the project will be matched. - - The special manifest group "default" will match any project that - does not have the special project group "notdefault" - """ - default_groups = self.manifest.default_groups or ['default'] - expanded_manifest_groups = manifest_groups or default_groups - expanded_project_groups = ['all'] + (self.groups or []) - if 'notdefault' not in expanded_project_groups: - expanded_project_groups += ['default'] - - matched = False - for group in expanded_manifest_groups: - if group.startswith('-') and group[1:] in expanded_project_groups: matched = False - elif group in expanded_project_groups: - matched = True + for group in expanded_manifest_groups: + if group.startswith("-") and group[1:] in expanded_project_groups: + matched = False + elif group in expanded_project_groups: + matched = True - return matched + return matched -# Status Display ## - def UncommitedFiles(self, get_all=True): - """Returns a list of strings, uncommitted files in the git tree. + def UncommitedFiles(self, get_all=True): + """Returns a list of strings, uncommitted files in the git tree. - Args: - get_all: a boolean, if True - get information about all different - uncommitted files. If False - return as soon as any kind of - uncommitted files is detected. - """ - details = [] - self.work_git.update_index('-q', - '--unmerged', - '--ignore-missing', - '--refresh') - if self.IsRebaseInProgress(): - details.append("rebase in progress") - if not get_all: - return details + Args: + get_all: a boolean, if True - get information about all different + uncommitted files. If False - return as soon as any kind of + uncommitted files is detected. + """ + details = [] + self.work_git.update_index( + "-q", "--unmerged", "--ignore-missing", "--refresh" + ) + if self.IsRebaseInProgress(): + details.append("rebase in progress") + if not get_all: + return details + + changes = self.work_git.DiffZ("diff-index", "--cached", HEAD).keys() + if changes: + details.extend(changes) + if not get_all: + return details + + changes = self.work_git.DiffZ("diff-files").keys() + if changes: + details.extend(changes) + if not get_all: + return details + + changes = self.UntrackedFiles() + if changes: + details.extend(changes) - changes = self.work_git.DiffZ('diff-index', '--cached', HEAD).keys() - if changes: - details.extend(changes) - if not get_all: return details - changes = self.work_git.DiffZ('diff-files').keys() - if changes: - details.extend(changes) - if not get_all: - return details + def UntrackedFiles(self): + """Returns a list of strings, untracked files in the git tree.""" + return self.work_git.LsOthers() - changes = self.UntrackedFiles() - if changes: - details.extend(changes) + def HasChanges(self): + """Returns true if there are uncommitted changes.""" + return bool(self.UncommitedFiles(get_all=False)) - return details + def PrintWorkTreeStatus(self, output_redir=None, quiet=False, local=False): + """Prints the status of the repository to stdout. - def UntrackedFiles(self): - """Returns a list of strings, untracked files in the git tree.""" - return self.work_git.LsOthers() + Args: + output_redir: If specified, redirect the output to this object. + quiet: If True then only print the project name. Do not print + the modified files, branch name, etc. + local: a boolean, if True, the path is relative to the local + (sub)manifest. If false, the path is relative to the outermost + manifest. + """ + if not platform_utils.isdir(self.worktree): + if output_redir is None: + output_redir = sys.stdout + print(file=output_redir) + print("project %s/" % self.RelPath(local), file=output_redir) + print(' missing (run "repo sync")', file=output_redir) + return + + self.work_git.update_index( + "-q", "--unmerged", "--ignore-missing", "--refresh" + ) + rb = self.IsRebaseInProgress() + di = self.work_git.DiffZ("diff-index", "-M", "--cached", HEAD) + df = self.work_git.DiffZ("diff-files") + do = self.work_git.LsOthers() + if not rb and not di and not df and not do and not self.CurrentBranch: + return "CLEAN" + + out = StatusColoring(self.config) + if output_redir is not None: + out.redirect(output_redir) + out.project("project %-40s", self.RelPath(local) + "/ ") + + if quiet: + out.nl() + return "DIRTY" + + branch = self.CurrentBranch + if branch is None: + out.nobranch("(*** NO BRANCH ***)") + else: + out.branch("branch %s", branch) + out.nl() - def HasChanges(self): - """Returns true if there are uncommitted changes. - """ - return bool(self.UncommitedFiles(get_all=False)) - - def PrintWorkTreeStatus(self, output_redir=None, quiet=False, local=False): - """Prints the status of the repository to stdout. - - Args: - output_redir: If specified, redirect the output to this object. - quiet: If True then only print the project name. Do not print - the modified files, branch name, etc. - local: a boolean, if True, the path is relative to the local - (sub)manifest. If false, the path is relative to the - outermost manifest. - """ - if not platform_utils.isdir(self.worktree): - if output_redir is None: - output_redir = sys.stdout - print(file=output_redir) - print('project %s/' % self.RelPath(local), file=output_redir) - print(' missing (run "repo sync")', file=output_redir) - return - - self.work_git.update_index('-q', - '--unmerged', - '--ignore-missing', - '--refresh') - rb = self.IsRebaseInProgress() - di = self.work_git.DiffZ('diff-index', '-M', '--cached', HEAD) - df = self.work_git.DiffZ('diff-files') - do = self.work_git.LsOthers() - if not rb and not di and not df and not do and not self.CurrentBranch: - return 'CLEAN' - - out = StatusColoring(self.config) - if output_redir is not None: - out.redirect(output_redir) - out.project('project %-40s', self.RelPath(local) + '/ ') - - if quiet: - out.nl() - return 'DIRTY' - - branch = self.CurrentBranch - if branch is None: - out.nobranch('(*** NO BRANCH ***)') - else: - out.branch('branch %s', branch) - out.nl() - - if rb: - out.important('prior sync failed; rebase still in progress') - out.nl() - - paths = list() - paths.extend(di.keys()) - paths.extend(df.keys()) - paths.extend(do) - - for p in sorted(set(paths)): - try: - i = di[p] - except KeyError: - i = None - - try: - f = df[p] - except KeyError: - f = None - - if i: - i_status = i.status.upper() - else: - i_status = '-' - - if f: - f_status = f.status.lower() - else: - f_status = '-' - - if i and i.src_path: - line = ' %s%s\t%s => %s (%s%%)' % (i_status, f_status, - i.src_path, p, i.level) - else: - line = ' %s%s\t%s' % (i_status, f_status, p) - - if i and not f: - out.added('%s', line) - elif (i and f) or (not i and f): - out.changed('%s', line) - elif not i and not f: - out.untracked('%s', line) - else: - out.write('%s', line) - out.nl() - - return 'DIRTY' - - def PrintWorkTreeDiff(self, absolute_paths=False, output_redir=None, - local=False): - """Prints the status of the repository to stdout. - """ - out = DiffColoring(self.config) - if output_redir: - out.redirect(output_redir) - cmd = ['diff'] - if out.is_on: - cmd.append('--color') - cmd.append(HEAD) - if absolute_paths: - cmd.append('--src-prefix=a/%s/' % self.RelPath(local)) - cmd.append('--dst-prefix=b/%s/' % self.RelPath(local)) - cmd.append('--') - try: - p = GitCommand(self, - cmd, - capture_stdout=True, - capture_stderr=True) - p.Wait() - except GitError as e: - out.nl() - out.project('project %s/' % self.RelPath(local)) - out.nl() - out.fail('%s', str(e)) - out.nl() - return False - if p.stdout: - out.nl() - out.project('project %s/' % self.RelPath(local)) - out.nl() - out.write('%s', p.stdout) - return p.Wait() == 0 - -# Publish / Upload ## - def WasPublished(self, branch, all_refs=None): - """Was the branch published (uploaded) for code review? - If so, returns the SHA-1 hash of the last published - state for the branch. - """ - key = R_PUB + branch - if all_refs is None: - try: - return self.bare_git.rev_parse(key) - except GitError: - return None - else: - try: - return all_refs[key] - except KeyError: - return None + if rb: + out.important("prior sync failed; rebase still in progress") + out.nl() - def CleanPublishedCache(self, all_refs=None): - """Prunes any stale published refs. - """ - if all_refs is None: - all_refs = self._allrefs - heads = set() - canrm = {} - for name, ref_id in all_refs.items(): - if name.startswith(R_HEADS): - heads.add(name) - elif name.startswith(R_PUB): - canrm[name] = ref_id - - for name, ref_id in canrm.items(): - n = name[len(R_PUB):] - if R_HEADS + n not in heads: - self.bare_git.DeleteRef(name, ref_id) - - def GetUploadableBranches(self, selected_branch=None): - """List any branches which can be uploaded for review. - """ - heads = {} - pubed = {} - - for name, ref_id in self._allrefs.items(): - if name.startswith(R_HEADS): - heads[name[len(R_HEADS):]] = ref_id - elif name.startswith(R_PUB): - pubed[name[len(R_PUB):]] = ref_id - - ready = [] - for branch, ref_id in heads.items(): - if branch in pubed and pubed[branch] == ref_id: - continue - if selected_branch and branch != selected_branch: - continue - - rb = self.GetUploadableBranch(branch) - if rb: - ready.append(rb) - return ready - - def GetUploadableBranch(self, branch_name): - """Get a single uploadable branch, or None. - """ - branch = self.GetBranch(branch_name) - base = branch.LocalMerge - if branch.LocalMerge: - rb = ReviewableBranch(self, branch, base) - if rb.commits: - return rb - return None - - def UploadForReview(self, branch=None, - people=([], []), - dryrun=False, - auto_topic=False, - hashtags=(), - labels=(), - private=False, - notify=None, - wip=False, - ready=False, - dest_branch=None, - validate_certs=True, - push_options=None): - """Uploads the named branch for code review. - """ - if branch is None: - branch = self.CurrentBranch - if branch is None: - raise GitError('not currently on a branch') - - branch = self.GetBranch(branch) - if not branch.LocalMerge: - raise GitError('branch %s does not track a remote' % branch.name) - if not branch.remote.review: - raise GitError('remote %s has no review url' % branch.remote.name) - - # Basic validity check on label syntax. - for label in labels: - if not re.match(r'^.+[+-][0-9]+$', label): - raise UploadError( - f'invalid label syntax "{label}": labels use forms like ' - 'CodeReview+1 or Verified-1') - - if dest_branch is None: - dest_branch = self.dest_branch - if dest_branch is None: - dest_branch = branch.merge - if not dest_branch.startswith(R_HEADS): - dest_branch = R_HEADS + dest_branch - - if not branch.remote.projectname: - branch.remote.projectname = self.name - branch.remote.Save() - - url = branch.remote.ReviewUrl(self.UserEmail, validate_certs) - if url is None: - raise UploadError('review not configured') - cmd = ['push'] - if dryrun: - cmd.append('-n') - - if url.startswith('ssh://'): - cmd.append('--receive-pack=gerrit receive-pack') - - for push_option in (push_options or []): - cmd.append('-o') - cmd.append(push_option) - - cmd.append(url) - - if dest_branch.startswith(R_HEADS): - dest_branch = dest_branch[len(R_HEADS):] - - ref_spec = '%s:refs/for/%s' % (R_HEADS + branch.name, dest_branch) - opts = [] - if auto_topic: - opts += ['topic=' + branch.name] - opts += ['t=%s' % p for p in hashtags] - # NB: No need to encode labels as they've been validated above. - opts += ['l=%s' % p for p in labels] - - opts += ['r=%s' % p for p in people[0]] - opts += ['cc=%s' % p for p in people[1]] - if notify: - opts += ['notify=' + notify] - if private: - opts += ['private'] - if wip: - opts += ['wip'] - if ready: - opts += ['ready'] - if opts: - ref_spec = ref_spec + '%' + ','.join(opts) - cmd.append(ref_spec) - - if GitCommand(self, cmd, bare=True).Wait() != 0: - raise UploadError('Upload failed') - - if not dryrun: - msg = "posted to %s for %s" % (branch.remote.review, dest_branch) - self.bare_git.UpdateRef(R_PUB + branch.name, - R_HEADS + branch.name, - message=msg) - -# Sync ## - def _ExtractArchive(self, tarpath, path=None): - """Extract the given tar on its current location - - Args: - - tarpath: The path to the actual tar file + paths = list() + paths.extend(di.keys()) + paths.extend(df.keys()) + paths.extend(do) - """ - try: - with tarfile.open(tarpath, 'r') as tar: - tar.extractall(path=path) - return True - except (IOError, tarfile.TarError) as e: - _error("Cannot extract archive %s: %s", tarpath, str(e)) - return False - - def Sync_NetworkHalf(self, - quiet=False, - verbose=False, - output_redir=None, - is_new=None, - current_branch_only=None, - force_sync=False, - clone_bundle=True, - tags=None, - archive=False, - optimized_fetch=False, - retry_fetches=0, - prune=False, - submodules=False, - ssh_proxy=None, - clone_filter=None, - partial_clone_exclude=set()): - """Perform only the network IO portion of the sync process. - Local working directory/branch state is not affected. - """ - if archive and not isinstance(self, MetaProject): - if self.remote.url.startswith(('http://', 'https://')): - _error("%s: Cannot fetch archives from http/https remotes.", self.name) - return SyncNetworkHalfResult(False, False) - - name = self.relpath.replace('\\', '/') - name = name.replace('/', '_') - tarpath = '%s.tar' % name - topdir = self.manifest.topdir - - try: - self._FetchArchive(tarpath, cwd=topdir) - except GitError as e: - _error('%s', e) - return SyncNetworkHalfResult(False, False) - - # From now on, we only need absolute tarpath - tarpath = os.path.join(topdir, tarpath) - - if not self._ExtractArchive(tarpath, path=topdir): - return SyncNetworkHalfResult(False, True) - try: - platform_utils.remove(tarpath) - except OSError as e: - _warn("Cannot remove archive %s: %s", tarpath, str(e)) - self._CopyAndLinkFiles() - return SyncNetworkHalfResult(True, True) - - # If the shared object dir already exists, don't try to rebootstrap with a - # clone bundle download. We should have the majority of objects already. - if clone_bundle and os.path.exists(self.objdir): - clone_bundle = False - - if self.name in partial_clone_exclude: - clone_bundle = True - clone_filter = None - - if is_new is None: - is_new = not self.Exists - if is_new: - self._InitGitDir(force_sync=force_sync, quiet=quiet) - else: - self._UpdateHooks(quiet=quiet) - self._InitRemote() - - if self.UseAlternates: - # If gitdir/objects is a symlink, migrate it from the old layout. - gitdir_objects = os.path.join(self.gitdir, 'objects') - if platform_utils.islink(gitdir_objects): - platform_utils.remove(gitdir_objects, missing_ok=True) - gitdir_alt = os.path.join(self.gitdir, 'objects/info/alternates') - if not os.path.exists(gitdir_alt): - os.makedirs(os.path.dirname(gitdir_alt), exist_ok=True) - _lwrite(gitdir_alt, os.path.join( - os.path.relpath(self.objdir, gitdir_objects), 'objects') + '\n') - - if is_new: - alt = os.path.join(self.objdir, 'objects/info/alternates') - try: - with open(alt) as fd: - # This works for both absolute and relative alternate directories. - alt_dir = os.path.join(self.objdir, 'objects', fd.readline().rstrip()) - except IOError: - alt_dir = None - else: - alt_dir = None - - if (clone_bundle - and alt_dir is None - and self._ApplyCloneBundle(initial=is_new, quiet=quiet, verbose=verbose)): - is_new = False - - if current_branch_only is None: - if self.sync_c: - current_branch_only = True - elif not self.manifest._loaded: - # Manifest cannot check defaults until it syncs. - current_branch_only = False - elif self.manifest.default.sync_c: - current_branch_only = True - - if tags is None: - tags = self.sync_tags - - if self.clone_depth: - depth = self.clone_depth - else: - depth = self.manifest.manifestProject.depth - - # See if we can skip the network fetch entirely. - remote_fetched = False - if not (optimized_fetch and - (ID_RE.match(self.revisionExpr) and - self._CheckForImmutableRevision())): - remote_fetched = True - if not self._RemoteFetch( - initial=is_new, - quiet=quiet, verbose=verbose, output_redir=output_redir, - alt_dir=alt_dir, current_branch_only=current_branch_only, - tags=tags, prune=prune, depth=depth, - submodules=submodules, force_sync=force_sync, - ssh_proxy=ssh_proxy, - clone_filter=clone_filter, retry_fetches=retry_fetches): - return SyncNetworkHalfResult(False, remote_fetched) - - mp = self.manifest.manifestProject - dissociate = mp.dissociate - if dissociate: - alternates_file = os.path.join(self.objdir, 'objects/info/alternates') - if os.path.exists(alternates_file): - cmd = ['repack', '-a', '-d'] - p = GitCommand(self, cmd, bare=True, capture_stdout=bool(output_redir), - merge_output=bool(output_redir)) - if p.stdout and output_redir: - output_redir.write(p.stdout) - if p.Wait() != 0: - return SyncNetworkHalfResult(False, remote_fetched) - platform_utils.remove(alternates_file) - - if self.worktree: - self._InitMRef() - else: - self._InitMirrorHead() - platform_utils.remove(os.path.join(self.gitdir, 'FETCH_HEAD'), - missing_ok=True) - return SyncNetworkHalfResult(True, remote_fetched) - - def PostRepoUpgrade(self): - self._InitHooks() - - def _CopyAndLinkFiles(self): - if self.client.isGitcClient: - return - for copyfile in self.copyfiles: - copyfile._Copy() - for linkfile in self.linkfiles: - linkfile._Link() - - def GetCommitRevisionId(self): - """Get revisionId of a commit. - - Use this method instead of GetRevisionId to get the id of the commit rather - than the id of the current git object (for example, a tag) + for p in sorted(set(paths)): + try: + i = di[p] + except KeyError: + i = None - """ - if not self.revisionExpr.startswith(R_TAGS): - return self.GetRevisionId(self._allrefs) + try: + f = df[p] + except KeyError: + f = None + + if i: + i_status = i.status.upper() + else: + i_status = "-" + + if f: + f_status = f.status.lower() + else: + f_status = "-" + + if i and i.src_path: + line = " %s%s\t%s => %s (%s%%)" % ( + i_status, + f_status, + i.src_path, + p, + i.level, + ) + else: + line = " %s%s\t%s" % (i_status, f_status, p) + + if i and not f: + out.added("%s", line) + elif (i and f) or (not i and f): + out.changed("%s", line) + elif not i and not f: + out.untracked("%s", line) + else: + out.write("%s", line) + out.nl() + + return "DIRTY" + + def PrintWorkTreeDiff( + self, absolute_paths=False, output_redir=None, local=False + ): + """Prints the status of the repository to stdout.""" + out = DiffColoring(self.config) + if output_redir: + out.redirect(output_redir) + cmd = ["diff"] + if out.is_on: + cmd.append("--color") + cmd.append(HEAD) + if absolute_paths: + cmd.append("--src-prefix=a/%s/" % self.RelPath(local)) + cmd.append("--dst-prefix=b/%s/" % self.RelPath(local)) + cmd.append("--") + try: + p = GitCommand(self, cmd, capture_stdout=True, capture_stderr=True) + p.Wait() + except GitError as e: + out.nl() + out.project("project %s/" % self.RelPath(local)) + out.nl() + out.fail("%s", str(e)) + out.nl() + return False + if p.stdout: + out.nl() + out.project("project %s/" % self.RelPath(local)) + out.nl() + out.write("%s", p.stdout) + return p.Wait() == 0 + + def WasPublished(self, branch, all_refs=None): + """Was the branch published (uploaded) for code review? + If so, returns the SHA-1 hash of the last published + state for the branch. + """ + key = R_PUB + branch + if all_refs is None: + try: + return self.bare_git.rev_parse(key) + except GitError: + return None + else: + try: + return all_refs[key] + except KeyError: + return None + + def CleanPublishedCache(self, all_refs=None): + """Prunes any stale published refs.""" + if all_refs is None: + all_refs = self._allrefs + heads = set() + canrm = {} + for name, ref_id in all_refs.items(): + if name.startswith(R_HEADS): + heads.add(name) + elif name.startswith(R_PUB): + canrm[name] = ref_id + + for name, ref_id in canrm.items(): + n = name[len(R_PUB) :] + if R_HEADS + n not in heads: + self.bare_git.DeleteRef(name, ref_id) + + def GetUploadableBranches(self, selected_branch=None): + """List any branches which can be uploaded for review.""" + heads = {} + pubed = {} + + for name, ref_id in self._allrefs.items(): + if name.startswith(R_HEADS): + heads[name[len(R_HEADS) :]] = ref_id + elif name.startswith(R_PUB): + pubed[name[len(R_PUB) :]] = ref_id + + ready = [] + for branch, ref_id in heads.items(): + if branch in pubed and pubed[branch] == ref_id: + continue + if selected_branch and branch != selected_branch: + continue + + rb = self.GetUploadableBranch(branch) + if rb: + ready.append(rb) + return ready + + def GetUploadableBranch(self, branch_name): + """Get a single uploadable branch, or None.""" + branch = self.GetBranch(branch_name) + base = branch.LocalMerge + if branch.LocalMerge: + rb = ReviewableBranch(self, branch, base) + if rb.commits: + return rb + return None - try: - return self.bare_git.rev_list(self.revisionExpr, '-1')[0] - except GitError: - raise ManifestInvalidRevisionError('revision %s in %s not found' % - (self.revisionExpr, self.name)) + def UploadForReview( + self, + branch=None, + people=([], []), + dryrun=False, + auto_topic=False, + hashtags=(), + labels=(), + private=False, + notify=None, + wip=False, + ready=False, + dest_branch=None, + validate_certs=True, + push_options=None, + ): + """Uploads the named branch for code review.""" + if branch is None: + branch = self.CurrentBranch + if branch is None: + raise GitError("not currently on a branch") - def GetRevisionId(self, all_refs=None): - if self.revisionId: - return self.revisionId + branch = self.GetBranch(branch) + if not branch.LocalMerge: + raise GitError("branch %s does not track a remote" % branch.name) + if not branch.remote.review: + raise GitError("remote %s has no review url" % branch.remote.name) + + # Basic validity check on label syntax. + for label in labels: + if not re.match(r"^.+[+-][0-9]+$", label): + raise UploadError( + f'invalid label syntax "{label}": labels use forms like ' + "CodeReview+1 or Verified-1" + ) + + if dest_branch is None: + dest_branch = self.dest_branch + if dest_branch is None: + dest_branch = branch.merge + if not dest_branch.startswith(R_HEADS): + dest_branch = R_HEADS + dest_branch + + if not branch.remote.projectname: + branch.remote.projectname = self.name + branch.remote.Save() + + url = branch.remote.ReviewUrl(self.UserEmail, validate_certs) + if url is None: + raise UploadError("review not configured") + cmd = ["push"] + if dryrun: + cmd.append("-n") + + if url.startswith("ssh://"): + cmd.append("--receive-pack=gerrit receive-pack") + + for push_option in push_options or []: + cmd.append("-o") + cmd.append(push_option) + + cmd.append(url) + + if dest_branch.startswith(R_HEADS): + dest_branch = dest_branch[len(R_HEADS) :] + + ref_spec = "%s:refs/for/%s" % (R_HEADS + branch.name, dest_branch) + opts = [] + if auto_topic: + opts += ["topic=" + branch.name] + opts += ["t=%s" % p for p in hashtags] + # NB: No need to encode labels as they've been validated above. + opts += ["l=%s" % p for p in labels] + + opts += ["r=%s" % p for p in people[0]] + opts += ["cc=%s" % p for p in people[1]] + if notify: + opts += ["notify=" + notify] + if private: + opts += ["private"] + if wip: + opts += ["wip"] + if ready: + opts += ["ready"] + if opts: + ref_spec = ref_spec + "%" + ",".join(opts) + cmd.append(ref_spec) + + if GitCommand(self, cmd, bare=True).Wait() != 0: + raise UploadError("Upload failed") + + if not dryrun: + msg = "posted to %s for %s" % (branch.remote.review, dest_branch) + self.bare_git.UpdateRef( + R_PUB + branch.name, R_HEADS + branch.name, message=msg + ) + + def _ExtractArchive(self, tarpath, path=None): + """Extract the given tar on its current location - rem = self.GetRemote() - rev = rem.ToLocal(self.revisionExpr) + Args: + tarpath: The path to the actual tar file - if all_refs is not None and rev in all_refs: - return all_refs[rev] + """ + try: + with tarfile.open(tarpath, "r") as tar: + tar.extractall(path=path) + return True + except (IOError, tarfile.TarError) as e: + _error("Cannot extract archive %s: %s", tarpath, str(e)) + return False - try: - return self.bare_git.rev_parse('--verify', '%s^0' % rev) - except GitError: - raise ManifestInvalidRevisionError('revision %s in %s not found' % - (self.revisionExpr, self.name)) + def Sync_NetworkHalf( + self, + quiet=False, + verbose=False, + output_redir=None, + is_new=None, + current_branch_only=None, + force_sync=False, + clone_bundle=True, + tags=None, + archive=False, + optimized_fetch=False, + retry_fetches=0, + prune=False, + submodules=False, + ssh_proxy=None, + clone_filter=None, + partial_clone_exclude=set(), + ): + """Perform only the network IO portion of the sync process. + Local working directory/branch state is not affected. + """ + if archive and not isinstance(self, MetaProject): + if self.remote.url.startswith(("http://", "https://")): + _error( + "%s: Cannot fetch archives from http/https remotes.", + self.name, + ) + return SyncNetworkHalfResult(False, False) + + name = self.relpath.replace("\\", "/") + name = name.replace("/", "_") + tarpath = "%s.tar" % name + topdir = self.manifest.topdir - def SetRevisionId(self, revisionId): - if self.revisionExpr: - self.upstream = self.revisionExpr + try: + self._FetchArchive(tarpath, cwd=topdir) + except GitError as e: + _error("%s", e) + return SyncNetworkHalfResult(False, False) - self.revisionId = revisionId + # From now on, we only need absolute tarpath. + tarpath = os.path.join(topdir, tarpath) - def Sync_LocalHalf(self, syncbuf, force_sync=False, submodules=False): - """Perform only the local IO portion of the sync process. - Network access is not required. - """ - if not os.path.exists(self.gitdir): - syncbuf.fail(self, - 'Cannot checkout %s due to missing network sync; Run ' - '`repo sync -n %s` first.' % - (self.name, self.name)) - return - - self._InitWorkTree(force_sync=force_sync, submodules=submodules) - all_refs = self.bare_ref.all - self.CleanPublishedCache(all_refs) - revid = self.GetRevisionId(all_refs) - - # Special case the root of the repo client checkout. Make sure it doesn't - # contain files being checked out to dirs we don't allow. - if self.relpath == '.': - PROTECTED_PATHS = {'.repo'} - paths = set(self.work_git.ls_tree('-z', '--name-only', '--', revid).split('\0')) - bad_paths = paths & PROTECTED_PATHS - if bad_paths: - syncbuf.fail(self, - 'Refusing to checkout project that writes to protected ' - 'paths: %s' % (', '.join(bad_paths),)) - return - - def _doff(): - self._FastForward(revid) - self._CopyAndLinkFiles() - - def _dosubmodules(): - self._SyncSubmodules(quiet=True) - - head = self.work_git.GetHead() - if head.startswith(R_HEADS): - branch = head[len(R_HEADS):] - try: - head = all_refs[head] - except KeyError: - head = None - else: - branch = None - - if branch is None or syncbuf.detach_head: - # Currently on a detached HEAD. The user is assumed to - # not have any local modifications worth worrying about. - # - if self.IsRebaseInProgress(): - syncbuf.fail(self, _PriorSyncFailedError()) - return - - if head == revid: - # No changes; don't do anything further. - # Except if the head needs to be detached - # - if not syncbuf.detach_head: - # The copy/linkfile config may have changed. - self._CopyAndLinkFiles() - return - else: - lost = self._revlist(not_rev(revid), HEAD) - if lost: - syncbuf.info(self, "discarding %d commits", len(lost)) - - try: - self._Checkout(revid, quiet=True) - if submodules: - self._SyncSubmodules(quiet=True) - except GitError as e: - syncbuf.fail(self, e) - return - self._CopyAndLinkFiles() - return - - if head == revid: - # No changes; don't do anything further. - # - # The copy/linkfile config may have changed. - self._CopyAndLinkFiles() - return - - branch = self.GetBranch(branch) - - if not branch.LocalMerge: - # The current branch has no tracking configuration. - # Jump off it to a detached HEAD. - # - syncbuf.info(self, - "leaving %s; does not track upstream", - branch.name) - try: - self._Checkout(revid, quiet=True) - if submodules: - self._SyncSubmodules(quiet=True) - except GitError as e: - syncbuf.fail(self, e) - return - self._CopyAndLinkFiles() - return + if not self._ExtractArchive(tarpath, path=topdir): + return SyncNetworkHalfResult(False, True) + try: + platform_utils.remove(tarpath) + except OSError as e: + _warn("Cannot remove archive %s: %s", tarpath, str(e)) + self._CopyAndLinkFiles() + return SyncNetworkHalfResult(True, True) + + # If the shared object dir already exists, don't try to rebootstrap with + # a clone bundle download. We should have the majority of objects + # already. + if clone_bundle and os.path.exists(self.objdir): + clone_bundle = False + + if self.name in partial_clone_exclude: + clone_bundle = True + clone_filter = None + + if is_new is None: + is_new = not self.Exists + if is_new: + self._InitGitDir(force_sync=force_sync, quiet=quiet) + else: + self._UpdateHooks(quiet=quiet) + self._InitRemote() + + if self.UseAlternates: + # If gitdir/objects is a symlink, migrate it from the old layout. + gitdir_objects = os.path.join(self.gitdir, "objects") + if platform_utils.islink(gitdir_objects): + platform_utils.remove(gitdir_objects, missing_ok=True) + gitdir_alt = os.path.join(self.gitdir, "objects/info/alternates") + if not os.path.exists(gitdir_alt): + os.makedirs(os.path.dirname(gitdir_alt), exist_ok=True) + _lwrite( + gitdir_alt, + os.path.join( + os.path.relpath(self.objdir, gitdir_objects), "objects" + ) + + "\n", + ) - upstream_gain = self._revlist(not_rev(HEAD), revid) + if is_new: + alt = os.path.join(self.objdir, "objects/info/alternates") + try: + with open(alt) as fd: + # This works for both absolute and relative alternate + # directories. + alt_dir = os.path.join( + self.objdir, "objects", fd.readline().rstrip() + ) + except IOError: + alt_dir = None + else: + alt_dir = None - # See if we can perform a fast forward merge. This can happen if our - # branch isn't in the exact same state as we last published. - try: - self.work_git.merge_base('--is-ancestor', HEAD, revid) - # Skip the published logic. - pub = False - except GitError: - pub = self.WasPublished(branch.name, all_refs) - - if pub: - not_merged = self._revlist(not_rev(revid), pub) - if not_merged: - if upstream_gain: - # The user has published this branch and some of those - # commits are not yet merged upstream. We do not want - # to rewrite the published commits so we punt. - # - syncbuf.fail(self, - "branch %s is published (but not merged) and is now " - "%d commits behind" % (branch.name, len(upstream_gain))) - return - elif pub == head: - # All published commits are merged, and thus we are a - # strict subset. We can fast-forward safely. - # - syncbuf.later1(self, _doff) - if submodules: - syncbuf.later1(self, _dosubmodules) - return - - # Examine the local commits not in the remote. Find the - # last one attributed to this user, if any. - # - local_changes = self._revlist(not_rev(revid), HEAD, format='%H %ce') - last_mine = None - cnt_mine = 0 - for commit in local_changes: - commit_id, committer_email = commit.split(' ', 1) - if committer_email == self.UserEmail: - last_mine = commit_id - cnt_mine += 1 - - if not upstream_gain and cnt_mine == len(local_changes): - # The copy/linkfile config may have changed. - self._CopyAndLinkFiles() - return - - if self.IsDirty(consider_untracked=False): - syncbuf.fail(self, _DirtyError()) - return - - # If the upstream switched on us, warn the user. - # - if branch.merge != self.revisionExpr: - if branch.merge and self.revisionExpr: - syncbuf.info(self, - 'manifest switched %s...%s', - branch.merge, - self.revisionExpr) - elif branch.merge: - syncbuf.info(self, - 'manifest no longer tracks %s', - branch.merge) - - if cnt_mine < len(local_changes): - # Upstream rebased. Not everything in HEAD - # was created by this user. - # - syncbuf.info(self, - "discarding %d commits removed from upstream", - len(local_changes) - cnt_mine) - - branch.remote = self.GetRemote() - if not ID_RE.match(self.revisionExpr): - # in case of manifest sync the revisionExpr might be a SHA1 - branch.merge = self.revisionExpr - if not branch.merge.startswith('refs/'): - branch.merge = R_HEADS + branch.merge - branch.Save() - - if cnt_mine > 0 and self.rebase: - def _docopyandlink(): - self._CopyAndLinkFiles() - - def _dorebase(): - self._Rebase(upstream='%s^1' % last_mine, onto=revid) - syncbuf.later2(self, _dorebase) - if submodules: - syncbuf.later2(self, _dosubmodules) - syncbuf.later2(self, _docopyandlink) - elif local_changes: - try: - self._ResetHard(revid) - if submodules: - self._SyncSubmodules(quiet=True) - self._CopyAndLinkFiles() - except GitError as e: - syncbuf.fail(self, e) - return - else: - syncbuf.later1(self, _doff) - if submodules: - syncbuf.later1(self, _dosubmodules) - - def AddCopyFile(self, src, dest, topdir): - """Mark |src| for copying to |dest| (relative to |topdir|). - - No filesystem changes occur here. Actual copying happens later on. - - Paths should have basic validation run on them before being queued. - Further checking will be handled when the actual copy happens. - """ - self.copyfiles.append(_CopyFile(self.worktree, src, topdir, dest)) + if ( + clone_bundle + and alt_dir is None + and self._ApplyCloneBundle( + initial=is_new, quiet=quiet, verbose=verbose + ) + ): + is_new = False + + if current_branch_only is None: + if self.sync_c: + current_branch_only = True + elif not self.manifest._loaded: + # Manifest cannot check defaults until it syncs. + current_branch_only = False + elif self.manifest.default.sync_c: + current_branch_only = True + + if tags is None: + tags = self.sync_tags + + if self.clone_depth: + depth = self.clone_depth + else: + depth = self.manifest.manifestProject.depth + + # See if we can skip the network fetch entirely. + remote_fetched = False + if not ( + optimized_fetch + and ( + ID_RE.match(self.revisionExpr) + and self._CheckForImmutableRevision() + ) + ): + remote_fetched = True + if not self._RemoteFetch( + initial=is_new, + quiet=quiet, + verbose=verbose, + output_redir=output_redir, + alt_dir=alt_dir, + current_branch_only=current_branch_only, + tags=tags, + prune=prune, + depth=depth, + submodules=submodules, + force_sync=force_sync, + ssh_proxy=ssh_proxy, + clone_filter=clone_filter, + retry_fetches=retry_fetches, + ): + return SyncNetworkHalfResult(False, remote_fetched) - def AddLinkFile(self, src, dest, topdir): - """Mark |dest| to create a symlink (relative to |topdir|) pointing to |src|. + mp = self.manifest.manifestProject + dissociate = mp.dissociate + if dissociate: + alternates_file = os.path.join( + self.objdir, "objects/info/alternates" + ) + if os.path.exists(alternates_file): + cmd = ["repack", "-a", "-d"] + p = GitCommand( + self, + cmd, + bare=True, + capture_stdout=bool(output_redir), + merge_output=bool(output_redir), + ) + if p.stdout and output_redir: + output_redir.write(p.stdout) + if p.Wait() != 0: + return SyncNetworkHalfResult(False, remote_fetched) + platform_utils.remove(alternates_file) + + if self.worktree: + self._InitMRef() + else: + self._InitMirrorHead() + platform_utils.remove( + os.path.join(self.gitdir, "FETCH_HEAD"), missing_ok=True + ) + return SyncNetworkHalfResult(True, remote_fetched) - No filesystem changes occur here. Actual linking happens later on. + def PostRepoUpgrade(self): + self._InitHooks() - Paths should have basic validation run on them before being queued. - Further checking will be handled when the actual link happens. - """ - self.linkfiles.append(_LinkFile(self.worktree, src, topdir, dest)) + def _CopyAndLinkFiles(self): + if self.client.isGitcClient: + return + for copyfile in self.copyfiles: + copyfile._Copy() + for linkfile in self.linkfiles: + linkfile._Link() - def AddAnnotation(self, name, value, keep): - self.annotations.append(Annotation(name, value, keep)) + def GetCommitRevisionId(self): + """Get revisionId of a commit. - def DownloadPatchSet(self, change_id, patch_id): - """Download a single patch set of a single change to FETCH_HEAD. - """ - remote = self.GetRemote() - - cmd = ['fetch', remote.name] - cmd.append('refs/changes/%2.2d/%d/%d' - % (change_id % 100, change_id, patch_id)) - if GitCommand(self, cmd, bare=True).Wait() != 0: - return None - return DownloadedChange(self, - self.GetRevisionId(), - change_id, - patch_id, - self.bare_git.rev_parse('FETCH_HEAD')) - - def DeleteWorktree(self, quiet=False, force=False): - """Delete the source checkout and any other housekeeping tasks. - - This currently leaves behind the internal .repo/ cache state. This helps - when switching branches or manifest changes get reverted as we don't have - to redownload all the git objects. But we should do some GC at some point. - - Args: - quiet: Whether to hide normal messages. - force: Always delete tree even if dirty. + Use this method instead of GetRevisionId to get the id of the commit + rather than the id of the current git object (for example, a tag) - Returns: - True if the worktree was completely cleaned out. - """ - if self.IsDirty(): - if force: - print('warning: %s: Removing dirty project: uncommitted changes lost.' % - (self.RelPath(local=False),), file=sys.stderr) - else: - print('error: %s: Cannot remove project: uncommitted changes are ' - 'present.\n' % (self.RelPath(local=False),), file=sys.stderr) - return False + """ + if not self.revisionExpr.startswith(R_TAGS): + return self.GetRevisionId(self._allrefs) - if not quiet: - print('%s: Deleting obsolete checkout.' % (self.RelPath(local=False),)) - - # Unlock and delink from the main worktree. We don't use git's worktree - # remove because it will recursively delete projects -- we handle that - # ourselves below. https://crbug.com/git/48 - if self.use_git_worktrees: - needle = platform_utils.realpath(self.gitdir) - # Find the git worktree commondir under .repo/worktrees/. - output = self.bare_git.worktree('list', '--porcelain').splitlines()[0] - assert output.startswith('worktree '), output - commondir = output[9:] - # Walk each of the git worktrees to see where they point. - configs = os.path.join(commondir, 'worktrees') - for name in os.listdir(configs): - gitdir = os.path.join(configs, name, 'gitdir') - with open(gitdir) as fp: - relpath = fp.read().strip() - # Resolve the checkout path and see if it matches this project. - fullpath = platform_utils.realpath(os.path.join(configs, name, relpath)) - if fullpath == needle: - platform_utils.rmtree(os.path.join(configs, name)) - - # Delete the .git directory first, so we're less likely to have a partially - # working git repository around. There shouldn't be any git projects here, - # so rmtree works. - - # Try to remove plain files first in case of git worktrees. If this fails - # for any reason, we'll fall back to rmtree, and that'll display errors if - # it can't remove things either. - try: - platform_utils.remove(self.gitdir) - except OSError: - pass - try: - platform_utils.rmtree(self.gitdir) - except OSError as e: - if e.errno != errno.ENOENT: - print('error: %s: %s' % (self.gitdir, e), file=sys.stderr) - print('error: %s: Failed to delete obsolete checkout; remove manually, ' - 'then run `repo sync -l`.' % (self.RelPath(local=False),), - file=sys.stderr) - return False - - # Delete everything under the worktree, except for directories that contain - # another git project. - dirs_to_remove = [] - failed = False - for root, dirs, files in platform_utils.walk(self.worktree): - for f in files: - path = os.path.join(root, f) - try: - platform_utils.remove(path) - except OSError as e: - if e.errno != errno.ENOENT: - print('error: %s: Failed to remove: %s' % (path, e), file=sys.stderr) - failed = True - dirs[:] = [d for d in dirs - if not os.path.lexists(os.path.join(root, d, '.git'))] - dirs_to_remove += [os.path.join(root, d) for d in dirs - if os.path.join(root, d) not in dirs_to_remove] - for d in reversed(dirs_to_remove): - if platform_utils.islink(d): try: - platform_utils.remove(d) - except OSError as e: - if e.errno != errno.ENOENT: - print('error: %s: Failed to remove: %s' % (d, e), file=sys.stderr) - failed = True - elif not platform_utils.listdir(d): - try: - platform_utils.rmdir(d) - except OSError as e: - if e.errno != errno.ENOENT: - print('error: %s: Failed to remove: %s' % (d, e), file=sys.stderr) - failed = True - if failed: - print('error: %s: Failed to delete obsolete checkout.' % (self.RelPath(local=False),), - file=sys.stderr) - print(' Remove manually, then run `repo sync -l`.', file=sys.stderr) - return False - - # Try deleting parent dirs if they are empty. - path = self.worktree - while path != self.manifest.topdir: - try: - platform_utils.rmdir(path) - except OSError as e: - if e.errno != errno.ENOENT: - break - path = os.path.dirname(path) - - return True - -# Branch Management ## - def StartBranch(self, name, branch_merge='', revision=None): - """Create a new branch off the manifest's revision. - """ - if not branch_merge: - branch_merge = self.revisionExpr - head = self.work_git.GetHead() - if head == (R_HEADS + name): - return True - - all_refs = self.bare_ref.all - if R_HEADS + name in all_refs: - return GitCommand(self, ['checkout', '-q', name, '--']).Wait() == 0 - - branch = self.GetBranch(name) - branch.remote = self.GetRemote() - branch.merge = branch_merge - if not branch.merge.startswith('refs/') and not ID_RE.match(branch_merge): - branch.merge = R_HEADS + branch_merge - - if revision is None: - revid = self.GetRevisionId(all_refs) - else: - revid = self.work_git.rev_parse(revision) - - if head.startswith(R_HEADS): - try: - head = all_refs[head] - except KeyError: - head = None - if revid and head and revid == head: - ref = R_HEADS + name - self.work_git.update_ref(ref, revid) - self.work_git.symbolic_ref(HEAD, ref) - branch.Save() - return True - - if GitCommand(self, ['checkout', '-q', '-b', branch.name, revid]).Wait() == 0: - branch.Save() - return True - return False - - def CheckoutBranch(self, name): - """Checkout a local topic branch. + return self.bare_git.rev_list(self.revisionExpr, "-1")[0] + except GitError: + raise ManifestInvalidRevisionError( + "revision %s in %s not found" % (self.revisionExpr, self.name) + ) - Args: - name: The name of the branch to checkout. + def GetRevisionId(self, all_refs=None): + if self.revisionId: + return self.revisionId - Returns: - True if the checkout succeeded; False if it didn't; None if the branch - didn't exist. - """ - rev = R_HEADS + name - head = self.work_git.GetHead() - if head == rev: - # Already on the branch - # - return True - - all_refs = self.bare_ref.all - try: - revid = all_refs[rev] - except KeyError: - # Branch does not exist in this project - # - return None - - if head.startswith(R_HEADS): - try: - head = all_refs[head] - except KeyError: - head = None - - if head == revid: - # Same revision; just update HEAD to point to the new - # target branch, but otherwise take no other action. - # - _lwrite(self.work_git.GetDotgitPath(subpath=HEAD), - 'ref: %s%s\n' % (R_HEADS, name)) - return True - - return GitCommand(self, - ['checkout', name, '--'], - capture_stdout=True, - capture_stderr=True).Wait() == 0 - - def AbandonBranch(self, name): - """Destroy a local topic branch. - - Args: - name: The name of the branch to abandon. + rem = self.GetRemote() + rev = rem.ToLocal(self.revisionExpr) - Returns: - True if the abandon succeeded; False if it didn't; None if the branch - didn't exist. - """ - rev = R_HEADS + name - all_refs = self.bare_ref.all - if rev not in all_refs: - # Doesn't exist - return None - - head = self.work_git.GetHead() - if head == rev: - # We can't destroy the branch while we are sitting - # on it. Switch to a detached HEAD. - # - head = all_refs[head] - - revid = self.GetRevisionId(all_refs) - if head == revid: - _lwrite(self.work_git.GetDotgitPath(subpath=HEAD), '%s\n' % revid) - else: - self._Checkout(revid, quiet=True) - - return GitCommand(self, - ['branch', '-D', name], - capture_stdout=True, - capture_stderr=True).Wait() == 0 - - def PruneHeads(self): - """Prune any topic branches already merged into upstream. - """ - cb = self.CurrentBranch - kill = [] - left = self._allrefs - for name in left.keys(): - if name.startswith(R_HEADS): - name = name[len(R_HEADS):] - if cb is None or name != cb: - kill.append(name) - - # Minor optimization: If there's nothing to prune, then don't try to read - # any project state. - if not kill and not cb: - return [] - - rev = self.GetRevisionId(left) - if cb is not None \ - and not self._revlist(HEAD + '...' + rev) \ - and not self.IsDirty(consider_untracked=False): - self.work_git.DetachHead(HEAD) - kill.append(cb) - - if kill: - old = self.bare_git.GetHead() - - try: - self.bare_git.DetachHead(rev) - - b = ['branch', '-d'] - b.extend(kill) - b = GitCommand(self, b, bare=True, - capture_stdout=True, - capture_stderr=True) - b.Wait() - finally: - if ID_RE.match(old): - self.bare_git.DetachHead(old) - else: - self.bare_git.SetHead(old) - left = self._allrefs + if all_refs is not None and rev in all_refs: + return all_refs[rev] - for branch in kill: - if (R_HEADS + branch) not in left: - self.CleanPublishedCache() - break + try: + return self.bare_git.rev_parse("--verify", "%s^0" % rev) + except GitError: + raise ManifestInvalidRevisionError( + "revision %s in %s not found" % (self.revisionExpr, self.name) + ) + + def SetRevisionId(self, revisionId): + if self.revisionExpr: + self.upstream = self.revisionExpr + + self.revisionId = revisionId + + def Sync_LocalHalf(self, syncbuf, force_sync=False, submodules=False): + """Perform only the local IO portion of the sync process. + + Network access is not required. + """ + if not os.path.exists(self.gitdir): + syncbuf.fail( + self, + "Cannot checkout %s due to missing network sync; Run " + "`repo sync -n %s` first." % (self.name, self.name), + ) + return + + self._InitWorkTree(force_sync=force_sync, submodules=submodules) + all_refs = self.bare_ref.all + self.CleanPublishedCache(all_refs) + revid = self.GetRevisionId(all_refs) + + # Special case the root of the repo client checkout. Make sure it + # doesn't contain files being checked out to dirs we don't allow. + if self.relpath == ".": + PROTECTED_PATHS = {".repo"} + paths = set( + self.work_git.ls_tree("-z", "--name-only", "--", revid).split( + "\0" + ) + ) + bad_paths = paths & PROTECTED_PATHS + if bad_paths: + syncbuf.fail( + self, + "Refusing to checkout project that writes to protected " + "paths: %s" % (", ".join(bad_paths),), + ) + return + + def _doff(): + self._FastForward(revid) + self._CopyAndLinkFiles() + + def _dosubmodules(): + self._SyncSubmodules(quiet=True) + + head = self.work_git.GetHead() + if head.startswith(R_HEADS): + branch = head[len(R_HEADS) :] + try: + head = all_refs[head] + except KeyError: + head = None + else: + branch = None + + if branch is None or syncbuf.detach_head: + # Currently on a detached HEAD. The user is assumed to + # not have any local modifications worth worrying about. + if self.IsRebaseInProgress(): + syncbuf.fail(self, _PriorSyncFailedError()) + return + + if head == revid: + # No changes; don't do anything further. + # Except if the head needs to be detached. + if not syncbuf.detach_head: + # The copy/linkfile config may have changed. + self._CopyAndLinkFiles() + return + else: + lost = self._revlist(not_rev(revid), HEAD) + if lost: + syncbuf.info(self, "discarding %d commits", len(lost)) - if cb and cb not in kill: - kill.append(cb) - kill.sort() + try: + self._Checkout(revid, quiet=True) + if submodules: + self._SyncSubmodules(quiet=True) + except GitError as e: + syncbuf.fail(self, e) + return + self._CopyAndLinkFiles() + return + + if head == revid: + # No changes; don't do anything further. + # + # The copy/linkfile config may have changed. + self._CopyAndLinkFiles() + return - kept = [] - for branch in kill: - if R_HEADS + branch in left: branch = self.GetBranch(branch) - base = branch.LocalMerge - if not base: - base = rev - kept.append(ReviewableBranch(self, branch, base)) - return kept - -# Submodule Management ## - def GetRegisteredSubprojects(self): - result = [] - - def rec(subprojects): - if not subprojects: - return - result.extend(subprojects) - for p in subprojects: - rec(p.subprojects) - rec(self.subprojects) - return result - - def _GetSubmodules(self): - # Unfortunately we cannot call `git submodule status --recursive` here - # because the working tree might not exist yet, and it cannot be used - # without a working tree in its current implementation. - - def get_submodules(gitdir, rev): - # Parse .gitmodules for submodule sub_paths and sub_urls - sub_paths, sub_urls = parse_gitmodules(gitdir, rev) - if not sub_paths: - return [] - # Run `git ls-tree` to read SHAs of submodule object, which happen to be - # revision of submodule repository - sub_revs = git_ls_tree(gitdir, rev, sub_paths) - submodules = [] - for sub_path, sub_url in zip(sub_paths, sub_urls): + + if not branch.LocalMerge: + # The current branch has no tracking configuration. + # Jump off it to a detached HEAD. + syncbuf.info( + self, "leaving %s; does not track upstream", branch.name + ) + try: + self._Checkout(revid, quiet=True) + if submodules: + self._SyncSubmodules(quiet=True) + except GitError as e: + syncbuf.fail(self, e) + return + self._CopyAndLinkFiles() + return + + upstream_gain = self._revlist(not_rev(HEAD), revid) + + # See if we can perform a fast forward merge. This can happen if our + # branch isn't in the exact same state as we last published. try: - sub_rev = sub_revs[sub_path] - except KeyError: - # Ignore non-exist submodules - continue - submodules.append((sub_rev, sub_path, sub_url)) - return submodules - - re_path = re.compile(r'^submodule\.(.+)\.path=(.*)$') - re_url = re.compile(r'^submodule\.(.+)\.url=(.*)$') - - def parse_gitmodules(gitdir, rev): - cmd = ['cat-file', 'blob', '%s:.gitmodules' % rev] - try: - p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True, - bare=True, gitdir=gitdir) - except GitError: - return [], [] - if p.Wait() != 0: - return [], [] - - gitmodules_lines = [] - fd, temp_gitmodules_path = tempfile.mkstemp() - try: - os.write(fd, p.stdout.encode('utf-8')) - os.close(fd) - cmd = ['config', '--file', temp_gitmodules_path, '--list'] - p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True, - bare=True, gitdir=gitdir) - if p.Wait() != 0: - return [], [] - gitmodules_lines = p.stdout.split('\n') - except GitError: - return [], [] - finally: - platform_utils.remove(temp_gitmodules_path) - - names = set() - paths = {} - urls = {} - for line in gitmodules_lines: - if not line: - continue - m = re_path.match(line) - if m: - names.add(m.group(1)) - paths[m.group(1)] = m.group(2) - continue - m = re_url.match(line) - if m: - names.add(m.group(1)) - urls[m.group(1)] = m.group(2) - continue - names = sorted(names) - return ([paths.get(name, '') for name in names], - [urls.get(name, '') for name in names]) - - def git_ls_tree(gitdir, rev, paths): - cmd = ['ls-tree', rev, '--'] - cmd.extend(paths) - try: - p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True, - bare=True, gitdir=gitdir) - except GitError: - return [] - if p.Wait() != 0: - return [] - objects = {} - for line in p.stdout.split('\n'): - if not line.strip(): - continue - object_rev, object_path = line.split()[2:4] - objects[object_path] = object_rev - return objects + self.work_git.merge_base("--is-ancestor", HEAD, revid) + # Skip the published logic. + pub = False + except GitError: + pub = self.WasPublished(branch.name, all_refs) + + if pub: + not_merged = self._revlist(not_rev(revid), pub) + if not_merged: + if upstream_gain: + # The user has published this branch and some of those + # commits are not yet merged upstream. We do not want + # to rewrite the published commits so we punt. + syncbuf.fail( + self, + "branch %s is published (but not merged) and is now " + "%d commits behind" % (branch.name, len(upstream_gain)), + ) + return + elif pub == head: + # All published commits are merged, and thus we are a + # strict subset. We can fast-forward safely. + syncbuf.later1(self, _doff) + if submodules: + syncbuf.later1(self, _dosubmodules) + return + + # Examine the local commits not in the remote. Find the + # last one attributed to this user, if any. + local_changes = self._revlist(not_rev(revid), HEAD, format="%H %ce") + last_mine = None + cnt_mine = 0 + for commit in local_changes: + commit_id, committer_email = commit.split(" ", 1) + if committer_email == self.UserEmail: + last_mine = commit_id + cnt_mine += 1 + + if not upstream_gain and cnt_mine == len(local_changes): + # The copy/linkfile config may have changed. + self._CopyAndLinkFiles() + return + + if self.IsDirty(consider_untracked=False): + syncbuf.fail(self, _DirtyError()) + return + + # If the upstream switched on us, warn the user. + if branch.merge != self.revisionExpr: + if branch.merge and self.revisionExpr: + syncbuf.info( + self, + "manifest switched %s...%s", + branch.merge, + self.revisionExpr, + ) + elif branch.merge: + syncbuf.info(self, "manifest no longer tracks %s", branch.merge) + + if cnt_mine < len(local_changes): + # Upstream rebased. Not everything in HEAD was created by this user. + syncbuf.info( + self, + "discarding %d commits removed from upstream", + len(local_changes) - cnt_mine, + ) + + branch.remote = self.GetRemote() + if not ID_RE.match(self.revisionExpr): + # In case of manifest sync the revisionExpr might be a SHA1. + branch.merge = self.revisionExpr + if not branch.merge.startswith("refs/"): + branch.merge = R_HEADS + branch.merge + branch.Save() + + if cnt_mine > 0 and self.rebase: + + def _docopyandlink(): + self._CopyAndLinkFiles() + + def _dorebase(): + self._Rebase(upstream="%s^1" % last_mine, onto=revid) + + syncbuf.later2(self, _dorebase) + if submodules: + syncbuf.later2(self, _dosubmodules) + syncbuf.later2(self, _docopyandlink) + elif local_changes: + try: + self._ResetHard(revid) + if submodules: + self._SyncSubmodules(quiet=True) + self._CopyAndLinkFiles() + except GitError as e: + syncbuf.fail(self, e) + return + else: + syncbuf.later1(self, _doff) + if submodules: + syncbuf.later1(self, _dosubmodules) - try: - rev = self.GetRevisionId() - except GitError: - return [] - return get_submodules(self.gitdir, rev) - - def GetDerivedSubprojects(self): - result = [] - if not self.Exists: - # If git repo does not exist yet, querying its submodules will - # mess up its states; so return here. - return result - for rev, path, url in self._GetSubmodules(): - name = self.manifest.GetSubprojectName(self, path) - relpath, worktree, gitdir, objdir = \ - self.manifest.GetSubprojectPaths(self, name, path) - project = self.manifest.paths.get(relpath) - if project: - result.extend(project.GetDerivedSubprojects()) - continue - - if url.startswith('..'): - url = urllib.parse.urljoin("%s/" % self.remote.url, url) - remote = RemoteSpec(self.remote.name, - url=url, - pushUrl=self.remote.pushUrl, - review=self.remote.review, - revision=self.remote.revision) - subproject = Project(manifest=self.manifest, - name=name, - remote=remote, - gitdir=gitdir, - objdir=objdir, - worktree=worktree, - relpath=relpath, - revisionExpr=rev, - revisionId=rev, - rebase=self.rebase, - groups=self.groups, - sync_c=self.sync_c, - sync_s=self.sync_s, - sync_tags=self.sync_tags, - parent=self, - is_derived=True) - result.append(subproject) - result.extend(subproject.GetDerivedSubprojects()) - return result - -# Direct Git Commands ## - def EnableRepositoryExtension(self, key, value='true', version=1): - """Enable git repository extension |key| with |value|. - - Args: - key: The extension to enabled. Omit the "extensions." prefix. - value: The value to use for the extension. - version: The minimum git repository version needed. - """ - # Make sure the git repo version is new enough already. - found_version = self.config.GetInt('core.repositoryFormatVersion') - if found_version is None: - found_version = 0 - if found_version < version: - self.config.SetString('core.repositoryFormatVersion', str(version)) + def AddCopyFile(self, src, dest, topdir): + """Mark |src| for copying to |dest| (relative to |topdir|). - # Enable the extension! - self.config.SetString('extensions.%s' % (key,), value) + No filesystem changes occur here. Actual copying happens later on. - def ResolveRemoteHead(self, name=None): - """Find out what the default branch (HEAD) points to. + Paths should have basic validation run on them before being queued. + Further checking will be handled when the actual copy happens. + """ + self.copyfiles.append(_CopyFile(self.worktree, src, topdir, dest)) - Normally this points to refs/heads/master, but projects are moving to main. - Support whatever the server uses rather than hardcoding "master" ourselves. - """ - if name is None: - name = self.remote.name + def AddLinkFile(self, src, dest, topdir): + """Mark |dest| to create a symlink (relative to |topdir|) pointing to + |src|. - # The output will look like (NB: tabs are separators): - # ref: refs/heads/master HEAD - # 5f6803b100bb3cd0f534e96e88c91373e8ed1c44 HEAD - output = self.bare_git.ls_remote('-q', '--symref', '--exit-code', name, 'HEAD') + No filesystem changes occur here. Actual linking happens later on. - for line in output.splitlines(): - lhs, rhs = line.split('\t', 1) - if rhs == 'HEAD' and lhs.startswith('ref:'): - return lhs[4:].strip() + Paths should have basic validation run on them before being queued. + Further checking will be handled when the actual link happens. + """ + self.linkfiles.append(_LinkFile(self.worktree, src, topdir, dest)) - return None + def AddAnnotation(self, name, value, keep): + self.annotations.append(Annotation(name, value, keep)) - def _CheckForImmutableRevision(self): - try: - # if revision (sha or tag) is not present then following function - # throws an error. - self.bare_git.rev_list('-1', '--missing=allow-any', - '%s^0' % self.revisionExpr, '--') - if self.upstream: - rev = self.GetRemote().ToLocal(self.upstream) - self.bare_git.rev_list('-1', '--missing=allow-any', - '%s^0' % rev, '--') - self.bare_git.merge_base('--is-ancestor', self.revisionExpr, rev) - return True - except GitError: - # There is no such persistent revision. We have to fetch it. - return False - - def _FetchArchive(self, tarpath, cwd=None): - cmd = ['archive', '-v', '-o', tarpath] - cmd.append('--remote=%s' % self.remote.url) - cmd.append('--prefix=%s/' % self.RelPath(local=False)) - cmd.append(self.revisionExpr) - - command = GitCommand(self, cmd, cwd=cwd, - capture_stdout=True, - capture_stderr=True) - - if command.Wait() != 0: - raise GitError('git archive %s: %s' % (self.name, command.stderr)) - - def _RemoteFetch(self, name=None, - current_branch_only=False, - initial=False, - quiet=False, - verbose=False, - output_redir=None, - alt_dir=None, - tags=True, - prune=False, - depth=None, - submodules=False, - ssh_proxy=None, - force_sync=False, - clone_filter=None, - retry_fetches=2, - retry_sleep_initial_sec=4.0, - retry_exp_factor=2.0): - is_sha1 = False - tag_name = None - # The depth should not be used when fetching to a mirror because - # it will result in a shallow repository that cannot be cloned or - # fetched from. - # The repo project should also never be synced with partial depth. - if self.manifest.IsMirror or self.relpath == '.repo/repo': - depth = None - - if depth: - current_branch_only = True - - if ID_RE.match(self.revisionExpr) is not None: - is_sha1 = True - - if current_branch_only: - if self.revisionExpr.startswith(R_TAGS): - # This is a tag and its commit id should never change. - tag_name = self.revisionExpr[len(R_TAGS):] - elif self.upstream and self.upstream.startswith(R_TAGS): - # This is a tag and its commit id should never change. - tag_name = self.upstream[len(R_TAGS):] - - if is_sha1 or tag_name is not None: - if self._CheckForImmutableRevision(): - if verbose: - print('Skipped fetching project %s (already have persistent ref)' - % self.name) - return True - if is_sha1 and not depth: - # When syncing a specific commit and --depth is not set: - # * if upstream is explicitly specified and is not a sha1, fetch only - # upstream as users expect only upstream to be fetch. - # Note: The commit might not be in upstream in which case the sync - # will fail. - # * otherwise, fetch all branches to make sure we end up with the - # specific commit. - if self.upstream: - current_branch_only = not ID_RE.match(self.upstream) - else: - current_branch_only = False + def DownloadPatchSet(self, change_id, patch_id): + """Download a single patch set of a single change to FETCH_HEAD.""" + remote = self.GetRemote() - if not name: - name = self.remote.name + cmd = ["fetch", remote.name] + cmd.append( + "refs/changes/%2.2d/%d/%d" % (change_id % 100, change_id, patch_id) + ) + if GitCommand(self, cmd, bare=True).Wait() != 0: + return None + return DownloadedChange( + self, + self.GetRevisionId(), + change_id, + patch_id, + self.bare_git.rev_parse("FETCH_HEAD"), + ) - remote = self.GetRemote(name) - if not remote.PreConnectFetch(ssh_proxy): - ssh_proxy = None + def DeleteWorktree(self, quiet=False, force=False): + """Delete the source checkout and any other housekeeping tasks. - if initial: - if alt_dir and 'objects' == os.path.basename(alt_dir): - ref_dir = os.path.dirname(alt_dir) - packed_refs = os.path.join(self.gitdir, 'packed-refs') + This currently leaves behind the internal .repo/ cache state. This + helps when switching branches or manifest changes get reverted as we + don't have to redownload all the git objects. But we should do some GC + at some point. - all_refs = self.bare_ref.all - ids = set(all_refs.values()) - tmp = set() - - for r, ref_id in GitRefs(ref_dir).all.items(): - if r not in all_refs: - if r.startswith(R_TAGS) or remote.WritesTo(r): - all_refs[r] = ref_id - ids.add(ref_id) - continue - - if ref_id in ids: - continue - - r = 'refs/_alt/%s' % ref_id - all_refs[r] = ref_id - ids.add(ref_id) - tmp.add(r) - - tmp_packed_lines = [] - old_packed_lines = [] - - for r in sorted(all_refs): - line = '%s %s\n' % (all_refs[r], r) - tmp_packed_lines.append(line) - if r not in tmp: - old_packed_lines.append(line) - - tmp_packed = ''.join(tmp_packed_lines) - old_packed = ''.join(old_packed_lines) - _lwrite(packed_refs, tmp_packed) - else: - alt_dir = None - - cmd = ['fetch'] - - if clone_filter: - git_require((2, 19, 0), fail=True, msg='partial clones') - cmd.append('--filter=%s' % clone_filter) - self.EnableRepositoryExtension('partialclone', self.remote.name) - - if depth: - cmd.append('--depth=%s' % depth) - else: - # If this repo has shallow objects, then we don't know which refs have - # shallow objects or not. Tell git to unshallow all fetched refs. Don't - # do this with projects that don't have shallow objects, since it is less - # efficient. - if os.path.exists(os.path.join(self.gitdir, 'shallow')): - cmd.append('--depth=2147483647') - - if not verbose: - cmd.append('--quiet') - if not quiet and sys.stdout.isatty(): - cmd.append('--progress') - if not self.worktree: - cmd.append('--update-head-ok') - cmd.append(name) - - if force_sync: - cmd.append('--force') - - if prune: - cmd.append('--prune') - - # Always pass something for --recurse-submodules, git with GIT_DIR behaves - # incorrectly when not given `--recurse-submodules=no`. (b/218891912) - cmd.append(f'--recurse-submodules={"on-demand" if submodules else "no"}') - - spec = [] - if not current_branch_only: - # Fetch whole repo - spec.append(str((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*'))) - elif tag_name is not None: - spec.append('tag') - spec.append(tag_name) - - if self.manifest.IsMirror and not current_branch_only: - branch = None - else: - branch = self.revisionExpr - if (not self.manifest.IsMirror and is_sha1 and depth - and git_require((1, 8, 3))): - # Shallow checkout of a specific commit, fetch from that commit and not - # the heads only as the commit might be deeper in the history. - spec.append(branch) - if self.upstream: - spec.append(self.upstream) - else: - if is_sha1: - branch = self.upstream - if branch is not None and branch.strip(): - if not branch.startswith('refs/'): - branch = R_HEADS + branch - spec.append(str((u'+%s:' % branch) + remote.ToLocal(branch))) - - # If mirroring repo and we cannot deduce the tag or branch to fetch, fetch - # whole repo. - if self.manifest.IsMirror and not spec: - spec.append(str((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*'))) - - # If using depth then we should not get all the tags since they may - # be outside of the depth. - if not tags or depth: - cmd.append('--no-tags') - else: - cmd.append('--tags') - spec.append(str((u'+refs/tags/*:') + remote.ToLocal('refs/tags/*'))) - - cmd.extend(spec) - - # At least one retry minimum due to git remote prune. - retry_fetches = max(retry_fetches, 2) - retry_cur_sleep = retry_sleep_initial_sec - ok = prune_tried = False - for try_n in range(retry_fetches): - gitcmd = GitCommand( - self, cmd, bare=True, objdir=os.path.join(self.objdir, 'objects'), - ssh_proxy=ssh_proxy, - merge_output=True, capture_stdout=quiet or bool(output_redir)) - if gitcmd.stdout and not quiet and output_redir: - output_redir.write(gitcmd.stdout) - ret = gitcmd.Wait() - if ret == 0: - ok = True - break - - # Retry later due to HTTP 429 Too Many Requests. - elif (gitcmd.stdout and - 'error:' in gitcmd.stdout and - 'HTTP 429' in gitcmd.stdout): - # Fallthru to sleep+retry logic at the bottom. - pass - - # Try to prune remote branches once in case there are conflicts. - # For example, if the remote had refs/heads/upstream, but deleted that and - # now has refs/heads/upstream/foo. - elif (gitcmd.stdout and - 'error:' in gitcmd.stdout and - 'git remote prune' in gitcmd.stdout and - not prune_tried): - prune_tried = True - prunecmd = GitCommand(self, ['remote', 'prune', name], bare=True, - ssh_proxy=ssh_proxy) - ret = prunecmd.Wait() - if ret: - break - print('retrying fetch after pruning remote branches', file=output_redir) - # Continue right away so we don't sleep as we shouldn't need to. - continue - elif current_branch_only and is_sha1 and ret == 128: - # Exit code 128 means "couldn't find the ref you asked for"; if we're - # in sha1 mode, we just tried sync'ing from the upstream field; it - # doesn't exist, thus abort the optimization attempt and do a full sync. - break - elif ret < 0: - # Git died with a signal, exit immediately - break - - # Figure out how long to sleep before the next attempt, if there is one. - if not verbose and gitcmd.stdout: - print('\n%s:\n%s' % (self.name, gitcmd.stdout), end='', file=output_redir) - if try_n < retry_fetches - 1: - print('%s: sleeping %s seconds before retrying' % (self.name, retry_cur_sleep), - file=output_redir) - time.sleep(retry_cur_sleep) - retry_cur_sleep = min(retry_exp_factor * retry_cur_sleep, - MAXIMUM_RETRY_SLEEP_SEC) - retry_cur_sleep *= (1 - random.uniform(-RETRY_JITTER_PERCENT, - RETRY_JITTER_PERCENT)) - - if initial: - if alt_dir: - if old_packed != '': - _lwrite(packed_refs, old_packed) - else: - platform_utils.remove(packed_refs) - self.bare_git.pack_refs('--all', '--prune') - - if is_sha1 and current_branch_only: - # We just synced the upstream given branch; verify we - # got what we wanted, else trigger a second run of all - # refs. - if not self._CheckForImmutableRevision(): - # Sync the current branch only with depth set to None. - # We always pass depth=None down to avoid infinite recursion. - return self._RemoteFetch( - name=name, quiet=quiet, verbose=verbose, output_redir=output_redir, - current_branch_only=current_branch_only and depth, - initial=False, alt_dir=alt_dir, - depth=None, ssh_proxy=ssh_proxy, clone_filter=clone_filter) - - return ok - - def _ApplyCloneBundle(self, initial=False, quiet=False, verbose=False): - if initial and (self.manifest.manifestProject.depth or self.clone_depth): - return False - - remote = self.GetRemote() - bundle_url = remote.url + '/clone.bundle' - bundle_url = GitConfig.ForUser().UrlInsteadOf(bundle_url) - if GetSchemeFromUrl(bundle_url) not in ('http', 'https', - 'persistent-http', - 'persistent-https'): - return False - - bundle_dst = os.path.join(self.gitdir, 'clone.bundle') - bundle_tmp = os.path.join(self.gitdir, 'clone.bundle.tmp') - - exist_dst = os.path.exists(bundle_dst) - exist_tmp = os.path.exists(bundle_tmp) - - if not initial and not exist_dst and not exist_tmp: - return False - - if not exist_dst: - exist_dst = self._FetchBundle(bundle_url, bundle_tmp, bundle_dst, quiet, - verbose) - if not exist_dst: - return False - - cmd = ['fetch'] - if not verbose: - cmd.append('--quiet') - if not quiet and sys.stdout.isatty(): - cmd.append('--progress') - if not self.worktree: - cmd.append('--update-head-ok') - cmd.append(bundle_dst) - for f in remote.fetch: - cmd.append(str(f)) - cmd.append('+refs/tags/*:refs/tags/*') - - ok = GitCommand( - self, cmd, bare=True, objdir=os.path.join(self.objdir, 'objects')).Wait() == 0 - platform_utils.remove(bundle_dst, missing_ok=True) - platform_utils.remove(bundle_tmp, missing_ok=True) - return ok - - def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet, verbose): - platform_utils.remove(dstPath, missing_ok=True) - - cmd = ['curl', '--fail', '--output', tmpPath, '--netrc', '--location'] - if quiet: - cmd += ['--silent', '--show-error'] - if os.path.exists(tmpPath): - size = os.stat(tmpPath).st_size - if size >= 1024: - cmd += ['--continue-at', '%d' % (size,)] - else: - platform_utils.remove(tmpPath) - with GetUrlCookieFile(srcUrl, quiet) as (cookiefile, proxy): - if cookiefile: - cmd += ['--cookie', cookiefile] - if proxy: - cmd += ['--proxy', proxy] - elif 'http_proxy' in os.environ and 'darwin' == sys.platform: - cmd += ['--proxy', os.environ['http_proxy']] - if srcUrl.startswith('persistent-https'): - srcUrl = 'http' + srcUrl[len('persistent-https'):] - elif srcUrl.startswith('persistent-http'): - srcUrl = 'http' + srcUrl[len('persistent-http'):] - cmd += [srcUrl] - - proc = None - with Trace('Fetching bundle: %s', ' '.join(cmd)): - if verbose: - print('%s: Downloading bundle: %s' % (self.name, srcUrl)) - stdout = None if verbose else subprocess.PIPE - stderr = None if verbose else subprocess.STDOUT + Args: + quiet: Whether to hide normal messages. + force: Always delete tree even if dirty. + + Returns: + True if the worktree was completely cleaned out. + """ + if self.IsDirty(): + if force: + print( + "warning: %s: Removing dirty project: uncommitted changes " + "lost." % (self.RelPath(local=False),), + file=sys.stderr, + ) + else: + print( + "error: %s: Cannot remove project: uncommitted changes are " + "present.\n" % (self.RelPath(local=False),), + file=sys.stderr, + ) + return False + + if not quiet: + print( + "%s: Deleting obsolete checkout." % (self.RelPath(local=False),) + ) + + # Unlock and delink from the main worktree. We don't use git's worktree + # remove because it will recursively delete projects -- we handle that + # ourselves below. https://crbug.com/git/48 + if self.use_git_worktrees: + needle = platform_utils.realpath(self.gitdir) + # Find the git worktree commondir under .repo/worktrees/. + output = self.bare_git.worktree("list", "--porcelain").splitlines()[ + 0 + ] + assert output.startswith("worktree "), output + commondir = output[9:] + # Walk each of the git worktrees to see where they point. + configs = os.path.join(commondir, "worktrees") + for name in os.listdir(configs): + gitdir = os.path.join(configs, name, "gitdir") + with open(gitdir) as fp: + relpath = fp.read().strip() + # Resolve the checkout path and see if it matches this project. + fullpath = platform_utils.realpath( + os.path.join(configs, name, relpath) + ) + if fullpath == needle: + platform_utils.rmtree(os.path.join(configs, name)) + + # Delete the .git directory first, so we're less likely to have a + # partially working git repository around. There shouldn't be any git + # projects here, so rmtree works. + + # Try to remove plain files first in case of git worktrees. If this + # fails for any reason, we'll fall back to rmtree, and that'll display + # errors if it can't remove things either. try: - proc = subprocess.Popen(cmd, stdout=stdout, stderr=stderr) + platform_utils.remove(self.gitdir) except OSError: - return False - - (output, _) = proc.communicate() - curlret = proc.returncode - - if curlret == 22: - # From curl man page: - # 22: HTTP page not retrieved. The requested url was not found or - # returned another error with the HTTP error code being 400 or above. - # This return code only appears if -f, --fail is used. - if verbose: - print('%s: Unable to retrieve clone.bundle; ignoring.' % self.name) - if output: - print('Curl output:\n%s' % output) - return False - elif curlret and not verbose and output: - print('%s' % output, file=sys.stderr) + pass + try: + platform_utils.rmtree(self.gitdir) + except OSError as e: + if e.errno != errno.ENOENT: + print("error: %s: %s" % (self.gitdir, e), file=sys.stderr) + print( + "error: %s: Failed to delete obsolete checkout; remove " + "manually, then run `repo sync -l`." + % (self.RelPath(local=False),), + file=sys.stderr, + ) + return False + + # Delete everything under the worktree, except for directories that + # contain another git project. + dirs_to_remove = [] + failed = False + for root, dirs, files in platform_utils.walk(self.worktree): + for f in files: + path = os.path.join(root, f) + try: + platform_utils.remove(path) + except OSError as e: + if e.errno != errno.ENOENT: + print( + "error: %s: Failed to remove: %s" % (path, e), + file=sys.stderr, + ) + failed = True + dirs[:] = [ + d + for d in dirs + if not os.path.lexists(os.path.join(root, d, ".git")) + ] + dirs_to_remove += [ + os.path.join(root, d) + for d in dirs + if os.path.join(root, d) not in dirs_to_remove + ] + for d in reversed(dirs_to_remove): + if platform_utils.islink(d): + try: + platform_utils.remove(d) + except OSError as e: + if e.errno != errno.ENOENT: + print( + "error: %s: Failed to remove: %s" % (d, e), + file=sys.stderr, + ) + failed = True + elif not platform_utils.listdir(d): + try: + platform_utils.rmdir(d) + except OSError as e: + if e.errno != errno.ENOENT: + print( + "error: %s: Failed to remove: %s" % (d, e), + file=sys.stderr, + ) + failed = True + if failed: + print( + "error: %s: Failed to delete obsolete checkout." + % (self.RelPath(local=False),), + file=sys.stderr, + ) + print( + " Remove manually, then run `repo sync -l`.", + file=sys.stderr, + ) + return False + + # Try deleting parent dirs if they are empty. + path = self.worktree + while path != self.manifest.topdir: + try: + platform_utils.rmdir(path) + except OSError as e: + if e.errno != errno.ENOENT: + break + path = os.path.dirname(path) - if os.path.exists(tmpPath): - if curlret == 0 and self._IsValidBundle(tmpPath, quiet): - platform_utils.rename(tmpPath, dstPath) return True - else: - platform_utils.remove(tmpPath) - return False - else: - return False - def _IsValidBundle(self, path, quiet): - try: - with open(path, 'rb') as f: - if f.read(16) == b'# v2 git bundle\n': - return True + def StartBranch(self, name, branch_merge="", revision=None): + """Create a new branch off the manifest's revision.""" + if not branch_merge: + branch_merge = self.revisionExpr + head = self.work_git.GetHead() + if head == (R_HEADS + name): + return True + + all_refs = self.bare_ref.all + if R_HEADS + name in all_refs: + return GitCommand(self, ["checkout", "-q", name, "--"]).Wait() == 0 + + branch = self.GetBranch(name) + branch.remote = self.GetRemote() + branch.merge = branch_merge + if not branch.merge.startswith("refs/") and not ID_RE.match( + branch_merge + ): + branch.merge = R_HEADS + branch_merge + + if revision is None: + revid = self.GetRevisionId(all_refs) else: - if not quiet: - print("Invalid clone.bundle file; ignoring.", file=sys.stderr) - return False - except OSError: - return False - - def _Checkout(self, rev, quiet=False): - cmd = ['checkout'] - if quiet: - cmd.append('-q') - cmd.append(rev) - cmd.append('--') - if GitCommand(self, cmd).Wait() != 0: - if self._allrefs: - raise GitError('%s checkout %s ' % (self.name, rev)) - - def _CherryPick(self, rev, ffonly=False, record_origin=False): - cmd = ['cherry-pick'] - if ffonly: - cmd.append('--ff') - if record_origin: - cmd.append('-x') - cmd.append(rev) - cmd.append('--') - if GitCommand(self, cmd).Wait() != 0: - if self._allrefs: - raise GitError('%s cherry-pick %s ' % (self.name, rev)) - - def _LsRemote(self, refs): - cmd = ['ls-remote', self.remote.name, refs] - p = GitCommand(self, cmd, capture_stdout=True) - if p.Wait() == 0: - return p.stdout - return None - - def _Revert(self, rev): - cmd = ['revert'] - cmd.append('--no-edit') - cmd.append(rev) - cmd.append('--') - if GitCommand(self, cmd).Wait() != 0: - if self._allrefs: - raise GitError('%s revert %s ' % (self.name, rev)) - - def _ResetHard(self, rev, quiet=True): - cmd = ['reset', '--hard'] - if quiet: - cmd.append('-q') - cmd.append(rev) - if GitCommand(self, cmd).Wait() != 0: - raise GitError('%s reset --hard %s ' % (self.name, rev)) - - def _SyncSubmodules(self, quiet=True): - cmd = ['submodule', 'update', '--init', '--recursive'] - if quiet: - cmd.append('-q') - if GitCommand(self, cmd).Wait() != 0: - raise GitError('%s submodule update --init --recursive ' % self.name) - - def _Rebase(self, upstream, onto=None): - cmd = ['rebase'] - if onto is not None: - cmd.extend(['--onto', onto]) - cmd.append(upstream) - if GitCommand(self, cmd).Wait() != 0: - raise GitError('%s rebase %s ' % (self.name, upstream)) - - def _FastForward(self, head, ffonly=False): - cmd = ['merge', '--no-stat', head] - if ffonly: - cmd.append("--ff-only") - if GitCommand(self, cmd).Wait() != 0: - raise GitError('%s merge %s ' % (self.name, head)) - - def _InitGitDir(self, mirror_git=None, force_sync=False, quiet=False): - init_git_dir = not os.path.exists(self.gitdir) - init_obj_dir = not os.path.exists(self.objdir) - try: - # Initialize the bare repository, which contains all of the objects. - if init_obj_dir: - os.makedirs(self.objdir) - self.bare_objdir.init() + revid = self.work_git.rev_parse(revision) + + if head.startswith(R_HEADS): + try: + head = all_refs[head] + except KeyError: + head = None + if revid and head and revid == head: + ref = R_HEADS + name + self.work_git.update_ref(ref, revid) + self.work_git.symbolic_ref(HEAD, ref) + branch.Save() + return True + + if ( + GitCommand( + self, ["checkout", "-q", "-b", branch.name, revid] + ).Wait() + == 0 + ): + branch.Save() + return True + return False - self._UpdateHooks(quiet=quiet) + def CheckoutBranch(self, name): + """Checkout a local topic branch. - if self.use_git_worktrees: - # Enable per-worktree config file support if possible. This is more a - # nice-to-have feature for users rather than a hard requirement. - if git_require((2, 20, 0)): - self.EnableRepositoryExtension('worktreeConfig') - - # If we have a separate directory to hold refs, initialize it as well. - if self.objdir != self.gitdir: - if init_git_dir: - os.makedirs(self.gitdir) - - if init_obj_dir or init_git_dir: - self._ReferenceGitDir(self.objdir, self.gitdir, copy_all=True) + Args: + name: The name of the branch to checkout. + + Returns: + True if the checkout succeeded; False if it didn't; None if the + branch didn't exist. + """ + rev = R_HEADS + name + head = self.work_git.GetHead() + if head == rev: + # Already on the branch. + return True + + all_refs = self.bare_ref.all try: - self._CheckDirReference(self.objdir, self.gitdir) - except GitError as e: - if force_sync: - print("Retrying clone after deleting %s" % - self.gitdir, file=sys.stderr) + revid = all_refs[rev] + except KeyError: + # Branch does not exist in this project. + return None + + if head.startswith(R_HEADS): try: - platform_utils.rmtree(platform_utils.realpath(self.gitdir)) - if self.worktree and os.path.exists(platform_utils.realpath - (self.worktree)): - platform_utils.rmtree(platform_utils.realpath(self.worktree)) - return self._InitGitDir(mirror_git=mirror_git, force_sync=False, - quiet=quiet) - except Exception: - raise e - raise e - - if init_git_dir: - mp = self.manifest.manifestProject - ref_dir = mp.reference or '' - - def _expanded_ref_dirs(): - """Iterate through the possible git reference directory paths.""" - name = self.name + '.git' - yield mirror_git or os.path.join(ref_dir, name) - for prefix in '', self.remote.name: - yield os.path.join(ref_dir, '.repo', 'project-objects', prefix, name) - yield os.path.join(ref_dir, '.repo', 'worktrees', prefix, name) - - if ref_dir or mirror_git: - found_ref_dir = None - for path in _expanded_ref_dirs(): - if os.path.exists(path): - found_ref_dir = path - break - ref_dir = found_ref_dir - - if ref_dir: - if not os.path.isabs(ref_dir): - # The alternate directory is relative to the object database. - ref_dir = os.path.relpath(ref_dir, - os.path.join(self.objdir, 'objects')) - _lwrite(os.path.join(self.objdir, 'objects/info/alternates'), - os.path.join(ref_dir, 'objects') + '\n') - - m = self.manifest.manifestProject.config - for key in ['user.name', 'user.email']: - if m.Has(key, include_defaults=False): - self.config.SetString(key, m.GetString(key)) - if not self.manifest.EnableGitLfs: - self.config.SetString('filter.lfs.smudge', 'git-lfs smudge --skip -- %f') - self.config.SetString('filter.lfs.process', 'git-lfs filter-process --skip') - self.config.SetBoolean('core.bare', True if self.manifest.IsMirror else None) - except Exception: - if init_obj_dir and os.path.exists(self.objdir): - platform_utils.rmtree(self.objdir) - if init_git_dir and os.path.exists(self.gitdir): - platform_utils.rmtree(self.gitdir) - raise - - def _UpdateHooks(self, quiet=False): - if os.path.exists(self.objdir): - self._InitHooks(quiet=quiet) - - def _InitHooks(self, quiet=False): - hooks = platform_utils.realpath(os.path.join(self.objdir, 'hooks')) - if not os.path.exists(hooks): - os.makedirs(hooks) - - # Delete sample hooks. They're noise. - for hook in glob.glob(os.path.join(hooks, '*.sample')): - try: - platform_utils.remove(hook, missing_ok=True) - except PermissionError: - pass - - for stock_hook in _ProjectHooks(): - name = os.path.basename(stock_hook) - - if name in ('commit-msg',) and not self.remote.review \ - and self is not self.manifest.manifestProject: - # Don't install a Gerrit Code Review hook if this - # project does not appear to use it for reviews. - # - # Since the manifest project is one of those, but also - # managed through gerrit, it's excluded - continue - - dst = os.path.join(hooks, name) - if platform_utils.islink(dst): - continue - if os.path.exists(dst): - # If the files are the same, we'll leave it alone. We create symlinks - # below by default but fallback to hardlinks if the OS blocks them. - # So if we're here, it's probably because we made a hardlink below. - if not filecmp.cmp(stock_hook, dst, shallow=False): - if not quiet: - _warn("%s: Not replacing locally modified %s hook", - self.RelPath(local=False), name) - continue - try: - platform_utils.symlink( - os.path.relpath(stock_hook, os.path.dirname(dst)), dst) - except OSError as e: - if e.errno == errno.EPERM: - try: - os.link(stock_hook, dst) - except OSError: - raise GitError(self._get_symlink_error_message()) - else: - raise - - def _InitRemote(self): - if self.remote.url: - remote = self.GetRemote() - remote.url = self.remote.url - remote.pushUrl = self.remote.pushUrl - remote.review = self.remote.review - remote.projectname = self.name - - if self.worktree: - remote.ResetFetch(mirror=False) - else: - remote.ResetFetch(mirror=True) - remote.Save() - - def _InitMRef(self): - """Initialize the pseudo m/ ref.""" - if self.manifest.branch: - if self.use_git_worktrees: - # Set up the m/ space to point to the worktree-specific ref space. - # We'll update the worktree-specific ref space on each checkout. - ref = R_M + self.manifest.branch - if not self.bare_ref.symref(ref): - self.bare_git.symbolic_ref( - '-m', 'redirecting to worktree scope', - ref, R_WORKTREE_M + self.manifest.branch) - - # We can't update this ref with git worktrees until it exists. - # We'll wait until the initial checkout to set it. - if not os.path.exists(self.worktree): - return - - base = R_WORKTREE_M - active_git = self.work_git - - self._InitAnyMRef(HEAD, self.bare_git, detach=True) - else: - base = R_M - active_git = self.bare_git - - self._InitAnyMRef(base + self.manifest.branch, active_git) - - def _InitMirrorHead(self): - self._InitAnyMRef(HEAD, self.bare_git) - - def _InitAnyMRef(self, ref, active_git, detach=False): - """Initialize |ref| in |active_git| to the value in the manifest. - - This points |ref| to the setting in the manifest. - - Args: - ref: The branch to update. - active_git: The git repository to make updates in. - detach: Whether to update target of symbolic refs, or overwrite the ref - directly (and thus make it non-symbolic). - """ - cur = self.bare_ref.symref(ref) - - if self.revisionId: - if cur != '' or self.bare_ref.get(ref) != self.revisionId: - msg = 'manifest set to %s' % self.revisionId - dst = self.revisionId + '^0' - active_git.UpdateRef(ref, dst, message=msg, detach=True) - else: - remote = self.GetRemote() - dst = remote.ToLocal(self.revisionExpr) - if cur != dst: - msg = 'manifest set to %s' % self.revisionExpr - if detach: - active_git.UpdateRef(ref, dst, message=msg, detach=True) - else: - active_git.symbolic_ref('-m', msg, ref, dst) - - def _CheckDirReference(self, srcdir, destdir): - # Git worktrees don't use symlinks to share at all. - if self.use_git_worktrees: - return - - for name in self.shareable_dirs: - # Try to self-heal a bit in simple cases. - dst_path = os.path.join(destdir, name) - src_path = os.path.join(srcdir, name) - - dst = platform_utils.realpath(dst_path) - if os.path.lexists(dst): - src = platform_utils.realpath(src_path) - # Fail if the links are pointing to the wrong place - if src != dst: - _error('%s is different in %s vs %s', name, destdir, srcdir) - raise GitError('--force-sync not enabled; cannot overwrite a local ' - 'work tree. If you\'re comfortable with the ' - 'possibility of losing the work tree\'s git metadata,' - ' use `repo sync --force-sync {0}` to ' - 'proceed.'.format(self.RelPath(local=False))) - - def _ReferenceGitDir(self, gitdir, dotgit, copy_all): - """Update |dotgit| to reference |gitdir|, using symlinks where possible. - - Args: - gitdir: The bare git repository. Must already be initialized. - dotgit: The repository you would like to initialize. - copy_all: If true, copy all remaining files from |gitdir| -> |dotgit|. - This saves you the effort of initializing |dotgit| yourself. - """ - symlink_dirs = self.shareable_dirs[:] - to_symlink = symlink_dirs - - to_copy = [] - if copy_all: - to_copy = platform_utils.listdir(gitdir) - - dotgit = platform_utils.realpath(dotgit) - for name in set(to_copy).union(to_symlink): - try: - src = platform_utils.realpath(os.path.join(gitdir, name)) - dst = os.path.join(dotgit, name) - - if os.path.lexists(dst): - continue - - # If the source dir doesn't exist, create an empty dir. - if name in symlink_dirs and not os.path.lexists(src): - os.makedirs(src) - - if name in to_symlink: - platform_utils.symlink( - os.path.relpath(src, os.path.dirname(dst)), dst) - elif copy_all and not platform_utils.islink(dst): - if platform_utils.isdir(src): - shutil.copytree(src, dst) - elif os.path.isfile(src): - shutil.copy(src, dst) - - except OSError as e: - if e.errno == errno.EPERM: - raise DownloadError(self._get_symlink_error_message()) - else: - raise - - def _InitGitWorktree(self): - """Init the project using git worktrees.""" - self.bare_git.worktree('prune') - self.bare_git.worktree('add', '-ff', '--checkout', '--detach', '--lock', - self.worktree, self.GetRevisionId()) - - # Rewrite the internal state files to use relative paths between the - # checkouts & worktrees. - dotgit = os.path.join(self.worktree, '.git') - with open(dotgit, 'r') as fp: - # Figure out the checkout->worktree path. - setting = fp.read() - assert setting.startswith('gitdir:') - git_worktree_path = setting.split(':', 1)[1].strip() - # Some platforms (e.g. Windows) won't let us update dotgit in situ because - # of file permissions. Delete it and recreate it from scratch to avoid. - platform_utils.remove(dotgit) - # Use relative path from checkout->worktree & maintain Unix line endings - # on all OS's to match git behavior. - with open(dotgit, 'w', newline='\n') as fp: - print('gitdir:', os.path.relpath(git_worktree_path, self.worktree), - file=fp) - # Use relative path from worktree->checkout & maintain Unix line endings - # on all OS's to match git behavior. - with open(os.path.join(git_worktree_path, 'gitdir'), 'w', newline='\n') as fp: - print(os.path.relpath(dotgit, git_worktree_path), file=fp) - - self._InitMRef() - - def _InitWorkTree(self, force_sync=False, submodules=False): - """Setup the worktree .git path. - - This is the user-visible path like src/foo/.git/. - - With non-git-worktrees, this will be a symlink to the .repo/projects/ path. - With git-worktrees, this will be a .git file using "gitdir: ..." syntax. - - Older checkouts had .git/ directories. If we see that, migrate it. - - This also handles changes in the manifest. Maybe this project was backed - by "foo/bar" on the server, but now it's "new/foo/bar". We have to update - the path we point to under .repo/projects/ to match. - """ - dotgit = os.path.join(self.worktree, '.git') - - # If using an old layout style (a directory), migrate it. - if not platform_utils.islink(dotgit) and platform_utils.isdir(dotgit): - self._MigrateOldWorkTreeGitDir(dotgit) - - init_dotgit = not os.path.exists(dotgit) - if self.use_git_worktrees: - if init_dotgit: - self._InitGitWorktree() - self._CopyAndLinkFiles() - else: - if not init_dotgit: - # See if the project has changed. - if platform_utils.realpath(self.gitdir) != platform_utils.realpath(dotgit): - platform_utils.remove(dotgit) - - if init_dotgit or not os.path.exists(dotgit): - os.makedirs(self.worktree, exist_ok=True) - platform_utils.symlink(os.path.relpath(self.gitdir, self.worktree), dotgit) - - if init_dotgit: - _lwrite(os.path.join(dotgit, HEAD), '%s\n' % self.GetRevisionId()) - - # Finish checking out the worktree. - cmd = ['read-tree', '--reset', '-u', '-v', HEAD] - if GitCommand(self, cmd).Wait() != 0: - raise GitError('Cannot initialize work tree for ' + self.name) + head = all_refs[head] + except KeyError: + head = None + + if head == revid: + # Same revision; just update HEAD to point to the new + # target branch, but otherwise take no other action. + _lwrite( + self.work_git.GetDotgitPath(subpath=HEAD), + "ref: %s%s\n" % (R_HEADS, name), + ) + return True + + return ( + GitCommand( + self, + ["checkout", name, "--"], + capture_stdout=True, + capture_stderr=True, + ).Wait() + == 0 + ) - if submodules: - self._SyncSubmodules(quiet=True) - self._CopyAndLinkFiles() + def AbandonBranch(self, name): + """Destroy a local topic branch. - @classmethod - def _MigrateOldWorkTreeGitDir(cls, dotgit): - """Migrate the old worktree .git/ dir style to a symlink. + Args: + name: The name of the branch to abandon. - This logic specifically only uses state from |dotgit| to figure out where to - move content and not |self|. This way if the backing project also changed - places, we only do the .git/ dir to .git symlink migration here. The path - updates will happen independently. - """ - # Figure out where in .repo/projects/ it's pointing to. - if not os.path.islink(os.path.join(dotgit, 'refs')): - raise GitError(f'{dotgit}: unsupported checkout state') - gitdir = os.path.dirname(os.path.realpath(os.path.join(dotgit, 'refs'))) - - # Remove known symlink paths that exist in .repo/projects/. - KNOWN_LINKS = { - 'config', 'description', 'hooks', 'info', 'logs', 'objects', - 'packed-refs', 'refs', 'rr-cache', 'shallow', 'svn', - } - # Paths that we know will be in both, but are safe to clobber in .repo/projects/. - SAFE_TO_CLOBBER = { - 'COMMIT_EDITMSG', 'FETCH_HEAD', 'HEAD', 'gc.log', 'gitk.cache', 'index', - 'ORIG_HEAD', - } - - # First see if we'd succeed before starting the migration. - unknown_paths = [] - for name in platform_utils.listdir(dotgit): - # Ignore all temporary/backup names. These are common with vim & emacs. - if name.endswith('~') or (name[0] == '#' and name[-1] == '#'): - continue - - dotgit_path = os.path.join(dotgit, name) - if name in KNOWN_LINKS: - if not platform_utils.islink(dotgit_path): - unknown_paths.append(f'{dotgit_path}: should be a symlink') - else: - gitdir_path = os.path.join(gitdir, name) - if name not in SAFE_TO_CLOBBER and os.path.exists(gitdir_path): - unknown_paths.append(f'{dotgit_path}: unknown file; please file a bug') - if unknown_paths: - raise GitError('Aborting migration: ' + '\n'.join(unknown_paths)) - - # Now walk the paths and sync the .git/ to .repo/projects/. - for name in platform_utils.listdir(dotgit): - dotgit_path = os.path.join(dotgit, name) - - # Ignore all temporary/backup names. These are common with vim & emacs. - if name.endswith('~') or (name[0] == '#' and name[-1] == '#'): - platform_utils.remove(dotgit_path) - elif name in KNOWN_LINKS: - platform_utils.remove(dotgit_path) - else: - gitdir_path = os.path.join(gitdir, name) - platform_utils.remove(gitdir_path, missing_ok=True) - platform_utils.rename(dotgit_path, gitdir_path) - - # Now that the dir should be empty, clear it out, and symlink it over. - platform_utils.rmdir(dotgit) - platform_utils.symlink(os.path.relpath(gitdir, os.path.dirname(dotgit)), dotgit) - - def _get_symlink_error_message(self): - if platform_utils.isWindows(): - return ('Unable to create symbolic link. Please re-run the command as ' - 'Administrator, or see ' - 'https://github.com/git-for-windows/git/wiki/Symbolic-Links ' - 'for other options.') - return 'filesystem must support symlinks' - - def _revlist(self, *args, **kw): - a = [] - a.extend(args) - a.append('--') - return self.work_git.rev_list(*a, **kw) - - @property - def _allrefs(self): - return self.bare_ref.all - - def _getLogs(self, rev1, rev2, oneline=False, color=True, pretty_format=None): - """Get logs between two revisions of this project.""" - comp = '..' - if rev1: - revs = [rev1] - if rev2: - revs.extend([comp, rev2]) - cmd = ['log', ''.join(revs)] - out = DiffColoring(self.config) - if out.is_on and color: - cmd.append('--color') - if pretty_format is not None: - cmd.append('--pretty=format:%s' % pretty_format) - if oneline: - cmd.append('--oneline') - - try: - log = GitCommand(self, cmd, capture_stdout=True, capture_stderr=True) - if log.Wait() == 0: - return log.stdout - except GitError: - # worktree may not exist if groups changed for example. In that case, - # try in gitdir instead. - if not os.path.exists(self.worktree): - return self.bare_git.log(*cmd[1:]) - else: - raise - return None - - def getAddedAndRemovedLogs(self, toProject, oneline=False, color=True, - pretty_format=None): - """Get the list of logs from this revision to given revisionId""" - logs = {} - selfId = self.GetRevisionId(self._allrefs) - toId = toProject.GetRevisionId(toProject._allrefs) - - logs['added'] = self._getLogs(selfId, toId, oneline=oneline, color=color, - pretty_format=pretty_format) - logs['removed'] = self._getLogs(toId, selfId, oneline=oneline, color=color, - pretty_format=pretty_format) - return logs - - class _GitGetByExec(object): - - def __init__(self, project, bare, gitdir): - self._project = project - self._bare = bare - self._gitdir = gitdir - - # __getstate__ and __setstate__ are required for pickling because __getattr__ exists. - def __getstate__(self): - return (self._project, self._bare, self._gitdir) - - def __setstate__(self, state): - self._project, self._bare, self._gitdir = state - - def LsOthers(self): - p = GitCommand(self._project, - ['ls-files', - '-z', - '--others', - '--exclude-standard'], - bare=False, - gitdir=self._gitdir, - capture_stdout=True, - capture_stderr=True) - if p.Wait() == 0: - out = p.stdout - if out: - # Backslash is not anomalous - return out[:-1].split('\0') - return [] - - def DiffZ(self, name, *args): - cmd = [name] - cmd.append('-z') - cmd.append('--ignore-submodules') - cmd.extend(args) - p = GitCommand(self._project, - cmd, - gitdir=self._gitdir, - bare=False, - capture_stdout=True, - capture_stderr=True) - p.Wait() - r = {} - out = p.stdout - if out: - out = iter(out[:-1].split('\0')) - while out: - try: - info = next(out) - path = next(out) - except StopIteration: - break - - class _Info(object): - - def __init__(self, path, omode, nmode, oid, nid, state): - self.path = path - self.src_path = None - self.old_mode = omode - self.new_mode = nmode - self.old_id = oid - self.new_id = nid - - if len(state) == 1: - self.status = state - self.level = None - else: - self.status = state[:1] - self.level = state[1:] - while self.level.startswith('0'): - self.level = self.level[1:] - - info = info[1:].split(' ') - info = _Info(path, *info) - if info.status in ('R', 'C'): - info.src_path = info.path - info.path = next(out) - r[info.path] = info - return r - - def GetDotgitPath(self, subpath=None): - """Return the full path to the .git dir. - - As a convenience, append |subpath| if provided. - """ - if self._bare: - dotgit = self._gitdir - else: - dotgit = os.path.join(self._project.worktree, '.git') - if os.path.isfile(dotgit): - # Git worktrees use a "gitdir:" syntax to point to the scratch space. - with open(dotgit) as fp: - setting = fp.read() - assert setting.startswith('gitdir:') - gitdir = setting.split(':', 1)[1].strip() - dotgit = os.path.normpath(os.path.join(self._project.worktree, gitdir)) - - return dotgit if subpath is None else os.path.join(dotgit, subpath) - - def GetHead(self): - """Return the ref that HEAD points to.""" - path = self.GetDotgitPath(subpath=HEAD) - try: - with open(path) as fd: - line = fd.readline() - except IOError as e: - raise NoManifestException(path, str(e)) - try: - line = line.decode() - except AttributeError: - pass - if line.startswith('ref: '): - return line[5:-1] - return line[:-1] - - def SetHead(self, ref, message=None): - cmdv = [] - if message is not None: - cmdv.extend(['-m', message]) - cmdv.append(HEAD) - cmdv.append(ref) - self.symbolic_ref(*cmdv) - - def DetachHead(self, new, message=None): - cmdv = ['--no-deref'] - if message is not None: - cmdv.extend(['-m', message]) - cmdv.append(HEAD) - cmdv.append(new) - self.update_ref(*cmdv) - - def UpdateRef(self, name, new, old=None, - message=None, - detach=False): - cmdv = [] - if message is not None: - cmdv.extend(['-m', message]) - if detach: - cmdv.append('--no-deref') - cmdv.append(name) - cmdv.append(new) - if old is not None: - cmdv.append(old) - self.update_ref(*cmdv) - - def DeleteRef(self, name, old=None): - if not old: - old = self.rev_parse(name) - self.update_ref('-d', name, old) - self._project.bare_ref.deleted(name) - - def rev_list(self, *args, **kw): - if 'format' in kw: - cmdv = ['log', '--pretty=format:%s' % kw['format']] - else: - cmdv = ['rev-list'] - cmdv.extend(args) - p = GitCommand(self._project, - cmdv, - bare=self._bare, - gitdir=self._gitdir, - capture_stdout=True, - capture_stderr=True) - if p.Wait() != 0: - raise GitError('%s rev-list %s: %s' % - (self._project.name, str(args), p.stderr)) - return p.stdout.splitlines() - - def __getattr__(self, name): - """Allow arbitrary git commands using pythonic syntax. - - This allows you to do things like: - git_obj.rev_parse('HEAD') - - Since we don't have a 'rev_parse' method defined, the __getattr__ will - run. We'll replace the '_' with a '-' and try to run a git command. - Any other positional arguments will be passed to the git command, and the - following keyword arguments are supported: - config: An optional dict of git config options to be passed with '-c'. - - Args: - name: The name of the git command to call. Any '_' characters will - be replaced with '-'. - - Returns: - A callable object that will try to call git with the named command. - """ - name = name.replace('_', '-') - - def runner(*args, **kwargs): - cmdv = [] - config = kwargs.pop('config', None) - for k in kwargs: - raise TypeError('%s() got an unexpected keyword argument %r' - % (name, k)) - if config is not None: - for k, v in config.items(): - cmdv.append('-c') - cmdv.append('%s=%s' % (k, v)) - cmdv.append(name) - cmdv.extend(args) - p = GitCommand(self._project, - cmdv, - bare=self._bare, - gitdir=self._gitdir, - capture_stdout=True, - capture_stderr=True) - if p.Wait() != 0: - raise GitError('%s %s: %s' % - (self._project.name, name, p.stderr)) - r = p.stdout - if r.endswith('\n') and r.index('\n') == len(r) - 1: - return r[:-1] - return r - return runner + Returns: + True if the abandon succeeded; False if it didn't; None if the + branch didn't exist. + """ + rev = R_HEADS + name + all_refs = self.bare_ref.all + if rev not in all_refs: + # Doesn't exist + return None + + head = self.work_git.GetHead() + if head == rev: + # We can't destroy the branch while we are sitting + # on it. Switch to a detached HEAD. + head = all_refs[head] + + revid = self.GetRevisionId(all_refs) + if head == revid: + _lwrite( + self.work_git.GetDotgitPath(subpath=HEAD), "%s\n" % revid + ) + else: + self._Checkout(revid, quiet=True) + + return ( + GitCommand( + self, + ["branch", "-D", name], + capture_stdout=True, + capture_stderr=True, + ).Wait() + == 0 + ) + def PruneHeads(self): + """Prune any topic branches already merged into upstream.""" + cb = self.CurrentBranch + kill = [] + left = self._allrefs + for name in left.keys(): + if name.startswith(R_HEADS): + name = name[len(R_HEADS) :] + if cb is None or name != cb: + kill.append(name) + + # Minor optimization: If there's nothing to prune, then don't try to + # read any project state. + if not kill and not cb: + return [] + + rev = self.GetRevisionId(left) + if ( + cb is not None + and not self._revlist(HEAD + "..." + rev) + and not self.IsDirty(consider_untracked=False) + ): + self.work_git.DetachHead(HEAD) + kill.append(cb) + + if kill: + old = self.bare_git.GetHead() -class _PriorSyncFailedError(Exception): + try: + self.bare_git.DetachHead(rev) + + b = ["branch", "-d"] + b.extend(kill) + b = GitCommand( + self, b, bare=True, capture_stdout=True, capture_stderr=True + ) + b.Wait() + finally: + if ID_RE.match(old): + self.bare_git.DetachHead(old) + else: + self.bare_git.SetHead(old) + left = self._allrefs + + for branch in kill: + if (R_HEADS + branch) not in left: + self.CleanPublishedCache() + break + + if cb and cb not in kill: + kill.append(cb) + kill.sort() + + kept = [] + for branch in kill: + if R_HEADS + branch in left: + branch = self.GetBranch(branch) + base = branch.LocalMerge + if not base: + base = rev + kept.append(ReviewableBranch(self, branch, base)) + return kept + + def GetRegisteredSubprojects(self): + result = [] + + def rec(subprojects): + if not subprojects: + return + result.extend(subprojects) + for p in subprojects: + rec(p.subprojects) + + rec(self.subprojects) + return result + + def _GetSubmodules(self): + # Unfortunately we cannot call `git submodule status --recursive` here + # because the working tree might not exist yet, and it cannot be used + # without a working tree in its current implementation. + + def get_submodules(gitdir, rev): + # Parse .gitmodules for submodule sub_paths and sub_urls. + sub_paths, sub_urls = parse_gitmodules(gitdir, rev) + if not sub_paths: + return [] + # Run `git ls-tree` to read SHAs of submodule object, which happen + # to be revision of submodule repository. + sub_revs = git_ls_tree(gitdir, rev, sub_paths) + submodules = [] + for sub_path, sub_url in zip(sub_paths, sub_urls): + try: + sub_rev = sub_revs[sub_path] + except KeyError: + # Ignore non-exist submodules. + continue + submodules.append((sub_rev, sub_path, sub_url)) + return submodules + + re_path = re.compile(r"^submodule\.(.+)\.path=(.*)$") + re_url = re.compile(r"^submodule\.(.+)\.url=(.*)$") + + def parse_gitmodules(gitdir, rev): + cmd = ["cat-file", "blob", "%s:.gitmodules" % rev] + try: + p = GitCommand( + None, + cmd, + capture_stdout=True, + capture_stderr=True, + bare=True, + gitdir=gitdir, + ) + except GitError: + return [], [] + if p.Wait() != 0: + return [], [] + + gitmodules_lines = [] + fd, temp_gitmodules_path = tempfile.mkstemp() + try: + os.write(fd, p.stdout.encode("utf-8")) + os.close(fd) + cmd = ["config", "--file", temp_gitmodules_path, "--list"] + p = GitCommand( + None, + cmd, + capture_stdout=True, + capture_stderr=True, + bare=True, + gitdir=gitdir, + ) + if p.Wait() != 0: + return [], [] + gitmodules_lines = p.stdout.split("\n") + except GitError: + return [], [] + finally: + platform_utils.remove(temp_gitmodules_path) + + names = set() + paths = {} + urls = {} + for line in gitmodules_lines: + if not line: + continue + m = re_path.match(line) + if m: + names.add(m.group(1)) + paths[m.group(1)] = m.group(2) + continue + m = re_url.match(line) + if m: + names.add(m.group(1)) + urls[m.group(1)] = m.group(2) + continue + names = sorted(names) + return ( + [paths.get(name, "") for name in names], + [urls.get(name, "") for name in names], + ) + + def git_ls_tree(gitdir, rev, paths): + cmd = ["ls-tree", rev, "--"] + cmd.extend(paths) + try: + p = GitCommand( + None, + cmd, + capture_stdout=True, + capture_stderr=True, + bare=True, + gitdir=gitdir, + ) + except GitError: + return [] + if p.Wait() != 0: + return [] + objects = {} + for line in p.stdout.split("\n"): + if not line.strip(): + continue + object_rev, object_path = line.split()[2:4] + objects[object_path] = object_rev + return objects - def __str__(self): - return 'prior sync failed; rebase still in progress' + try: + rev = self.GetRevisionId() + except GitError: + return [] + return get_submodules(self.gitdir, rev) + + def GetDerivedSubprojects(self): + result = [] + if not self.Exists: + # If git repo does not exist yet, querying its submodules will + # mess up its states; so return here. + return result + for rev, path, url in self._GetSubmodules(): + name = self.manifest.GetSubprojectName(self, path) + ( + relpath, + worktree, + gitdir, + objdir, + ) = self.manifest.GetSubprojectPaths(self, name, path) + project = self.manifest.paths.get(relpath) + if project: + result.extend(project.GetDerivedSubprojects()) + continue + + if url.startswith(".."): + url = urllib.parse.urljoin("%s/" % self.remote.url, url) + remote = RemoteSpec( + self.remote.name, + url=url, + pushUrl=self.remote.pushUrl, + review=self.remote.review, + revision=self.remote.revision, + ) + subproject = Project( + manifest=self.manifest, + name=name, + remote=remote, + gitdir=gitdir, + objdir=objdir, + worktree=worktree, + relpath=relpath, + revisionExpr=rev, + revisionId=rev, + rebase=self.rebase, + groups=self.groups, + sync_c=self.sync_c, + sync_s=self.sync_s, + sync_tags=self.sync_tags, + parent=self, + is_derived=True, + ) + result.append(subproject) + result.extend(subproject.GetDerivedSubprojects()) + return result + + def EnableRepositoryExtension(self, key, value="true", version=1): + """Enable git repository extension |key| with |value|. + Args: + key: The extension to enabled. Omit the "extensions." prefix. + value: The value to use for the extension. + version: The minimum git repository version needed. + """ + # Make sure the git repo version is new enough already. + found_version = self.config.GetInt("core.repositoryFormatVersion") + if found_version is None: + found_version = 0 + if found_version < version: + self.config.SetString("core.repositoryFormatVersion", str(version)) + + # Enable the extension! + self.config.SetString("extensions.%s" % (key,), value) + + def ResolveRemoteHead(self, name=None): + """Find out what the default branch (HEAD) points to. + + Normally this points to refs/heads/master, but projects are moving to + main. Support whatever the server uses rather than hardcoding "master" + ourselves. + """ + if name is None: + name = self.remote.name + + # The output will look like (NB: tabs are separators): + # ref: refs/heads/master HEAD + # 5f6803b100bb3cd0f534e96e88c91373e8ed1c44 HEAD + output = self.bare_git.ls_remote( + "-q", "--symref", "--exit-code", name, "HEAD" + ) -class _DirtyError(Exception): + for line in output.splitlines(): + lhs, rhs = line.split("\t", 1) + if rhs == "HEAD" and lhs.startswith("ref:"): + return lhs[4:].strip() - def __str__(self): - return 'contains uncommitted changes' + return None + def _CheckForImmutableRevision(self): + try: + # if revision (sha or tag) is not present then following function + # throws an error. + self.bare_git.rev_list( + "-1", "--missing=allow-any", "%s^0" % self.revisionExpr, "--" + ) + if self.upstream: + rev = self.GetRemote().ToLocal(self.upstream) + self.bare_git.rev_list( + "-1", "--missing=allow-any", "%s^0" % rev, "--" + ) + self.bare_git.merge_base( + "--is-ancestor", self.revisionExpr, rev + ) + return True + except GitError: + # There is no such persistent revision. We have to fetch it. + return False -class _InfoMessage(object): + def _FetchArchive(self, tarpath, cwd=None): + cmd = ["archive", "-v", "-o", tarpath] + cmd.append("--remote=%s" % self.remote.url) + cmd.append("--prefix=%s/" % self.RelPath(local=False)) + cmd.append(self.revisionExpr) - def __init__(self, project, text): - self.project = project - self.text = text + command = GitCommand( + self, cmd, cwd=cwd, capture_stdout=True, capture_stderr=True + ) - def Print(self, syncbuf): - syncbuf.out.info('%s/: %s', self.project.RelPath(local=False), self.text) - syncbuf.out.nl() + if command.Wait() != 0: + raise GitError("git archive %s: %s" % (self.name, command.stderr)) + + def _RemoteFetch( + self, + name=None, + current_branch_only=False, + initial=False, + quiet=False, + verbose=False, + output_redir=None, + alt_dir=None, + tags=True, + prune=False, + depth=None, + submodules=False, + ssh_proxy=None, + force_sync=False, + clone_filter=None, + retry_fetches=2, + retry_sleep_initial_sec=4.0, + retry_exp_factor=2.0, + ): + is_sha1 = False + tag_name = None + # The depth should not be used when fetching to a mirror because + # it will result in a shallow repository that cannot be cloned or + # fetched from. + # The repo project should also never be synced with partial depth. + if self.manifest.IsMirror or self.relpath == ".repo/repo": + depth = None + + if depth: + current_branch_only = True + + if ID_RE.match(self.revisionExpr) is not None: + is_sha1 = True + + if current_branch_only: + if self.revisionExpr.startswith(R_TAGS): + # This is a tag and its commit id should never change. + tag_name = self.revisionExpr[len(R_TAGS) :] + elif self.upstream and self.upstream.startswith(R_TAGS): + # This is a tag and its commit id should never change. + tag_name = self.upstream[len(R_TAGS) :] + + if is_sha1 or tag_name is not None: + if self._CheckForImmutableRevision(): + if verbose: + print( + "Skipped fetching project %s (already have " + "persistent ref)" % self.name + ) + return True + if is_sha1 and not depth: + # When syncing a specific commit and --depth is not set: + # * if upstream is explicitly specified and is not a sha1, fetch + # only upstream as users expect only upstream to be fetch. + # Note: The commit might not be in upstream in which case the + # sync will fail. + # * otherwise, fetch all branches to make sure we end up with + # the specific commit. + if self.upstream: + current_branch_only = not ID_RE.match(self.upstream) + else: + current_branch_only = False + + if not name: + name = self.remote.name + + remote = self.GetRemote(name) + if not remote.PreConnectFetch(ssh_proxy): + ssh_proxy = None + + if initial: + if alt_dir and "objects" == os.path.basename(alt_dir): + ref_dir = os.path.dirname(alt_dir) + packed_refs = os.path.join(self.gitdir, "packed-refs") + + all_refs = self.bare_ref.all + ids = set(all_refs.values()) + tmp = set() + + for r, ref_id in GitRefs(ref_dir).all.items(): + if r not in all_refs: + if r.startswith(R_TAGS) or remote.WritesTo(r): + all_refs[r] = ref_id + ids.add(ref_id) + continue + + if ref_id in ids: + continue + + r = "refs/_alt/%s" % ref_id + all_refs[r] = ref_id + ids.add(ref_id) + tmp.add(r) + + tmp_packed_lines = [] + old_packed_lines = [] + + for r in sorted(all_refs): + line = "%s %s\n" % (all_refs[r], r) + tmp_packed_lines.append(line) + if r not in tmp: + old_packed_lines.append(line) + + tmp_packed = "".join(tmp_packed_lines) + old_packed = "".join(old_packed_lines) + _lwrite(packed_refs, tmp_packed) + else: + alt_dir = None + + cmd = ["fetch"] + + if clone_filter: + git_require((2, 19, 0), fail=True, msg="partial clones") + cmd.append("--filter=%s" % clone_filter) + self.EnableRepositoryExtension("partialclone", self.remote.name) + + if depth: + cmd.append("--depth=%s" % depth) + else: + # If this repo has shallow objects, then we don't know which refs + # have shallow objects or not. Tell git to unshallow all fetched + # refs. Don't do this with projects that don't have shallow + # objects, since it is less efficient. + if os.path.exists(os.path.join(self.gitdir, "shallow")): + cmd.append("--depth=2147483647") + + if not verbose: + cmd.append("--quiet") + if not quiet and sys.stdout.isatty(): + cmd.append("--progress") + if not self.worktree: + cmd.append("--update-head-ok") + cmd.append(name) + + if force_sync: + cmd.append("--force") + + if prune: + cmd.append("--prune") + + # Always pass something for --recurse-submodules, git with GIT_DIR + # behaves incorrectly when not given `--recurse-submodules=no`. + # (b/218891912) + cmd.append( + f'--recurse-submodules={"on-demand" if submodules else "no"}' + ) + spec = [] + if not current_branch_only: + # Fetch whole repo. + spec.append( + str(("+refs/heads/*:") + remote.ToLocal("refs/heads/*")) + ) + elif tag_name is not None: + spec.append("tag") + spec.append(tag_name) + + if self.manifest.IsMirror and not current_branch_only: + branch = None + else: + branch = self.revisionExpr + if ( + not self.manifest.IsMirror + and is_sha1 + and depth + and git_require((1, 8, 3)) + ): + # Shallow checkout of a specific commit, fetch from that commit and + # not the heads only as the commit might be deeper in the history. + spec.append(branch) + if self.upstream: + spec.append(self.upstream) + else: + if is_sha1: + branch = self.upstream + if branch is not None and branch.strip(): + if not branch.startswith("refs/"): + branch = R_HEADS + branch + spec.append(str(("+%s:" % branch) + remote.ToLocal(branch))) + + # If mirroring repo and we cannot deduce the tag or branch to fetch, + # fetch whole repo. + if self.manifest.IsMirror and not spec: + spec.append( + str(("+refs/heads/*:") + remote.ToLocal("refs/heads/*")) + ) + + # If using depth then we should not get all the tags since they may + # be outside of the depth. + if not tags or depth: + cmd.append("--no-tags") + else: + cmd.append("--tags") + spec.append(str(("+refs/tags/*:") + remote.ToLocal("refs/tags/*"))) + + cmd.extend(spec) + + # At least one retry minimum due to git remote prune. + retry_fetches = max(retry_fetches, 2) + retry_cur_sleep = retry_sleep_initial_sec + ok = prune_tried = False + for try_n in range(retry_fetches): + gitcmd = GitCommand( + self, + cmd, + bare=True, + objdir=os.path.join(self.objdir, "objects"), + ssh_proxy=ssh_proxy, + merge_output=True, + capture_stdout=quiet or bool(output_redir), + ) + if gitcmd.stdout and not quiet and output_redir: + output_redir.write(gitcmd.stdout) + ret = gitcmd.Wait() + if ret == 0: + ok = True + break + + # Retry later due to HTTP 429 Too Many Requests. + elif ( + gitcmd.stdout + and "error:" in gitcmd.stdout + and "HTTP 429" in gitcmd.stdout + ): + # Fallthru to sleep+retry logic at the bottom. + pass + + # Try to prune remote branches once in case there are conflicts. + # For example, if the remote had refs/heads/upstream, but deleted + # that and now has refs/heads/upstream/foo. + elif ( + gitcmd.stdout + and "error:" in gitcmd.stdout + and "git remote prune" in gitcmd.stdout + and not prune_tried + ): + prune_tried = True + prunecmd = GitCommand( + self, + ["remote", "prune", name], + bare=True, + ssh_proxy=ssh_proxy, + ) + ret = prunecmd.Wait() + if ret: + break + print( + "retrying fetch after pruning remote branches", + file=output_redir, + ) + # Continue right away so we don't sleep as we shouldn't need to. + continue + elif current_branch_only and is_sha1 and ret == 128: + # Exit code 128 means "couldn't find the ref you asked for"; if + # we're in sha1 mode, we just tried sync'ing from the upstream + # field; it doesn't exist, thus abort the optimization attempt + # and do a full sync. + break + elif ret < 0: + # Git died with a signal, exit immediately. + break + + # Figure out how long to sleep before the next attempt, if there is + # one. + if not verbose and gitcmd.stdout: + print( + "\n%s:\n%s" % (self.name, gitcmd.stdout), + end="", + file=output_redir, + ) + if try_n < retry_fetches - 1: + print( + "%s: sleeping %s seconds before retrying" + % (self.name, retry_cur_sleep), + file=output_redir, + ) + time.sleep(retry_cur_sleep) + retry_cur_sleep = min( + retry_exp_factor * retry_cur_sleep, MAXIMUM_RETRY_SLEEP_SEC + ) + retry_cur_sleep *= 1 - random.uniform( + -RETRY_JITTER_PERCENT, RETRY_JITTER_PERCENT + ) + + if initial: + if alt_dir: + if old_packed != "": + _lwrite(packed_refs, old_packed) + else: + platform_utils.remove(packed_refs) + self.bare_git.pack_refs("--all", "--prune") + + if is_sha1 and current_branch_only: + # We just synced the upstream given branch; verify we + # got what we wanted, else trigger a second run of all + # refs. + if not self._CheckForImmutableRevision(): + # Sync the current branch only with depth set to None. + # We always pass depth=None down to avoid infinite recursion. + return self._RemoteFetch( + name=name, + quiet=quiet, + verbose=verbose, + output_redir=output_redir, + current_branch_only=current_branch_only and depth, + initial=False, + alt_dir=alt_dir, + depth=None, + ssh_proxy=ssh_proxy, + clone_filter=clone_filter, + ) + + return ok + + def _ApplyCloneBundle(self, initial=False, quiet=False, verbose=False): + if initial and ( + self.manifest.manifestProject.depth or self.clone_depth + ): + return False -class _Failure(object): + remote = self.GetRemote() + bundle_url = remote.url + "/clone.bundle" + bundle_url = GitConfig.ForUser().UrlInsteadOf(bundle_url) + if GetSchemeFromUrl(bundle_url) not in ( + "http", + "https", + "persistent-http", + "persistent-https", + ): + return False - def __init__(self, project, why): - self.project = project - self.why = why + bundle_dst = os.path.join(self.gitdir, "clone.bundle") + bundle_tmp = os.path.join(self.gitdir, "clone.bundle.tmp") - def Print(self, syncbuf): - syncbuf.out.fail('error: %s/: %s', - self.project.RelPath(local=False), - str(self.why)) - syncbuf.out.nl() + exist_dst = os.path.exists(bundle_dst) + exist_tmp = os.path.exists(bundle_tmp) + if not initial and not exist_dst and not exist_tmp: + return False -class _Later(object): + if not exist_dst: + exist_dst = self._FetchBundle( + bundle_url, bundle_tmp, bundle_dst, quiet, verbose + ) + if not exist_dst: + return False - def __init__(self, project, action): - self.project = project - self.action = action + cmd = ["fetch"] + if not verbose: + cmd.append("--quiet") + if not quiet and sys.stdout.isatty(): + cmd.append("--progress") + if not self.worktree: + cmd.append("--update-head-ok") + cmd.append(bundle_dst) + for f in remote.fetch: + cmd.append(str(f)) + cmd.append("+refs/tags/*:refs/tags/*") + + ok = ( + GitCommand( + self, + cmd, + bare=True, + objdir=os.path.join(self.objdir, "objects"), + ).Wait() + == 0 + ) + platform_utils.remove(bundle_dst, missing_ok=True) + platform_utils.remove(bundle_tmp, missing_ok=True) + return ok + + def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet, verbose): + platform_utils.remove(dstPath, missing_ok=True) + + cmd = ["curl", "--fail", "--output", tmpPath, "--netrc", "--location"] + if quiet: + cmd += ["--silent", "--show-error"] + if os.path.exists(tmpPath): + size = os.stat(tmpPath).st_size + if size >= 1024: + cmd += ["--continue-at", "%d" % (size,)] + else: + platform_utils.remove(tmpPath) + with GetUrlCookieFile(srcUrl, quiet) as (cookiefile, proxy): + if cookiefile: + cmd += ["--cookie", cookiefile] + if proxy: + cmd += ["--proxy", proxy] + elif "http_proxy" in os.environ and "darwin" == sys.platform: + cmd += ["--proxy", os.environ["http_proxy"]] + if srcUrl.startswith("persistent-https"): + srcUrl = "http" + srcUrl[len("persistent-https") :] + elif srcUrl.startswith("persistent-http"): + srcUrl = "http" + srcUrl[len("persistent-http") :] + cmd += [srcUrl] + + proc = None + with Trace("Fetching bundle: %s", " ".join(cmd)): + if verbose: + print("%s: Downloading bundle: %s" % (self.name, srcUrl)) + stdout = None if verbose else subprocess.PIPE + stderr = None if verbose else subprocess.STDOUT + try: + proc = subprocess.Popen(cmd, stdout=stdout, stderr=stderr) + except OSError: + return False + + (output, _) = proc.communicate() + curlret = proc.returncode + + if curlret == 22: + # From curl man page: + # 22: HTTP page not retrieved. The requested url was not found + # or returned another error with the HTTP error code being 400 + # or above. This return code only appears if -f, --fail is used. + if verbose: + print( + "%s: Unable to retrieve clone.bundle; ignoring." + % self.name + ) + if output: + print("Curl output:\n%s" % output) + return False + elif curlret and not verbose and output: + print("%s" % output, file=sys.stderr) + + if os.path.exists(tmpPath): + if curlret == 0 and self._IsValidBundle(tmpPath, quiet): + platform_utils.rename(tmpPath, dstPath) + return True + else: + platform_utils.remove(tmpPath) + return False + else: + return False - def Run(self, syncbuf): - out = syncbuf.out - out.project('project %s/', self.project.RelPath(local=False)) - out.nl() - try: - self.action() - out.nl() - return True - except GitError: - out.nl() - return False + def _IsValidBundle(self, path, quiet): + try: + with open(path, "rb") as f: + if f.read(16) == b"# v2 git bundle\n": + return True + else: + if not quiet: + print( + "Invalid clone.bundle file; ignoring.", + file=sys.stderr, + ) + return False + except OSError: + return False + def _Checkout(self, rev, quiet=False): + cmd = ["checkout"] + if quiet: + cmd.append("-q") + cmd.append(rev) + cmd.append("--") + if GitCommand(self, cmd).Wait() != 0: + if self._allrefs: + raise GitError("%s checkout %s " % (self.name, rev)) + + def _CherryPick(self, rev, ffonly=False, record_origin=False): + cmd = ["cherry-pick"] + if ffonly: + cmd.append("--ff") + if record_origin: + cmd.append("-x") + cmd.append(rev) + cmd.append("--") + if GitCommand(self, cmd).Wait() != 0: + if self._allrefs: + raise GitError("%s cherry-pick %s " % (self.name, rev)) + + def _LsRemote(self, refs): + cmd = ["ls-remote", self.remote.name, refs] + p = GitCommand(self, cmd, capture_stdout=True) + if p.Wait() == 0: + return p.stdout + return None -class _SyncColoring(Coloring): + def _Revert(self, rev): + cmd = ["revert"] + cmd.append("--no-edit") + cmd.append(rev) + cmd.append("--") + if GitCommand(self, cmd).Wait() != 0: + if self._allrefs: + raise GitError("%s revert %s " % (self.name, rev)) + + def _ResetHard(self, rev, quiet=True): + cmd = ["reset", "--hard"] + if quiet: + cmd.append("-q") + cmd.append(rev) + if GitCommand(self, cmd).Wait() != 0: + raise GitError("%s reset --hard %s " % (self.name, rev)) - def __init__(self, config): - super().__init__(config, 'reposync') - self.project = self.printer('header', attr='bold') - self.info = self.printer('info') - self.fail = self.printer('fail', fg='red') + def _SyncSubmodules(self, quiet=True): + cmd = ["submodule", "update", "--init", "--recursive"] + if quiet: + cmd.append("-q") + if GitCommand(self, cmd).Wait() != 0: + raise GitError( + "%s submodule update --init --recursive " % self.name + ) + + def _Rebase(self, upstream, onto=None): + cmd = ["rebase"] + if onto is not None: + cmd.extend(["--onto", onto]) + cmd.append(upstream) + if GitCommand(self, cmd).Wait() != 0: + raise GitError("%s rebase %s " % (self.name, upstream)) + def _FastForward(self, head, ffonly=False): + cmd = ["merge", "--no-stat", head] + if ffonly: + cmd.append("--ff-only") + if GitCommand(self, cmd).Wait() != 0: + raise GitError("%s merge %s " % (self.name, head)) -class SyncBuffer(object): + def _InitGitDir(self, mirror_git=None, force_sync=False, quiet=False): + init_git_dir = not os.path.exists(self.gitdir) + init_obj_dir = not os.path.exists(self.objdir) + try: + # Initialize the bare repository, which contains all of the objects. + if init_obj_dir: + os.makedirs(self.objdir) + self.bare_objdir.init() + + self._UpdateHooks(quiet=quiet) + + if self.use_git_worktrees: + # Enable per-worktree config file support if possible. This + # is more a nice-to-have feature for users rather than a + # hard requirement. + if git_require((2, 20, 0)): + self.EnableRepositoryExtension("worktreeConfig") + + # If we have a separate directory to hold refs, initialize it as + # well. + if self.objdir != self.gitdir: + if init_git_dir: + os.makedirs(self.gitdir) + + if init_obj_dir or init_git_dir: + self._ReferenceGitDir( + self.objdir, self.gitdir, copy_all=True + ) + try: + self._CheckDirReference(self.objdir, self.gitdir) + except GitError as e: + if force_sync: + print( + "Retrying clone after deleting %s" % self.gitdir, + file=sys.stderr, + ) + try: + platform_utils.rmtree( + platform_utils.realpath(self.gitdir) + ) + if self.worktree and os.path.exists( + platform_utils.realpath(self.worktree) + ): + platform_utils.rmtree( + platform_utils.realpath(self.worktree) + ) + return self._InitGitDir( + mirror_git=mirror_git, + force_sync=False, + quiet=quiet, + ) + except Exception: + raise e + raise e + + if init_git_dir: + mp = self.manifest.manifestProject + ref_dir = mp.reference or "" + + def _expanded_ref_dirs(): + """Iterate through possible git reference dir paths.""" + name = self.name + ".git" + yield mirror_git or os.path.join(ref_dir, name) + for prefix in "", self.remote.name: + yield os.path.join( + ref_dir, ".repo", "project-objects", prefix, name + ) + yield os.path.join( + ref_dir, ".repo", "worktrees", prefix, name + ) + + if ref_dir or mirror_git: + found_ref_dir = None + for path in _expanded_ref_dirs(): + if os.path.exists(path): + found_ref_dir = path + break + ref_dir = found_ref_dir + + if ref_dir: + if not os.path.isabs(ref_dir): + # The alternate directory is relative to the object + # database. + ref_dir = os.path.relpath( + ref_dir, os.path.join(self.objdir, "objects") + ) + _lwrite( + os.path.join( + self.objdir, "objects/info/alternates" + ), + os.path.join(ref_dir, "objects") + "\n", + ) + + m = self.manifest.manifestProject.config + for key in ["user.name", "user.email"]: + if m.Has(key, include_defaults=False): + self.config.SetString(key, m.GetString(key)) + if not self.manifest.EnableGitLfs: + self.config.SetString( + "filter.lfs.smudge", "git-lfs smudge --skip -- %f" + ) + self.config.SetString( + "filter.lfs.process", "git-lfs filter-process --skip" + ) + self.config.SetBoolean( + "core.bare", True if self.manifest.IsMirror else None + ) + except Exception: + if init_obj_dir and os.path.exists(self.objdir): + platform_utils.rmtree(self.objdir) + if init_git_dir and os.path.exists(self.gitdir): + platform_utils.rmtree(self.gitdir) + raise + + def _UpdateHooks(self, quiet=False): + if os.path.exists(self.objdir): + self._InitHooks(quiet=quiet) + + def _InitHooks(self, quiet=False): + hooks = platform_utils.realpath(os.path.join(self.objdir, "hooks")) + if not os.path.exists(hooks): + os.makedirs(hooks) + + # Delete sample hooks. They're noise. + for hook in glob.glob(os.path.join(hooks, "*.sample")): + try: + platform_utils.remove(hook, missing_ok=True) + except PermissionError: + pass + + for stock_hook in _ProjectHooks(): + name = os.path.basename(stock_hook) + + if ( + name in ("commit-msg",) + and not self.remote.review + and self is not self.manifest.manifestProject + ): + # Don't install a Gerrit Code Review hook if this + # project does not appear to use it for reviews. + # + # Since the manifest project is one of those, but also + # managed through gerrit, it's excluded. + continue + + dst = os.path.join(hooks, name) + if platform_utils.islink(dst): + continue + if os.path.exists(dst): + # If the files are the same, we'll leave it alone. We create + # symlinks below by default but fallback to hardlinks if the OS + # blocks them. So if we're here, it's probably because we made a + # hardlink below. + if not filecmp.cmp(stock_hook, dst, shallow=False): + if not quiet: + _warn( + "%s: Not replacing locally modified %s hook", + self.RelPath(local=False), + name, + ) + continue + try: + platform_utils.symlink( + os.path.relpath(stock_hook, os.path.dirname(dst)), dst + ) + except OSError as e: + if e.errno == errno.EPERM: + try: + os.link(stock_hook, dst) + except OSError: + raise GitError(self._get_symlink_error_message()) + else: + raise + + def _InitRemote(self): + if self.remote.url: + remote = self.GetRemote() + remote.url = self.remote.url + remote.pushUrl = self.remote.pushUrl + remote.review = self.remote.review + remote.projectname = self.name + + if self.worktree: + remote.ResetFetch(mirror=False) + else: + remote.ResetFetch(mirror=True) + remote.Save() + + def _InitMRef(self): + """Initialize the pseudo m/ ref.""" + if self.manifest.branch: + if self.use_git_worktrees: + # Set up the m/ space to point to the worktree-specific ref + # space. We'll update the worktree-specific ref space on each + # checkout. + ref = R_M + self.manifest.branch + if not self.bare_ref.symref(ref): + self.bare_git.symbolic_ref( + "-m", + "redirecting to worktree scope", + ref, + R_WORKTREE_M + self.manifest.branch, + ) + + # We can't update this ref with git worktrees until it exists. + # We'll wait until the initial checkout to set it. + if not os.path.exists(self.worktree): + return + + base = R_WORKTREE_M + active_git = self.work_git + + self._InitAnyMRef(HEAD, self.bare_git, detach=True) + else: + base = R_M + active_git = self.bare_git + + self._InitAnyMRef(base + self.manifest.branch, active_git) + + def _InitMirrorHead(self): + self._InitAnyMRef(HEAD, self.bare_git) + + def _InitAnyMRef(self, ref, active_git, detach=False): + """Initialize |ref| in |active_git| to the value in the manifest. + + This points |ref| to the setting in the manifest. + + Args: + ref: The branch to update. + active_git: The git repository to make updates in. + detach: Whether to update target of symbolic refs, or overwrite the + ref directly (and thus make it non-symbolic). + """ + cur = self.bare_ref.symref(ref) + + if self.revisionId: + if cur != "" or self.bare_ref.get(ref) != self.revisionId: + msg = "manifest set to %s" % self.revisionId + dst = self.revisionId + "^0" + active_git.UpdateRef(ref, dst, message=msg, detach=True) + else: + remote = self.GetRemote() + dst = remote.ToLocal(self.revisionExpr) + if cur != dst: + msg = "manifest set to %s" % self.revisionExpr + if detach: + active_git.UpdateRef(ref, dst, message=msg, detach=True) + else: + active_git.symbolic_ref("-m", msg, ref, dst) + + def _CheckDirReference(self, srcdir, destdir): + # Git worktrees don't use symlinks to share at all. + if self.use_git_worktrees: + return + + for name in self.shareable_dirs: + # Try to self-heal a bit in simple cases. + dst_path = os.path.join(destdir, name) + src_path = os.path.join(srcdir, name) + + dst = platform_utils.realpath(dst_path) + if os.path.lexists(dst): + src = platform_utils.realpath(src_path) + # Fail if the links are pointing to the wrong place. + if src != dst: + _error("%s is different in %s vs %s", name, destdir, srcdir) + raise GitError( + "--force-sync not enabled; cannot overwrite a local " + "work tree. If you're comfortable with the " + "possibility of losing the work tree's git metadata," + " use `repo sync --force-sync {0}` to " + "proceed.".format(self.RelPath(local=False)) + ) + + def _ReferenceGitDir(self, gitdir, dotgit, copy_all): + """Update |dotgit| to reference |gitdir|, using symlinks where possible. + + Args: + gitdir: The bare git repository. Must already be initialized. + dotgit: The repository you would like to initialize. + copy_all: If true, copy all remaining files from |gitdir| -> + |dotgit|. This saves you the effort of initializing |dotgit| + yourself. + """ + symlink_dirs = self.shareable_dirs[:] + to_symlink = symlink_dirs + + to_copy = [] + if copy_all: + to_copy = platform_utils.listdir(gitdir) + + dotgit = platform_utils.realpath(dotgit) + for name in set(to_copy).union(to_symlink): + try: + src = platform_utils.realpath(os.path.join(gitdir, name)) + dst = os.path.join(dotgit, name) + + if os.path.lexists(dst): + continue + + # If the source dir doesn't exist, create an empty dir. + if name in symlink_dirs and not os.path.lexists(src): + os.makedirs(src) + + if name in to_symlink: + platform_utils.symlink( + os.path.relpath(src, os.path.dirname(dst)), dst + ) + elif copy_all and not platform_utils.islink(dst): + if platform_utils.isdir(src): + shutil.copytree(src, dst) + elif os.path.isfile(src): + shutil.copy(src, dst) + + except OSError as e: + if e.errno == errno.EPERM: + raise DownloadError(self._get_symlink_error_message()) + else: + raise + + def _InitGitWorktree(self): + """Init the project using git worktrees.""" + self.bare_git.worktree("prune") + self.bare_git.worktree( + "add", + "-ff", + "--checkout", + "--detach", + "--lock", + self.worktree, + self.GetRevisionId(), + ) - def __init__(self, config, detach_head=False): - self._messages = [] - self._failures = [] - self._later_queue1 = [] - self._later_queue2 = [] + # Rewrite the internal state files to use relative paths between the + # checkouts & worktrees. + dotgit = os.path.join(self.worktree, ".git") + with open(dotgit, "r") as fp: + # Figure out the checkout->worktree path. + setting = fp.read() + assert setting.startswith("gitdir:") + git_worktree_path = setting.split(":", 1)[1].strip() + # Some platforms (e.g. Windows) won't let us update dotgit in situ + # because of file permissions. Delete it and recreate it from scratch + # to avoid. + platform_utils.remove(dotgit) + # Use relative path from checkout->worktree & maintain Unix line endings + # on all OS's to match git behavior. + with open(dotgit, "w", newline="\n") as fp: + print( + "gitdir:", + os.path.relpath(git_worktree_path, self.worktree), + file=fp, + ) + # Use relative path from worktree->checkout & maintain Unix line endings + # on all OS's to match git behavior. + with open( + os.path.join(git_worktree_path, "gitdir"), "w", newline="\n" + ) as fp: + print(os.path.relpath(dotgit, git_worktree_path), file=fp) + + self._InitMRef() + + def _InitWorkTree(self, force_sync=False, submodules=False): + """Setup the worktree .git path. + + This is the user-visible path like src/foo/.git/. + + With non-git-worktrees, this will be a symlink to the .repo/projects/ + path. With git-worktrees, this will be a .git file using "gitdir: ..." + syntax. + + Older checkouts had .git/ directories. If we see that, migrate it. + + This also handles changes in the manifest. Maybe this project was + backed by "foo/bar" on the server, but now it's "new/foo/bar". We have + to update the path we point to under .repo/projects/ to match. + """ + dotgit = os.path.join(self.worktree, ".git") + + # If using an old layout style (a directory), migrate it. + if not platform_utils.islink(dotgit) and platform_utils.isdir(dotgit): + self._MigrateOldWorkTreeGitDir(dotgit) + + init_dotgit = not os.path.exists(dotgit) + if self.use_git_worktrees: + if init_dotgit: + self._InitGitWorktree() + self._CopyAndLinkFiles() + else: + if not init_dotgit: + # See if the project has changed. + if platform_utils.realpath( + self.gitdir + ) != platform_utils.realpath(dotgit): + platform_utils.remove(dotgit) + + if init_dotgit or not os.path.exists(dotgit): + os.makedirs(self.worktree, exist_ok=True) + platform_utils.symlink( + os.path.relpath(self.gitdir, self.worktree), dotgit + ) + + if init_dotgit: + _lwrite( + os.path.join(dotgit, HEAD), "%s\n" % self.GetRevisionId() + ) + + # Finish checking out the worktree. + cmd = ["read-tree", "--reset", "-u", "-v", HEAD] + if GitCommand(self, cmd).Wait() != 0: + raise GitError( + "Cannot initialize work tree for " + self.name + ) + + if submodules: + self._SyncSubmodules(quiet=True) + self._CopyAndLinkFiles() + + @classmethod + def _MigrateOldWorkTreeGitDir(cls, dotgit): + """Migrate the old worktree .git/ dir style to a symlink. + + This logic specifically only uses state from |dotgit| to figure out + where to move content and not |self|. This way if the backing project + also changed places, we only do the .git/ dir to .git symlink migration + here. The path updates will happen independently. + """ + # Figure out where in .repo/projects/ it's pointing to. + if not os.path.islink(os.path.join(dotgit, "refs")): + raise GitError(f"{dotgit}: unsupported checkout state") + gitdir = os.path.dirname(os.path.realpath(os.path.join(dotgit, "refs"))) + + # Remove known symlink paths that exist in .repo/projects/. + KNOWN_LINKS = { + "config", + "description", + "hooks", + "info", + "logs", + "objects", + "packed-refs", + "refs", + "rr-cache", + "shallow", + "svn", + } + # Paths that we know will be in both, but are safe to clobber in + # .repo/projects/. + SAFE_TO_CLOBBER = { + "COMMIT_EDITMSG", + "FETCH_HEAD", + "HEAD", + "gc.log", + "gitk.cache", + "index", + "ORIG_HEAD", + } + + # First see if we'd succeed before starting the migration. + unknown_paths = [] + for name in platform_utils.listdir(dotgit): + # Ignore all temporary/backup names. These are common with vim & + # emacs. + if name.endswith("~") or (name[0] == "#" and name[-1] == "#"): + continue + + dotgit_path = os.path.join(dotgit, name) + if name in KNOWN_LINKS: + if not platform_utils.islink(dotgit_path): + unknown_paths.append(f"{dotgit_path}: should be a symlink") + else: + gitdir_path = os.path.join(gitdir, name) + if name not in SAFE_TO_CLOBBER and os.path.exists(gitdir_path): + unknown_paths.append( + f"{dotgit_path}: unknown file; please file a bug" + ) + if unknown_paths: + raise GitError("Aborting migration: " + "\n".join(unknown_paths)) + + # Now walk the paths and sync the .git/ to .repo/projects/. + for name in platform_utils.listdir(dotgit): + dotgit_path = os.path.join(dotgit, name) + + # Ignore all temporary/backup names. These are common with vim & + # emacs. + if name.endswith("~") or (name[0] == "#" and name[-1] == "#"): + platform_utils.remove(dotgit_path) + elif name in KNOWN_LINKS: + platform_utils.remove(dotgit_path) + else: + gitdir_path = os.path.join(gitdir, name) + platform_utils.remove(gitdir_path, missing_ok=True) + platform_utils.rename(dotgit_path, gitdir_path) + + # Now that the dir should be empty, clear it out, and symlink it over. + platform_utils.rmdir(dotgit) + platform_utils.symlink( + os.path.relpath(gitdir, os.path.dirname(dotgit)), dotgit + ) - self.out = _SyncColoring(config) - self.out.redirect(sys.stderr) + def _get_symlink_error_message(self): + if platform_utils.isWindows(): + return ( + "Unable to create symbolic link. Please re-run the command as " + "Administrator, or see " + "https://github.com/git-for-windows/git/wiki/Symbolic-Links " + "for other options." + ) + return "filesystem must support symlinks" + + def _revlist(self, *args, **kw): + a = [] + a.extend(args) + a.append("--") + return self.work_git.rev_list(*a, **kw) + + @property + def _allrefs(self): + return self.bare_ref.all + + def _getLogs( + self, rev1, rev2, oneline=False, color=True, pretty_format=None + ): + """Get logs between two revisions of this project.""" + comp = ".." + if rev1: + revs = [rev1] + if rev2: + revs.extend([comp, rev2]) + cmd = ["log", "".join(revs)] + out = DiffColoring(self.config) + if out.is_on and color: + cmd.append("--color") + if pretty_format is not None: + cmd.append("--pretty=format:%s" % pretty_format) + if oneline: + cmd.append("--oneline") - self.detach_head = detach_head - self.clean = True - self.recent_clean = True + try: + log = GitCommand( + self, cmd, capture_stdout=True, capture_stderr=True + ) + if log.Wait() == 0: + return log.stdout + except GitError: + # worktree may not exist if groups changed for example. In that + # case, try in gitdir instead. + if not os.path.exists(self.worktree): + return self.bare_git.log(*cmd[1:]) + else: + raise + return None - def info(self, project, fmt, *args): - self._messages.append(_InfoMessage(project, fmt % args)) + def getAddedAndRemovedLogs( + self, toProject, oneline=False, color=True, pretty_format=None + ): + """Get the list of logs from this revision to given revisionId""" + logs = {} + selfId = self.GetRevisionId(self._allrefs) + toId = toProject.GetRevisionId(toProject._allrefs) + + logs["added"] = self._getLogs( + selfId, + toId, + oneline=oneline, + color=color, + pretty_format=pretty_format, + ) + logs["removed"] = self._getLogs( + toId, + selfId, + oneline=oneline, + color=color, + pretty_format=pretty_format, + ) + return logs + + class _GitGetByExec(object): + def __init__(self, project, bare, gitdir): + self._project = project + self._bare = bare + self._gitdir = gitdir + + # __getstate__ and __setstate__ are required for pickling because + # __getattr__ exists. + def __getstate__(self): + return (self._project, self._bare, self._gitdir) + + def __setstate__(self, state): + self._project, self._bare, self._gitdir = state + + def LsOthers(self): + p = GitCommand( + self._project, + ["ls-files", "-z", "--others", "--exclude-standard"], + bare=False, + gitdir=self._gitdir, + capture_stdout=True, + capture_stderr=True, + ) + if p.Wait() == 0: + out = p.stdout + if out: + # Backslash is not anomalous. + return out[:-1].split("\0") + return [] + + def DiffZ(self, name, *args): + cmd = [name] + cmd.append("-z") + cmd.append("--ignore-submodules") + cmd.extend(args) + p = GitCommand( + self._project, + cmd, + gitdir=self._gitdir, + bare=False, + capture_stdout=True, + capture_stderr=True, + ) + p.Wait() + r = {} + out = p.stdout + if out: + out = iter(out[:-1].split("\0")) + while out: + try: + info = next(out) + path = next(out) + except StopIteration: + break + + class _Info(object): + def __init__(self, path, omode, nmode, oid, nid, state): + self.path = path + self.src_path = None + self.old_mode = omode + self.new_mode = nmode + self.old_id = oid + self.new_id = nid + + if len(state) == 1: + self.status = state + self.level = None + else: + self.status = state[:1] + self.level = state[1:] + while self.level.startswith("0"): + self.level = self.level[1:] + + info = info[1:].split(" ") + info = _Info(path, *info) + if info.status in ("R", "C"): + info.src_path = info.path + info.path = next(out) + r[info.path] = info + return r + + def GetDotgitPath(self, subpath=None): + """Return the full path to the .git dir. + + As a convenience, append |subpath| if provided. + """ + if self._bare: + dotgit = self._gitdir + else: + dotgit = os.path.join(self._project.worktree, ".git") + if os.path.isfile(dotgit): + # Git worktrees use a "gitdir:" syntax to point to the + # scratch space. + with open(dotgit) as fp: + setting = fp.read() + assert setting.startswith("gitdir:") + gitdir = setting.split(":", 1)[1].strip() + dotgit = os.path.normpath( + os.path.join(self._project.worktree, gitdir) + ) + + return dotgit if subpath is None else os.path.join(dotgit, subpath) + + def GetHead(self): + """Return the ref that HEAD points to.""" + path = self.GetDotgitPath(subpath=HEAD) + try: + with open(path) as fd: + line = fd.readline() + except IOError as e: + raise NoManifestException(path, str(e)) + try: + line = line.decode() + except AttributeError: + pass + if line.startswith("ref: "): + return line[5:-1] + return line[:-1] + + def SetHead(self, ref, message=None): + cmdv = [] + if message is not None: + cmdv.extend(["-m", message]) + cmdv.append(HEAD) + cmdv.append(ref) + self.symbolic_ref(*cmdv) + + def DetachHead(self, new, message=None): + cmdv = ["--no-deref"] + if message is not None: + cmdv.extend(["-m", message]) + cmdv.append(HEAD) + cmdv.append(new) + self.update_ref(*cmdv) + + def UpdateRef(self, name, new, old=None, message=None, detach=False): + cmdv = [] + if message is not None: + cmdv.extend(["-m", message]) + if detach: + cmdv.append("--no-deref") + cmdv.append(name) + cmdv.append(new) + if old is not None: + cmdv.append(old) + self.update_ref(*cmdv) + + def DeleteRef(self, name, old=None): + if not old: + old = self.rev_parse(name) + self.update_ref("-d", name, old) + self._project.bare_ref.deleted(name) + + def rev_list(self, *args, **kw): + if "format" in kw: + cmdv = ["log", "--pretty=format:%s" % kw["format"]] + else: + cmdv = ["rev-list"] + cmdv.extend(args) + p = GitCommand( + self._project, + cmdv, + bare=self._bare, + gitdir=self._gitdir, + capture_stdout=True, + capture_stderr=True, + ) + if p.Wait() != 0: + raise GitError( + "%s rev-list %s: %s" + % (self._project.name, str(args), p.stderr) + ) + return p.stdout.splitlines() + + def __getattr__(self, name): + """Allow arbitrary git commands using pythonic syntax. + + This allows you to do things like: + git_obj.rev_parse('HEAD') + + Since we don't have a 'rev_parse' method defined, the __getattr__ + will run. We'll replace the '_' with a '-' and try to run a git + command. Any other positional arguments will be passed to the git + command, and the following keyword arguments are supported: + config: An optional dict of git config options to be passed with + '-c'. + + Args: + name: The name of the git command to call. Any '_' characters + will be replaced with '-'. + + Returns: + A callable object that will try to call git with the named + command. + """ + name = name.replace("_", "-") + + def runner(*args, **kwargs): + cmdv = [] + config = kwargs.pop("config", None) + for k in kwargs: + raise TypeError( + "%s() got an unexpected keyword argument %r" % (name, k) + ) + if config is not None: + for k, v in config.items(): + cmdv.append("-c") + cmdv.append("%s=%s" % (k, v)) + cmdv.append(name) + cmdv.extend(args) + p = GitCommand( + self._project, + cmdv, + bare=self._bare, + gitdir=self._gitdir, + capture_stdout=True, + capture_stderr=True, + ) + if p.Wait() != 0: + raise GitError( + "%s %s: %s" % (self._project.name, name, p.stderr) + ) + r = p.stdout + if r.endswith("\n") and r.index("\n") == len(r) - 1: + return r[:-1] + return r + + return runner - def fail(self, project, err=None): - self._failures.append(_Failure(project, err)) - self._MarkUnclean() - def later1(self, project, what): - self._later_queue1.append(_Later(project, what)) +class _PriorSyncFailedError(Exception): + def __str__(self): + return "prior sync failed; rebase still in progress" - def later2(self, project, what): - self._later_queue2.append(_Later(project, what)) - def Finish(self): - self._PrintMessages() - self._RunLater() - self._PrintMessages() - return self.clean +class _DirtyError(Exception): + def __str__(self): + return "contains uncommitted changes" - def Recently(self): - recent_clean = self.recent_clean - self.recent_clean = True - return recent_clean - def _MarkUnclean(self): - self.clean = False - self.recent_clean = False +class _InfoMessage(object): + def __init__(self, project, text): + self.project = project + self.text = text - def _RunLater(self): - for q in ['_later_queue1', '_later_queue2']: - if not self._RunQueue(q): - return + def Print(self, syncbuf): + syncbuf.out.info( + "%s/: %s", self.project.RelPath(local=False), self.text + ) + syncbuf.out.nl() - def _RunQueue(self, queue): - for m in getattr(self, queue): - if not m.Run(self): - self._MarkUnclean() - return False - setattr(self, queue, []) - return True - def _PrintMessages(self): - if self._messages or self._failures: - if os.isatty(2): - self.out.write(progress.CSI_ERASE_LINE) - self.out.write('\r') +class _Failure(object): + def __init__(self, project, why): + self.project = project + self.why = why - for m in self._messages: - m.Print(self) - for m in self._failures: - m.Print(self) + def Print(self, syncbuf): + syncbuf.out.fail( + "error: %s/: %s", self.project.RelPath(local=False), str(self.why) + ) + syncbuf.out.nl() - self._messages = [] - self._failures = [] +class _Later(object): + def __init__(self, project, action): + self.project = project + self.action = action + + def Run(self, syncbuf): + out = syncbuf.out + out.project("project %s/", self.project.RelPath(local=False)) + out.nl() + try: + self.action() + out.nl() + return True + except GitError: + out.nl() + return False -class MetaProject(Project): - """A special project housed under .repo.""" - - def __init__(self, manifest, name, gitdir, worktree): - Project.__init__(self, - manifest=manifest, - name=name, - gitdir=gitdir, - objdir=gitdir, - worktree=worktree, - remote=RemoteSpec('origin'), - relpath='.repo/%s' % name, - revisionExpr='refs/heads/master', - revisionId=None, - groups=None) - - def PreSync(self): - if self.Exists: - cb = self.CurrentBranch - if cb: - base = self.GetBranch(cb).merge - if base: - self.revisionExpr = base - self.revisionId = None - - @property - def HasChanges(self): - """Has the remote received new commits not yet checked out?""" - if not self.remote or not self.revisionExpr: - return False - - all_refs = self.bare_ref.all - revid = self.GetRevisionId(all_refs) - head = self.work_git.GetHead() - if head.startswith(R_HEADS): - try: - head = all_refs[head] - except KeyError: - head = None - - if revid == head: - return False - elif self._revlist(not_rev(HEAD), revid): - return True - return False +class _SyncColoring(Coloring): + def __init__(self, config): + super().__init__(config, "reposync") + self.project = self.printer("header", attr="bold") + self.info = self.printer("info") + self.fail = self.printer("fail", fg="red") -class RepoProject(MetaProject): - """The MetaProject for repo itself.""" - @property - def LastFetch(self): - try: - fh = os.path.join(self.gitdir, 'FETCH_HEAD') - return os.path.getmtime(fh) - except OSError: - return 0 +class SyncBuffer(object): + def __init__(self, config, detach_head=False): + self._messages = [] + self._failures = [] + self._later_queue1 = [] + self._later_queue2 = [] + self.out = _SyncColoring(config) + self.out.redirect(sys.stderr) -class ManifestProject(MetaProject): - """The MetaProject for manifests.""" - - def MetaBranchSwitch(self, submodules=False): - """Prepare for manifest branch switch.""" - - # detach and delete manifest branch, allowing a new - # branch to take over - syncbuf = SyncBuffer(self.config, detach_head=True) - self.Sync_LocalHalf(syncbuf, submodules=submodules) - syncbuf.Finish() - - return GitCommand(self, - ['update-ref', '-d', 'refs/heads/default'], - capture_stdout=True, - capture_stderr=True).Wait() == 0 - - @property - def standalone_manifest_url(self): - """The URL of the standalone manifest, or None.""" - return self.config.GetString('manifest.standalone') - - @property - def manifest_groups(self): - """The manifest groups string.""" - return self.config.GetString('manifest.groups') - - @property - def reference(self): - """The --reference for this manifest.""" - return self.config.GetString('repo.reference') - - @property - def dissociate(self): - """Whether to dissociate.""" - return self.config.GetBoolean('repo.dissociate') - - @property - def archive(self): - """Whether we use archive.""" - return self.config.GetBoolean('repo.archive') - - @property - def mirror(self): - """Whether we use mirror.""" - return self.config.GetBoolean('repo.mirror') - - @property - def use_worktree(self): - """Whether we use worktree.""" - return self.config.GetBoolean('repo.worktree') - - @property - def clone_bundle(self): - """Whether we use clone_bundle.""" - return self.config.GetBoolean('repo.clonebundle') - - @property - def submodules(self): - """Whether we use submodules.""" - return self.config.GetBoolean('repo.submodules') - - @property - def git_lfs(self): - """Whether we use git_lfs.""" - return self.config.GetBoolean('repo.git-lfs') - - @property - def use_superproject(self): - """Whether we use superproject.""" - return self.config.GetBoolean('repo.superproject') - - @property - def partial_clone(self): - """Whether this is a partial clone.""" - return self.config.GetBoolean('repo.partialclone') - - @property - def depth(self): - """Partial clone depth.""" - return self.config.GetString('repo.depth') - - @property - def clone_filter(self): - """The clone filter.""" - return self.config.GetString('repo.clonefilter') - - @property - def partial_clone_exclude(self): - """Partial clone exclude string""" - return self.config.GetString('repo.partialcloneexclude') - - @property - def manifest_platform(self): - """The --platform argument from `repo init`.""" - return self.config.GetString('manifest.platform') - - @property - def _platform_name(self): - """Return the name of the platform.""" - return platform.system().lower() - - def SyncWithPossibleInit(self, submanifest, verbose=False, - current_branch_only=False, tags='', git_event_log=None): - """Sync a manifestProject, possibly for the first time. - - Call Sync() with arguments from the most recent `repo init`. If this is a - new sub manifest, then inherit options from the parent's manifestProject. - - This is used by subcmds.Sync() to do an initial download of new sub - manifests. - - Args: - submanifest: an XmlSubmanifest, the submanifest to re-sync. - verbose: a boolean, whether to show all output, rather than only errors. - current_branch_only: a boolean, whether to only fetch the current manifest - branch from the server. - tags: a boolean, whether to fetch tags. - git_event_log: an EventLog, for git tracing. - """ - # TODO(lamontjones): when refactoring sync (and init?) consider how to - # better get the init options that we should use for new submanifests that - # are added when syncing an existing workspace. - git_event_log = git_event_log or EventLog() - spec = submanifest.ToSubmanifestSpec() - # Use the init options from the existing manifestProject, or the parent if - # it doesn't exist. - # - # Today, we only support changing manifest_groups on the sub-manifest, with - # no supported-for-the-user way to change the other arguments from those - # specified by the outermost manifest. - # - # TODO(lamontjones): determine which of these should come from the outermost - # manifest and which should come from the parent manifest. - mp = self if self.Exists else submanifest.parent.manifestProject - return self.Sync( - manifest_url=spec.manifestUrl, - manifest_branch=spec.revision, - standalone_manifest=mp.standalone_manifest_url, - groups=mp.manifest_groups, - platform=mp.manifest_platform, - mirror=mp.mirror, - dissociate=mp.dissociate, - reference=mp.reference, - worktree=mp.use_worktree, - submodules=mp.submodules, - archive=mp.archive, - partial_clone=mp.partial_clone, - clone_filter=mp.clone_filter, - partial_clone_exclude=mp.partial_clone_exclude, - clone_bundle=mp.clone_bundle, - git_lfs=mp.git_lfs, - use_superproject=mp.use_superproject, - verbose=verbose, - current_branch_only=current_branch_only, - tags=tags, - depth=mp.depth, - git_event_log=git_event_log, - manifest_name=spec.manifestName, - this_manifest_only=True, - outer_manifest=False, - ) - - def Sync(self, _kwargs_only=(), manifest_url='', manifest_branch=None, - standalone_manifest=False, groups='', mirror=False, reference='', - dissociate=False, worktree=False, submodules=False, archive=False, - partial_clone=None, depth=None, clone_filter='blob:none', - partial_clone_exclude=None, clone_bundle=None, git_lfs=None, - use_superproject=None, verbose=False, current_branch_only=False, - git_event_log=None, platform='', manifest_name='default.xml', - tags='', this_manifest_only=False, outer_manifest=True): - """Sync the manifest and all submanifests. - - Args: - manifest_url: a string, the URL of the manifest project. - manifest_branch: a string, the manifest branch to use. - standalone_manifest: a boolean, whether to store the manifest as a static - file. - groups: a string, restricts the checkout to projects with the specified - groups. - mirror: a boolean, whether to create a mirror of the remote repository. - reference: a string, location of a repo instance to use as a reference. - dissociate: a boolean, whether to dissociate from reference mirrors after - clone. - worktree: a boolean, whether to use git-worktree to manage projects. - submodules: a boolean, whether sync submodules associated with the - manifest project. - archive: a boolean, whether to checkout each project as an archive. See - git-archive. - partial_clone: a boolean, whether to perform a partial clone. - depth: an int, how deep of a shallow clone to create. - clone_filter: a string, filter to use with partial_clone. - partial_clone_exclude : a string, comma-delimeted list of project namess - to exclude from partial clone. - clone_bundle: a boolean, whether to enable /clone.bundle on HTTP/HTTPS. - git_lfs: a boolean, whether to enable git LFS support. - use_superproject: a boolean, whether to use the manifest superproject to - sync projects. - verbose: a boolean, whether to show all output, rather than only errors. - current_branch_only: a boolean, whether to only fetch the current manifest - branch from the server. - platform: a string, restrict the checkout to projects with the specified - platform group. - git_event_log: an EventLog, for git tracing. - tags: a boolean, whether to fetch tags. - manifest_name: a string, the name of the manifest file to use. - this_manifest_only: a boolean, whether to only operate on the current sub - manifest. - outer_manifest: a boolean, whether to start at the outermost manifest. + self.detach_head = detach_head + self.clean = True + self.recent_clean = True - Returns: - a boolean, whether the sync was successful. - """ - assert _kwargs_only == (), 'Sync only accepts keyword arguments.' - - groups = groups or self.manifest.GetDefaultGroupsStr(with_platform=False) - platform = platform or 'auto' - git_event_log = git_event_log or EventLog() - if outer_manifest and self.manifest.is_submanifest: - # In a multi-manifest checkout, use the outer manifest unless we are told - # not to. - return self.client.outer_manifest.manifestProject.Sync( - manifest_url=manifest_url, - manifest_branch=manifest_branch, - standalone_manifest=standalone_manifest, - groups=groups, - platform=platform, - mirror=mirror, - dissociate=dissociate, - reference=reference, - worktree=worktree, - submodules=submodules, - archive=archive, - partial_clone=partial_clone, - clone_filter=clone_filter, - partial_clone_exclude=partial_clone_exclude, - clone_bundle=clone_bundle, - git_lfs=git_lfs, - use_superproject=use_superproject, - verbose=verbose, - current_branch_only=current_branch_only, - tags=tags, - depth=depth, - git_event_log=git_event_log, - manifest_name=manifest_name, - this_manifest_only=this_manifest_only, - outer_manifest=False) - - # If repo has already been initialized, we take -u with the absence of - # --standalone-manifest to mean "transition to a standard repo set up", - # which necessitates starting fresh. - # If --standalone-manifest is set, we always tear everything down and start - # anew. - if self.Exists: - was_standalone_manifest = self.config.GetString('manifest.standalone') - if was_standalone_manifest and not manifest_url: - print('fatal: repo was initialized with a standlone manifest, ' - 'cannot be re-initialized without --manifest-url/-u') - return False + def info(self, project, fmt, *args): + self._messages.append(_InfoMessage(project, fmt % args)) - if standalone_manifest or (was_standalone_manifest and manifest_url): - self.config.ClearCache() - if self.gitdir and os.path.exists(self.gitdir): - platform_utils.rmtree(self.gitdir) - if self.worktree and os.path.exists(self.worktree): - platform_utils.rmtree(self.worktree) - - is_new = not self.Exists - if is_new: - if not manifest_url: - print('fatal: manifest url is required.', file=sys.stderr) - return False + def fail(self, project, err=None): + self._failures.append(_Failure(project, err)) + self._MarkUnclean() - if verbose: - print('Downloading manifest from %s' % - (GitConfig.ForUser().UrlInsteadOf(manifest_url),), - file=sys.stderr) - - # The manifest project object doesn't keep track of the path on the - # server where this git is located, so let's save that here. - mirrored_manifest_git = None - if reference: - manifest_git_path = urllib.parse.urlparse(manifest_url).path[1:] - mirrored_manifest_git = os.path.join(reference, manifest_git_path) - if not mirrored_manifest_git.endswith(".git"): - mirrored_manifest_git += ".git" - if not os.path.exists(mirrored_manifest_git): - mirrored_manifest_git = os.path.join(reference, - '.repo/manifests.git') - - self._InitGitDir(mirror_git=mirrored_manifest_git) - - # If standalone_manifest is set, mark the project as "standalone" -- we'll - # still do much of the manifests.git set up, but will avoid actual syncs to - # a remote. - if standalone_manifest: - self.config.SetString('manifest.standalone', manifest_url) - elif not manifest_url and not manifest_branch: - # If -u is set and --standalone-manifest is not, then we're not in - # standalone mode. Otherwise, use config to infer what we were in the last - # init. - standalone_manifest = bool(self.config.GetString('manifest.standalone')) - if not standalone_manifest: - self.config.SetString('manifest.standalone', None) - - self._ConfigureDepth(depth) - - # Set the remote URL before the remote branch as we might need it below. - if manifest_url: - r = self.GetRemote() - r.url = manifest_url - r.ResetFetch() - r.Save() - - if not standalone_manifest: - if manifest_branch: - if manifest_branch == 'HEAD': - manifest_branch = self.ResolveRemoteHead() - if manifest_branch is None: - print('fatal: unable to resolve HEAD', file=sys.stderr) + def later1(self, project, what): + self._later_queue1.append(_Later(project, what)) + + def later2(self, project, what): + self._later_queue2.append(_Later(project, what)) + + def Finish(self): + self._PrintMessages() + self._RunLater() + self._PrintMessages() + return self.clean + + def Recently(self): + recent_clean = self.recent_clean + self.recent_clean = True + return recent_clean + + def _MarkUnclean(self): + self.clean = False + self.recent_clean = False + + def _RunLater(self): + for q in ["_later_queue1", "_later_queue2"]: + if not self._RunQueue(q): + return + + def _RunQueue(self, queue): + for m in getattr(self, queue): + if not m.Run(self): + self._MarkUnclean() + return False + setattr(self, queue, []) + return True + + def _PrintMessages(self): + if self._messages or self._failures: + if os.isatty(2): + self.out.write(progress.CSI_ERASE_LINE) + self.out.write("\r") + + for m in self._messages: + m.Print(self) + for m in self._failures: + m.Print(self) + + self._messages = [] + self._failures = [] + + +class MetaProject(Project): + """A special project housed under .repo.""" + + def __init__(self, manifest, name, gitdir, worktree): + Project.__init__( + self, + manifest=manifest, + name=name, + gitdir=gitdir, + objdir=gitdir, + worktree=worktree, + remote=RemoteSpec("origin"), + relpath=".repo/%s" % name, + revisionExpr="refs/heads/master", + revisionId=None, + groups=None, + ) + + def PreSync(self): + if self.Exists: + cb = self.CurrentBranch + if cb: + base = self.GetBranch(cb).merge + if base: + self.revisionExpr = base + self.revisionId = None + + @property + def HasChanges(self): + """Has the remote received new commits not yet checked out?""" + if not self.remote or not self.revisionExpr: return False - self.revisionExpr = manifest_branch - else: - if is_new: - default_branch = self.ResolveRemoteHead() - if default_branch is None: - # If the remote doesn't have HEAD configured, default to master. - default_branch = 'refs/heads/master' - self.revisionExpr = default_branch - else: - self.PreSync() - - groups = re.split(r'[,\s]+', groups or '') - all_platforms = ['linux', 'darwin', 'windows'] - platformize = lambda x: 'platform-' + x - if platform == 'auto': - if not mirror and not self.mirror: - groups.append(platformize(self._platform_name)) - elif platform == 'all': - groups.extend(map(platformize, all_platforms)) - elif platform in all_platforms: - groups.append(platformize(platform)) - elif platform != 'none': - print('fatal: invalid platform flag', file=sys.stderr) - return False - self.config.SetString('manifest.platform', platform) - - groups = [x for x in groups if x] - groupstr = ','.join(groups) - if platform == 'auto' and groupstr == self.manifest.GetDefaultGroupsStr(): - groupstr = None - self.config.SetString('manifest.groups', groupstr) - - if reference: - self.config.SetString('repo.reference', reference) - - if dissociate: - self.config.SetBoolean('repo.dissociate', dissociate) - - if worktree: - if mirror: - print('fatal: --mirror and --worktree are incompatible', - file=sys.stderr) - return False - if submodules: - print('fatal: --submodules and --worktree are incompatible', - file=sys.stderr) - return False - self.config.SetBoolean('repo.worktree', worktree) - if is_new: - self.use_git_worktrees = True - print('warning: --worktree is experimental!', file=sys.stderr) - - if archive: - if is_new: - self.config.SetBoolean('repo.archive', archive) - else: - print('fatal: --archive is only supported when initializing a new ' - 'workspace.', file=sys.stderr) - print('Either delete the .repo folder in this workspace, or initialize ' - 'in another location.', file=sys.stderr) - return False - if mirror: - if is_new: - self.config.SetBoolean('repo.mirror', mirror) - else: - print('fatal: --mirror is only supported when initializing a new ' - 'workspace.', file=sys.stderr) - print('Either delete the .repo folder in this workspace, or initialize ' - 'in another location.', file=sys.stderr) - return False + all_refs = self.bare_ref.all + revid = self.GetRevisionId(all_refs) + head = self.work_git.GetHead() + if head.startswith(R_HEADS): + try: + head = all_refs[head] + except KeyError: + head = None - if partial_clone is not None: - if mirror: - print('fatal: --mirror and --partial-clone are mutually exclusive', - file=sys.stderr) - return False - self.config.SetBoolean('repo.partialclone', partial_clone) - if clone_filter: - self.config.SetString('repo.clonefilter', clone_filter) - elif self.partial_clone: - clone_filter = self.clone_filter - else: - clone_filter = None - - if partial_clone_exclude is not None: - self.config.SetString('repo.partialcloneexclude', partial_clone_exclude) - - if clone_bundle is None: - clone_bundle = False if partial_clone else True - else: - self.config.SetBoolean('repo.clonebundle', clone_bundle) - - if submodules: - self.config.SetBoolean('repo.submodules', submodules) - - if git_lfs is not None: - if git_lfs: - git_require((2, 17, 0), fail=True, msg='Git LFS support') - - self.config.SetBoolean('repo.git-lfs', git_lfs) - if not is_new: - print('warning: Changing --git-lfs settings will only affect new project checkouts.\n' - ' Existing projects will require manual updates.\n', file=sys.stderr) - - if use_superproject is not None: - self.config.SetBoolean('repo.superproject', use_superproject) - - if not standalone_manifest: - success = self.Sync_NetworkHalf( - is_new=is_new, quiet=not verbose, verbose=verbose, - clone_bundle=clone_bundle, current_branch_only=current_branch_only, - tags=tags, submodules=submodules, clone_filter=clone_filter, - partial_clone_exclude=self.manifest.PartialCloneExclude).success - if not success: - r = self.GetRemote() - print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr) - - # Better delete the manifest git dir if we created it; otherwise next - # time (when user fixes problems) we won't go through the "is_new" logic. - if is_new: - platform_utils.rmtree(self.gitdir) + if revid == head: + return False + elif self._revlist(not_rev(HEAD), revid): + return True return False - if manifest_branch: - self.MetaBranchSwitch(submodules=submodules) - syncbuf = SyncBuffer(self.config) - self.Sync_LocalHalf(syncbuf, submodules=submodules) - syncbuf.Finish() +class RepoProject(MetaProject): + """The MetaProject for repo itself.""" - if is_new or self.CurrentBranch is None: - if not self.StartBranch('default'): - print('fatal: cannot create default in manifest', file=sys.stderr) - return False + @property + def LastFetch(self): + try: + fh = os.path.join(self.gitdir, "FETCH_HEAD") + return os.path.getmtime(fh) + except OSError: + return 0 - if not manifest_name: - print('fatal: manifest name (-m) is required.', file=sys.stderr) - return False - elif is_new: - # This is a new standalone manifest. - manifest_name = 'default.xml' - manifest_data = fetch.fetch_file(manifest_url, verbose=verbose) - dest = os.path.join(self.worktree, manifest_name) - os.makedirs(os.path.dirname(dest), exist_ok=True) - with open(dest, 'wb') as f: - f.write(manifest_data) +class ManifestProject(MetaProject): + """The MetaProject for manifests.""" + + def MetaBranchSwitch(self, submodules=False): + """Prepare for manifest branch switch.""" + + # detach and delete manifest branch, allowing a new + # branch to take over + syncbuf = SyncBuffer(self.config, detach_head=True) + self.Sync_LocalHalf(syncbuf, submodules=submodules) + syncbuf.Finish() + + return ( + GitCommand( + self, + ["update-ref", "-d", "refs/heads/default"], + capture_stdout=True, + capture_stderr=True, + ).Wait() + == 0 + ) + + @property + def standalone_manifest_url(self): + """The URL of the standalone manifest, or None.""" + return self.config.GetString("manifest.standalone") + + @property + def manifest_groups(self): + """The manifest groups string.""" + return self.config.GetString("manifest.groups") + + @property + def reference(self): + """The --reference for this manifest.""" + return self.config.GetString("repo.reference") + + @property + def dissociate(self): + """Whether to dissociate.""" + return self.config.GetBoolean("repo.dissociate") + + @property + def archive(self): + """Whether we use archive.""" + return self.config.GetBoolean("repo.archive") + + @property + def mirror(self): + """Whether we use mirror.""" + return self.config.GetBoolean("repo.mirror") + + @property + def use_worktree(self): + """Whether we use worktree.""" + return self.config.GetBoolean("repo.worktree") + + @property + def clone_bundle(self): + """Whether we use clone_bundle.""" + return self.config.GetBoolean("repo.clonebundle") + + @property + def submodules(self): + """Whether we use submodules.""" + return self.config.GetBoolean("repo.submodules") + + @property + def git_lfs(self): + """Whether we use git_lfs.""" + return self.config.GetBoolean("repo.git-lfs") + + @property + def use_superproject(self): + """Whether we use superproject.""" + return self.config.GetBoolean("repo.superproject") + + @property + def partial_clone(self): + """Whether this is a partial clone.""" + return self.config.GetBoolean("repo.partialclone") + + @property + def depth(self): + """Partial clone depth.""" + return self.config.GetString("repo.depth") + + @property + def clone_filter(self): + """The clone filter.""" + return self.config.GetString("repo.clonefilter") + + @property + def partial_clone_exclude(self): + """Partial clone exclude string""" + return self.config.GetString("repo.partialcloneexclude") + + @property + def manifest_platform(self): + """The --platform argument from `repo init`.""" + return self.config.GetString("manifest.platform") + + @property + def _platform_name(self): + """Return the name of the platform.""" + return platform.system().lower() + + def SyncWithPossibleInit( + self, + submanifest, + verbose=False, + current_branch_only=False, + tags="", + git_event_log=None, + ): + """Sync a manifestProject, possibly for the first time. + + Call Sync() with arguments from the most recent `repo init`. If this is + a new sub manifest, then inherit options from the parent's + manifestProject. + + This is used by subcmds.Sync() to do an initial download of new sub + manifests. - try: - self.manifest.Link(manifest_name) - except ManifestParseError as e: - print("fatal: manifest '%s' not available" % manifest_name, - file=sys.stderr) - print('fatal: %s' % str(e), file=sys.stderr) - return False - - if not this_manifest_only: - for submanifest in self.manifest.submanifests.values(): + Args: + submanifest: an XmlSubmanifest, the submanifest to re-sync. + verbose: a boolean, whether to show all output, rather than only + errors. + current_branch_only: a boolean, whether to only fetch the current + manifest branch from the server. + tags: a boolean, whether to fetch tags. + git_event_log: an EventLog, for git tracing. + """ + # TODO(lamontjones): when refactoring sync (and init?) consider how to + # better get the init options that we should use for new submanifests + # that are added when syncing an existing workspace. + git_event_log = git_event_log or EventLog() spec = submanifest.ToSubmanifestSpec() - submanifest.repo_client.manifestProject.Sync( + # Use the init options from the existing manifestProject, or the parent + # if it doesn't exist. + # + # Today, we only support changing manifest_groups on the sub-manifest, + # with no supported-for-the-user way to change the other arguments from + # those specified by the outermost manifest. + # + # TODO(lamontjones): determine which of these should come from the + # outermost manifest and which should come from the parent manifest. + mp = self if self.Exists else submanifest.parent.manifestProject + return self.Sync( manifest_url=spec.manifestUrl, manifest_branch=spec.revision, - standalone_manifest=standalone_manifest, - groups=self.manifest_groups, - platform=platform, - mirror=mirror, - dissociate=dissociate, - reference=reference, - worktree=worktree, - submodules=submodules, - archive=archive, - partial_clone=partial_clone, - clone_filter=clone_filter, - partial_clone_exclude=partial_clone_exclude, - clone_bundle=clone_bundle, - git_lfs=git_lfs, - use_superproject=use_superproject, + standalone_manifest=mp.standalone_manifest_url, + groups=mp.manifest_groups, + platform=mp.manifest_platform, + mirror=mp.mirror, + dissociate=mp.dissociate, + reference=mp.reference, + worktree=mp.use_worktree, + submodules=mp.submodules, + archive=mp.archive, + partial_clone=mp.partial_clone, + clone_filter=mp.clone_filter, + partial_clone_exclude=mp.partial_clone_exclude, + clone_bundle=mp.clone_bundle, + git_lfs=mp.git_lfs, + use_superproject=mp.use_superproject, verbose=verbose, current_branch_only=current_branch_only, tags=tags, - depth=depth, + depth=mp.depth, git_event_log=git_event_log, manifest_name=spec.manifestName, - this_manifest_only=False, + this_manifest_only=True, outer_manifest=False, ) - # Lastly, if the manifest has a then have the superproject - # sync it (if it will be used). - if git_superproject.UseSuperproject(use_superproject, self.manifest): - sync_result = self.manifest.superproject.Sync(git_event_log) - if not sync_result.success: - submanifest = '' - if self.manifest.path_prefix: - submanifest = f'for {self.manifest.path_prefix} ' - print(f'warning: git update of superproject {submanifest}failed, repo ' - 'sync will not use superproject to fetch source; while this ' - 'error is not fatal, and you can continue to run repo sync, ' - 'please run repo init with the --no-use-superproject option to ' - 'stop seeing this warning', file=sys.stderr) - if sync_result.fatal and use_superproject is not None: - return False - - return True - - def _ConfigureDepth(self, depth): - """Configure the depth we'll sync down. - - Args: - depth: an int, how deep of a partial clone to create. - """ - # Opt.depth will be non-None if user actually passed --depth to repo init. - if depth is not None: - if depth > 0: - # Positive values will set the depth. - depth = str(depth) - else: - # Negative numbers will clear the depth; passing None to SetString - # will do that. - depth = None - - # We store the depth in the main manifest project. - self.config.SetString('repo.depth', depth) + def Sync( + self, + _kwargs_only=(), + manifest_url="", + manifest_branch=None, + standalone_manifest=False, + groups="", + mirror=False, + reference="", + dissociate=False, + worktree=False, + submodules=False, + archive=False, + partial_clone=None, + depth=None, + clone_filter="blob:none", + partial_clone_exclude=None, + clone_bundle=None, + git_lfs=None, + use_superproject=None, + verbose=False, + current_branch_only=False, + git_event_log=None, + platform="", + manifest_name="default.xml", + tags="", + this_manifest_only=False, + outer_manifest=True, + ): + """Sync the manifest and all submanifests. + + Args: + manifest_url: a string, the URL of the manifest project. + manifest_branch: a string, the manifest branch to use. + standalone_manifest: a boolean, whether to store the manifest as a + static file. + groups: a string, restricts the checkout to projects with the + specified groups. + mirror: a boolean, whether to create a mirror of the remote + repository. + reference: a string, location of a repo instance to use as a + reference. + dissociate: a boolean, whether to dissociate from reference mirrors + after clone. + worktree: a boolean, whether to use git-worktree to manage projects. + submodules: a boolean, whether sync submodules associated with the + manifest project. + archive: a boolean, whether to checkout each project as an archive. + See git-archive. + partial_clone: a boolean, whether to perform a partial clone. + depth: an int, how deep of a shallow clone to create. + clone_filter: a string, filter to use with partial_clone. + partial_clone_exclude : a string, comma-delimeted list of project + names to exclude from partial clone. + clone_bundle: a boolean, whether to enable /clone.bundle on + HTTP/HTTPS. + git_lfs: a boolean, whether to enable git LFS support. + use_superproject: a boolean, whether to use the manifest + superproject to sync projects. + verbose: a boolean, whether to show all output, rather than only + errors. + current_branch_only: a boolean, whether to only fetch the current + manifest branch from the server. + platform: a string, restrict the checkout to projects with the + specified platform group. + git_event_log: an EventLog, for git tracing. + tags: a boolean, whether to fetch tags. + manifest_name: a string, the name of the manifest file to use. + this_manifest_only: a boolean, whether to only operate on the + current sub manifest. + outer_manifest: a boolean, whether to start at the outermost + manifest. + + Returns: + a boolean, whether the sync was successful. + """ + assert _kwargs_only == (), "Sync only accepts keyword arguments." + + groups = groups or self.manifest.GetDefaultGroupsStr( + with_platform=False + ) + platform = platform or "auto" + git_event_log = git_event_log or EventLog() + if outer_manifest and self.manifest.is_submanifest: + # In a multi-manifest checkout, use the outer manifest unless we are + # told not to. + return self.client.outer_manifest.manifestProject.Sync( + manifest_url=manifest_url, + manifest_branch=manifest_branch, + standalone_manifest=standalone_manifest, + groups=groups, + platform=platform, + mirror=mirror, + dissociate=dissociate, + reference=reference, + worktree=worktree, + submodules=submodules, + archive=archive, + partial_clone=partial_clone, + clone_filter=clone_filter, + partial_clone_exclude=partial_clone_exclude, + clone_bundle=clone_bundle, + git_lfs=git_lfs, + use_superproject=use_superproject, + verbose=verbose, + current_branch_only=current_branch_only, + tags=tags, + depth=depth, + git_event_log=git_event_log, + manifest_name=manifest_name, + this_manifest_only=this_manifest_only, + outer_manifest=False, + ) + + # If repo has already been initialized, we take -u with the absence of + # --standalone-manifest to mean "transition to a standard repo set up", + # which necessitates starting fresh. + # If --standalone-manifest is set, we always tear everything down and + # start anew. + if self.Exists: + was_standalone_manifest = self.config.GetString( + "manifest.standalone" + ) + if was_standalone_manifest and not manifest_url: + print( + "fatal: repo was initialized with a standlone manifest, " + "cannot be re-initialized without --manifest-url/-u" + ) + return False + + if standalone_manifest or ( + was_standalone_manifest and manifest_url + ): + self.config.ClearCache() + if self.gitdir and os.path.exists(self.gitdir): + platform_utils.rmtree(self.gitdir) + if self.worktree and os.path.exists(self.worktree): + platform_utils.rmtree(self.worktree) + + is_new = not self.Exists + if is_new: + if not manifest_url: + print("fatal: manifest url is required.", file=sys.stderr) + return False + + if verbose: + print( + "Downloading manifest from %s" + % (GitConfig.ForUser().UrlInsteadOf(manifest_url),), + file=sys.stderr, + ) + + # The manifest project object doesn't keep track of the path on the + # server where this git is located, so let's save that here. + mirrored_manifest_git = None + if reference: + manifest_git_path = urllib.parse.urlparse(manifest_url).path[1:] + mirrored_manifest_git = os.path.join( + reference, manifest_git_path + ) + if not mirrored_manifest_git.endswith(".git"): + mirrored_manifest_git += ".git" + if not os.path.exists(mirrored_manifest_git): + mirrored_manifest_git = os.path.join( + reference, ".repo/manifests.git" + ) + + self._InitGitDir(mirror_git=mirrored_manifest_git) + + # If standalone_manifest is set, mark the project as "standalone" -- + # we'll still do much of the manifests.git set up, but will avoid actual + # syncs to a remote. + if standalone_manifest: + self.config.SetString("manifest.standalone", manifest_url) + elif not manifest_url and not manifest_branch: + # If -u is set and --standalone-manifest is not, then we're not in + # standalone mode. Otherwise, use config to infer what we were in + # the last init. + standalone_manifest = bool( + self.config.GetString("manifest.standalone") + ) + if not standalone_manifest: + self.config.SetString("manifest.standalone", None) + + self._ConfigureDepth(depth) + + # Set the remote URL before the remote branch as we might need it below. + if manifest_url: + r = self.GetRemote() + r.url = manifest_url + r.ResetFetch() + r.Save() + + if not standalone_manifest: + if manifest_branch: + if manifest_branch == "HEAD": + manifest_branch = self.ResolveRemoteHead() + if manifest_branch is None: + print("fatal: unable to resolve HEAD", file=sys.stderr) + return False + self.revisionExpr = manifest_branch + else: + if is_new: + default_branch = self.ResolveRemoteHead() + if default_branch is None: + # If the remote doesn't have HEAD configured, default to + # master. + default_branch = "refs/heads/master" + self.revisionExpr = default_branch + else: + self.PreSync() + + groups = re.split(r"[,\s]+", groups or "") + all_platforms = ["linux", "darwin", "windows"] + platformize = lambda x: "platform-" + x + if platform == "auto": + if not mirror and not self.mirror: + groups.append(platformize(self._platform_name)) + elif platform == "all": + groups.extend(map(platformize, all_platforms)) + elif platform in all_platforms: + groups.append(platformize(platform)) + elif platform != "none": + print("fatal: invalid platform flag", file=sys.stderr) + return False + self.config.SetString("manifest.platform", platform) + + groups = [x for x in groups if x] + groupstr = ",".join(groups) + if ( + platform == "auto" + and groupstr == self.manifest.GetDefaultGroupsStr() + ): + groupstr = None + self.config.SetString("manifest.groups", groupstr) + + if reference: + self.config.SetString("repo.reference", reference) + + if dissociate: + self.config.SetBoolean("repo.dissociate", dissociate) + + if worktree: + if mirror: + print( + "fatal: --mirror and --worktree are incompatible", + file=sys.stderr, + ) + return False + if submodules: + print( + "fatal: --submodules and --worktree are incompatible", + file=sys.stderr, + ) + return False + self.config.SetBoolean("repo.worktree", worktree) + if is_new: + self.use_git_worktrees = True + print("warning: --worktree is experimental!", file=sys.stderr) + + if archive: + if is_new: + self.config.SetBoolean("repo.archive", archive) + else: + print( + "fatal: --archive is only supported when initializing a " + "new workspace.", + file=sys.stderr, + ) + print( + "Either delete the .repo folder in this workspace, or " + "initialize in another location.", + file=sys.stderr, + ) + return False + + if mirror: + if is_new: + self.config.SetBoolean("repo.mirror", mirror) + else: + print( + "fatal: --mirror is only supported when initializing a new " + "workspace.", + file=sys.stderr, + ) + print( + "Either delete the .repo folder in this workspace, or " + "initialize in another location.", + file=sys.stderr, + ) + return False + + if partial_clone is not None: + if mirror: + print( + "fatal: --mirror and --partial-clone are mutually " + "exclusive", + file=sys.stderr, + ) + return False + self.config.SetBoolean("repo.partialclone", partial_clone) + if clone_filter: + self.config.SetString("repo.clonefilter", clone_filter) + elif self.partial_clone: + clone_filter = self.clone_filter + else: + clone_filter = None + + if partial_clone_exclude is not None: + self.config.SetString( + "repo.partialcloneexclude", partial_clone_exclude + ) + + if clone_bundle is None: + clone_bundle = False if partial_clone else True + else: + self.config.SetBoolean("repo.clonebundle", clone_bundle) + + if submodules: + self.config.SetBoolean("repo.submodules", submodules) + + if git_lfs is not None: + if git_lfs: + git_require((2, 17, 0), fail=True, msg="Git LFS support") + + self.config.SetBoolean("repo.git-lfs", git_lfs) + if not is_new: + print( + "warning: Changing --git-lfs settings will only affect new " + "project checkouts.\n" + " Existing projects will require manual updates.\n", + file=sys.stderr, + ) + + if use_superproject is not None: + self.config.SetBoolean("repo.superproject", use_superproject) + + if not standalone_manifest: + success = self.Sync_NetworkHalf( + is_new=is_new, + quiet=not verbose, + verbose=verbose, + clone_bundle=clone_bundle, + current_branch_only=current_branch_only, + tags=tags, + submodules=submodules, + clone_filter=clone_filter, + partial_clone_exclude=self.manifest.PartialCloneExclude, + ).success + if not success: + r = self.GetRemote() + print( + "fatal: cannot obtain manifest %s" % r.url, file=sys.stderr + ) + + # Better delete the manifest git dir if we created it; otherwise + # next time (when user fixes problems) we won't go through the + # "is_new" logic. + if is_new: + platform_utils.rmtree(self.gitdir) + return False + + if manifest_branch: + self.MetaBranchSwitch(submodules=submodules) + + syncbuf = SyncBuffer(self.config) + self.Sync_LocalHalf(syncbuf, submodules=submodules) + syncbuf.Finish() + + if is_new or self.CurrentBranch is None: + if not self.StartBranch("default"): + print( + "fatal: cannot create default in manifest", + file=sys.stderr, + ) + return False + + if not manifest_name: + print("fatal: manifest name (-m) is required.", file=sys.stderr) + return False + + elif is_new: + # This is a new standalone manifest. + manifest_name = "default.xml" + manifest_data = fetch.fetch_file(manifest_url, verbose=verbose) + dest = os.path.join(self.worktree, manifest_name) + os.makedirs(os.path.dirname(dest), exist_ok=True) + with open(dest, "wb") as f: + f.write(manifest_data) + + try: + self.manifest.Link(manifest_name) + except ManifestParseError as e: + print( + "fatal: manifest '%s' not available" % manifest_name, + file=sys.stderr, + ) + print("fatal: %s" % str(e), file=sys.stderr) + return False + + if not this_manifest_only: + for submanifest in self.manifest.submanifests.values(): + spec = submanifest.ToSubmanifestSpec() + submanifest.repo_client.manifestProject.Sync( + manifest_url=spec.manifestUrl, + manifest_branch=spec.revision, + standalone_manifest=standalone_manifest, + groups=self.manifest_groups, + platform=platform, + mirror=mirror, + dissociate=dissociate, + reference=reference, + worktree=worktree, + submodules=submodules, + archive=archive, + partial_clone=partial_clone, + clone_filter=clone_filter, + partial_clone_exclude=partial_clone_exclude, + clone_bundle=clone_bundle, + git_lfs=git_lfs, + use_superproject=use_superproject, + verbose=verbose, + current_branch_only=current_branch_only, + tags=tags, + depth=depth, + git_event_log=git_event_log, + manifest_name=spec.manifestName, + this_manifest_only=False, + outer_manifest=False, + ) + + # Lastly, if the manifest has a then have the + # superproject sync it (if it will be used). + if git_superproject.UseSuperproject(use_superproject, self.manifest): + sync_result = self.manifest.superproject.Sync(git_event_log) + if not sync_result.success: + submanifest = "" + if self.manifest.path_prefix: + submanifest = f"for {self.manifest.path_prefix} " + print( + f"warning: git update of superproject {submanifest}failed, " + "repo sync will not use superproject to fetch source; " + "while this error is not fatal, and you can continue to " + "run repo sync, please run repo init with the " + "--no-use-superproject option to stop seeing this warning", + file=sys.stderr, + ) + if sync_result.fatal and use_superproject is not None: + return False + + return True + + def _ConfigureDepth(self, depth): + """Configure the depth we'll sync down. + + Args: + depth: an int, how deep of a partial clone to create. + """ + # Opt.depth will be non-None if user actually passed --depth to repo + # init. + if depth is not None: + if depth > 0: + # Positive values will set the depth. + depth = str(depth) + else: + # Negative numbers will clear the depth; passing None to + # SetString will do that. + depth = None + + # We store the depth in the main manifest project. + self.config.SetString("repo.depth", depth) -- cgit v1.2.3-54-g00ecf