From ea2e330e43c182dc16b0111ebc69ee5a71ee4ce1 Mon Sep 17 00:00:00 2001 From: Gavin Mak Date: Sat, 11 Mar 2023 06:46:20 +0000 Subject: Format codebase with black and check formatting in CQ Apply rules set by https://gerrit-review.googlesource.com/c/git-repo/+/362954/ across the codebase and fix any lingering errors caught by flake8. Also check black formatting in run_tests (and CQ). Bug: b/267675342 Change-Id: I972d77649dac351150dcfeb1cd1ad0ea2efc1956 Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/363474 Reviewed-by: Mike Frysinger Tested-by: Gavin Mak Commit-Queue: Gavin Mak --- subcmds/__init__.py | 50 +- subcmds/abandon.py | 167 +-- subcmds/branches.py | 287 ++--- subcmds/checkout.py | 83 +- subcmds/cherry_pick.py | 177 +-- subcmds/diff.py | 85 +- subcmds/diffmanifests.py | 374 +++--- subcmds/download.py | 304 +++-- subcmds/forall.py | 516 ++++---- subcmds/gitc_delete.py | 42 +- subcmds/gitc_init.py | 69 +- subcmds/grep.py | 529 ++++---- subcmds/help.py | 330 ++--- subcmds/info.py | 413 +++--- subcmds/init.py | 544 ++++---- subcmds/list.py | 158 ++- subcmds/manifest.py | 230 ++-- subcmds/overview.py | 124 +- subcmds/prune.py | 107 +- subcmds/rebase.py | 313 +++-- subcmds/selfupdate.py | 63 +- subcmds/smartsync.py | 18 +- subcmds/stage.py | 181 +-- subcmds/start.py | 250 ++-- subcmds/status.py | 239 ++-- subcmds/sync.py | 3130 +++++++++++++++++++++++++--------------------- subcmds/upload.py | 1152 +++++++++-------- subcmds/version.py | 75 +- 28 files changed, 5584 insertions(+), 4426 deletions(-) (limited to 'subcmds') diff --git a/subcmds/__init__.py b/subcmds/__init__.py index 051dda06..4e41afc0 100644 --- a/subcmds/__init__.py +++ b/subcmds/__init__.py @@ -19,31 +19,29 @@ all_commands = {} my_dir = os.path.dirname(__file__) for py in os.listdir(my_dir): - if py == '__init__.py': - continue - - if py.endswith('.py'): - name = py[:-3] - - clsn = name.capitalize() - while clsn.find('_') > 0: - h = clsn.index('_') - clsn = clsn[0:h] + clsn[h + 1:].capitalize() - - mod = __import__(__name__, - globals(), - locals(), - ['%s' % name]) - mod = getattr(mod, name) - try: - cmd = getattr(mod, clsn) - except AttributeError: - raise SyntaxError('%s/%s does not define class %s' % ( - __name__, py, clsn)) - - name = name.replace('_', '-') - cmd.NAME = name - all_commands[name] = cmd + if py == "__init__.py": + continue + + if py.endswith(".py"): + name = py[:-3] + + clsn = name.capitalize() + while clsn.find("_") > 0: + h = clsn.index("_") + clsn = clsn[0:h] + clsn[h + 1 :].capitalize() + + mod = __import__(__name__, globals(), locals(), ["%s" % name]) + mod = getattr(mod, name) + try: + cmd = getattr(mod, clsn) + except AttributeError: + raise SyntaxError( + "%s/%s does not define class %s" % (__name__, py, clsn) + ) + + name = name.replace("_", "-") + cmd.NAME = name + all_commands[name] = cmd # Add 'branch' as an alias for 'branches'. -all_commands['branch'] = all_commands['branches'] +all_commands["branch"] = all_commands["branches"] diff --git a/subcmds/abandon.py b/subcmds/abandon.py index c3d2d5b7..1f687f53 100644 --- a/subcmds/abandon.py +++ b/subcmds/abandon.py @@ -23,9 +23,9 @@ from progress import Progress class Abandon(Command): - COMMON = True - helpSummary = "Permanently abandon a development branch" - helpUsage = """ + COMMON = True + helpSummary = "Permanently abandon a development branch" + helpUsage = """ %prog [--all | ] [...] This subcommand permanently abandons a development branch by @@ -33,83 +33,104 @@ deleting it (and all its history) from your local repository. It is equivalent to "git branch -D ". """ - PARALLEL_JOBS = DEFAULT_LOCAL_JOBS + PARALLEL_JOBS = DEFAULT_LOCAL_JOBS - def _Options(self, p): - p.add_option('--all', - dest='all', action='store_true', - help='delete all branches in all projects') + def _Options(self, p): + p.add_option( + "--all", + dest="all", + action="store_true", + help="delete all branches in all projects", + ) - def ValidateOptions(self, opt, args): - if not opt.all and not args: - self.Usage() + def ValidateOptions(self, opt, args): + if not opt.all and not args: + self.Usage() - if not opt.all: - nb = args[0] - if not git.check_ref_format('heads/%s' % nb): - self.OptionParser.error("'%s' is not a valid branch name" % nb) - else: - args.insert(0, "'All local branches'") + if not opt.all: + nb = args[0] + if not git.check_ref_format("heads/%s" % nb): + self.OptionParser.error("'%s' is not a valid branch name" % nb) + else: + args.insert(0, "'All local branches'") - def _ExecuteOne(self, all_branches, nb, project): - """Abandon one project.""" - if all_branches: - branches = project.GetBranches() - else: - branches = [nb] + def _ExecuteOne(self, all_branches, nb, project): + """Abandon one project.""" + if all_branches: + branches = project.GetBranches() + else: + branches = [nb] - ret = {} - for name in branches: - status = project.AbandonBranch(name) - if status is not None: - ret[name] = status - return (ret, project) + ret = {} + for name in branches: + status = project.AbandonBranch(name) + if status is not None: + ret[name] = status + return (ret, project) - def Execute(self, opt, args): - nb = args[0] - err = defaultdict(list) - success = defaultdict(list) - all_projects = self.GetProjects(args[1:], all_manifests=not opt.this_manifest_only) - _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only) + def Execute(self, opt, args): + nb = args[0] + err = defaultdict(list) + success = defaultdict(list) + all_projects = self.GetProjects( + args[1:], all_manifests=not opt.this_manifest_only + ) + _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only) - def _ProcessResults(_pool, pm, states): - for (results, project) in states: - for branch, status in results.items(): - if status: - success[branch].append(project) - else: - err[branch].append(project) - pm.update() + def _ProcessResults(_pool, pm, states): + for results, project in states: + for branch, status in results.items(): + if status: + success[branch].append(project) + else: + err[branch].append(project) + pm.update() - self.ExecuteInParallel( - opt.jobs, - functools.partial(self._ExecuteOne, opt.all, nb), - all_projects, - callback=_ProcessResults, - output=Progress('Abandon %s' % (nb,), len(all_projects), quiet=opt.quiet)) + self.ExecuteInParallel( + opt.jobs, + functools.partial(self._ExecuteOne, opt.all, nb), + all_projects, + callback=_ProcessResults, + output=Progress( + "Abandon %s" % (nb,), len(all_projects), quiet=opt.quiet + ), + ) - width = max(itertools.chain( - [25], (len(x) for x in itertools.chain(success, err)))) - if err: - for br in err.keys(): - err_msg = "error: cannot abandon %s" % br - print(err_msg, file=sys.stderr) - for proj in err[br]: - print(' ' * len(err_msg) + " | %s" % _RelPath(proj), file=sys.stderr) - sys.exit(1) - elif not success: - print('error: no project has local branch(es) : %s' % nb, - file=sys.stderr) - sys.exit(1) - else: - # Everything below here is displaying status. - if opt.quiet: - return - print('Abandoned branches:') - for br in success.keys(): - if len(all_projects) > 1 and len(all_projects) == len(success[br]): - result = "all project" + width = max( + itertools.chain( + [25], (len(x) for x in itertools.chain(success, err)) + ) + ) + if err: + for br in err.keys(): + err_msg = "error: cannot abandon %s" % br + print(err_msg, file=sys.stderr) + for proj in err[br]: + print( + " " * len(err_msg) + " | %s" % _RelPath(proj), + file=sys.stderr, + ) + sys.exit(1) + elif not success: + print( + "error: no project has local branch(es) : %s" % nb, + file=sys.stderr, + ) + sys.exit(1) else: - result = "%s" % ( - ('\n' + ' ' * width + '| ').join(_RelPath(p) for p in success[br])) - print("%s%s| %s\n" % (br, ' ' * (width - len(br)), result)) + # Everything below here is displaying status. + if opt.quiet: + return + print("Abandoned branches:") + for br in success.keys(): + if len(all_projects) > 1 and len(all_projects) == len( + success[br] + ): + result = "all project" + else: + result = "%s" % ( + ("\n" + " " * width + "| ").join( + _RelPath(p) for p in success[br] + ) + ) + print("%s%s| %s\n" % (br, " " * (width - len(br)), result)) diff --git a/subcmds/branches.py b/subcmds/branches.py index fcf67ef5..4d5bb196 100644 --- a/subcmds/branches.py +++ b/subcmds/branches.py @@ -20,51 +20,51 @@ from command import Command, DEFAULT_LOCAL_JOBS class BranchColoring(Coloring): - def __init__(self, config): - Coloring.__init__(self, config, 'branch') - self.current = self.printer('current', fg='green') - self.local = self.printer('local') - self.notinproject = self.printer('notinproject', fg='red') + def __init__(self, config): + Coloring.__init__(self, config, "branch") + self.current = self.printer("current", fg="green") + self.local = self.printer("local") + self.notinproject = self.printer("notinproject", fg="red") class BranchInfo(object): - def __init__(self, name): - self.name = name - self.current = 0 - self.published = 0 - self.published_equal = 0 - self.projects = [] - - def add(self, b): - if b.current: - self.current += 1 - if b.published: - self.published += 1 - if b.revision == b.published: - self.published_equal += 1 - self.projects.append(b) - - @property - def IsCurrent(self): - return self.current > 0 - - @property - def IsSplitCurrent(self): - return self.current != 0 and self.current != len(self.projects) - - @property - def IsPublished(self): - return self.published > 0 - - @property - def IsPublishedEqual(self): - return self.published_equal == len(self.projects) + def __init__(self, name): + self.name = name + self.current = 0 + self.published = 0 + self.published_equal = 0 + self.projects = [] + + def add(self, b): + if b.current: + self.current += 1 + if b.published: + self.published += 1 + if b.revision == b.published: + self.published_equal += 1 + self.projects.append(b) + + @property + def IsCurrent(self): + return self.current > 0 + + @property + def IsSplitCurrent(self): + return self.current != 0 and self.current != len(self.projects) + + @property + def IsPublished(self): + return self.published > 0 + + @property + def IsPublishedEqual(self): + return self.published_equal == len(self.projects) class Branches(Command): - COMMON = True - helpSummary = "View current topic branches" - helpUsage = """ + COMMON = True + helpSummary = "View current topic branches" + helpUsage = """ %prog [...] Summarizes the currently available topic branches. @@ -95,111 +95,114 @@ the branch appears in, or does not appear in. If no project list is shown, then the branch appears in all projects. """ - PARALLEL_JOBS = DEFAULT_LOCAL_JOBS - - def Execute(self, opt, args): - projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only) - out = BranchColoring(self.manifest.manifestProject.config) - all_branches = {} - project_cnt = len(projects) - - def _ProcessResults(_pool, _output, results): - for name, b in itertools.chain.from_iterable(results): - if name not in all_branches: - all_branches[name] = BranchInfo(name) - all_branches[name].add(b) - - self.ExecuteInParallel( - opt.jobs, - expand_project_to_branches, - projects, - callback=_ProcessResults) - - names = sorted(all_branches) - - if not names: - print(' (no branches)', file=sys.stderr) - return - - width = 25 - for name in names: - if width < len(name): - width = len(name) - - for name in names: - i = all_branches[name] - in_cnt = len(i.projects) - - if i.IsCurrent: - current = '*' - hdr = out.current - else: - current = ' ' - hdr = out.local - - if i.IsPublishedEqual: - published = 'P' - elif i.IsPublished: - published = 'p' - else: - published = ' ' - - hdr('%c%c %-*s' % (current, published, width, name)) - out.write(' |') - - _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only) - if in_cnt < project_cnt: - fmt = out.write - paths = [] - non_cur_paths = [] - if i.IsSplitCurrent or (in_cnt <= project_cnt - in_cnt): - in_type = 'in' - for b in i.projects: - relpath = _RelPath(b.project) - if not i.IsSplitCurrent or b.current: - paths.append(relpath) + PARALLEL_JOBS = DEFAULT_LOCAL_JOBS + + def Execute(self, opt, args): + projects = self.GetProjects( + args, all_manifests=not opt.this_manifest_only + ) + out = BranchColoring(self.manifest.manifestProject.config) + all_branches = {} + project_cnt = len(projects) + + def _ProcessResults(_pool, _output, results): + for name, b in itertools.chain.from_iterable(results): + if name not in all_branches: + all_branches[name] = BranchInfo(name) + all_branches[name].add(b) + + self.ExecuteInParallel( + opt.jobs, + expand_project_to_branches, + projects, + callback=_ProcessResults, + ) + + names = sorted(all_branches) + + if not names: + print(" (no branches)", file=sys.stderr) + return + + width = 25 + for name in names: + if width < len(name): + width = len(name) + + for name in names: + i = all_branches[name] + in_cnt = len(i.projects) + + if i.IsCurrent: + current = "*" + hdr = out.current else: - non_cur_paths.append(relpath) - else: - fmt = out.notinproject - in_type = 'not in' - have = set() - for b in i.projects: - have.add(_RelPath(b.project)) - for p in projects: - if _RelPath(p) not in have: - paths.append(_RelPath(p)) - - s = ' %s %s' % (in_type, ', '.join(paths)) - if not i.IsSplitCurrent and (width + 7 + len(s) < 80): - fmt = out.current if i.IsCurrent else fmt - fmt(s) - else: - fmt(' %s:' % in_type) - fmt = out.current if i.IsCurrent else out.write - for p in paths: - out.nl() - fmt(width * ' ' + ' %s' % p) - fmt = out.write - for p in non_cur_paths: + current = " " + hdr = out.local + + if i.IsPublishedEqual: + published = "P" + elif i.IsPublished: + published = "p" + else: + published = " " + + hdr("%c%c %-*s" % (current, published, width, name)) + out.write(" |") + + _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only) + if in_cnt < project_cnt: + fmt = out.write + paths = [] + non_cur_paths = [] + if i.IsSplitCurrent or (in_cnt <= project_cnt - in_cnt): + in_type = "in" + for b in i.projects: + relpath = _RelPath(b.project) + if not i.IsSplitCurrent or b.current: + paths.append(relpath) + else: + non_cur_paths.append(relpath) + else: + fmt = out.notinproject + in_type = "not in" + have = set() + for b in i.projects: + have.add(_RelPath(b.project)) + for p in projects: + if _RelPath(p) not in have: + paths.append(_RelPath(p)) + + s = " %s %s" % (in_type, ", ".join(paths)) + if not i.IsSplitCurrent and (width + 7 + len(s) < 80): + fmt = out.current if i.IsCurrent else fmt + fmt(s) + else: + fmt(" %s:" % in_type) + fmt = out.current if i.IsCurrent else out.write + for p in paths: + out.nl() + fmt(width * " " + " %s" % p) + fmt = out.write + for p in non_cur_paths: + out.nl() + fmt(width * " " + " %s" % p) + else: + out.write(" in all projects") out.nl() - fmt(width * ' ' + ' %s' % p) - else: - out.write(' in all projects') - out.nl() def expand_project_to_branches(project): - """Expands a project into a list of branch names & associated information. - - Args: - project: project.Project - - Returns: - List[Tuple[str, git_config.Branch]] - """ - branches = [] - for name, b in project.GetBranches().items(): - b.project = project - branches.append((name, b)) - return branches + """Expands a project into a list of branch names & associated information. + + Args: + project: project.Project + + Returns: + List[Tuple[str, git_config.Branch]] + """ + branches = [] + for name, b in project.GetBranches().items(): + b.project = project + branches.append((name, b)) + return branches diff --git a/subcmds/checkout.py b/subcmds/checkout.py index 768b6027..08012a82 100644 --- a/subcmds/checkout.py +++ b/subcmds/checkout.py @@ -20,12 +20,12 @@ from progress import Progress class Checkout(Command): - COMMON = True - helpSummary = "Checkout a branch for development" - helpUsage = """ + COMMON = True + helpSummary = "Checkout a branch for development" + helpUsage = """ %prog [...] """ - helpDescription = """ + helpDescription = """ The '%prog' command checks out an existing branch that was previously created by 'repo start'. @@ -33,43 +33,50 @@ The command is equivalent to: repo forall [...] -c git checkout """ - PARALLEL_JOBS = DEFAULT_LOCAL_JOBS + PARALLEL_JOBS = DEFAULT_LOCAL_JOBS - def ValidateOptions(self, opt, args): - if not args: - self.Usage() + def ValidateOptions(self, opt, args): + if not args: + self.Usage() - def _ExecuteOne(self, nb, project): - """Checkout one project.""" - return (project.CheckoutBranch(nb), project) + def _ExecuteOne(self, nb, project): + """Checkout one project.""" + return (project.CheckoutBranch(nb), project) - def Execute(self, opt, args): - nb = args[0] - err = [] - success = [] - all_projects = self.GetProjects(args[1:], all_manifests=not opt.this_manifest_only) + def Execute(self, opt, args): + nb = args[0] + err = [] + success = [] + all_projects = self.GetProjects( + args[1:], all_manifests=not opt.this_manifest_only + ) - def _ProcessResults(_pool, pm, results): - for status, project in results: - if status is not None: - if status: - success.append(project) - else: - err.append(project) - pm.update() + def _ProcessResults(_pool, pm, results): + for status, project in results: + if status is not None: + if status: + success.append(project) + else: + err.append(project) + pm.update() - self.ExecuteInParallel( - opt.jobs, - functools.partial(self._ExecuteOne, nb), - all_projects, - callback=_ProcessResults, - output=Progress('Checkout %s' % (nb,), len(all_projects), quiet=opt.quiet)) + self.ExecuteInParallel( + opt.jobs, + functools.partial(self._ExecuteOne, nb), + all_projects, + callback=_ProcessResults, + output=Progress( + "Checkout %s" % (nb,), len(all_projects), quiet=opt.quiet + ), + ) - if err: - for p in err: - print("error: %s/: cannot checkout %s" % (p.relpath, nb), - file=sys.stderr) - sys.exit(1) - elif not success: - print('error: no project has branch %s' % nb, file=sys.stderr) - sys.exit(1) + if err: + for p in err: + print( + "error: %s/: cannot checkout %s" % (p.relpath, nb), + file=sys.stderr, + ) + sys.exit(1) + elif not success: + print("error: no project has branch %s" % nb, file=sys.stderr) + sys.exit(1) diff --git a/subcmds/cherry_pick.py b/subcmds/cherry_pick.py index eecf4e17..4cfb8c88 100644 --- a/subcmds/cherry_pick.py +++ b/subcmds/cherry_pick.py @@ -17,96 +17,107 @@ import sys from command import Command from git_command import GitCommand -CHANGE_ID_RE = re.compile(r'^\s*Change-Id: I([0-9a-f]{40})\s*$') +CHANGE_ID_RE = re.compile(r"^\s*Change-Id: I([0-9a-f]{40})\s*$") class CherryPick(Command): - COMMON = True - helpSummary = "Cherry-pick a change." - helpUsage = """ + COMMON = True + helpSummary = "Cherry-pick a change." + helpUsage = """ %prog """ - helpDescription = """ + helpDescription = """ '%prog' cherry-picks a change from one branch to another. The change id will be updated, and a reference to the old change id will be added. """ - def ValidateOptions(self, opt, args): - if len(args) != 1: - self.Usage() - - def Execute(self, opt, args): - reference = args[0] - - p = GitCommand(None, - ['rev-parse', '--verify', reference], - capture_stdout=True, - capture_stderr=True) - if p.Wait() != 0: - print(p.stderr, file=sys.stderr) - sys.exit(1) - sha1 = p.stdout.strip() - - p = GitCommand(None, ['cat-file', 'commit', sha1], capture_stdout=True) - if p.Wait() != 0: - print("error: Failed to retrieve old commit message", file=sys.stderr) - sys.exit(1) - old_msg = self._StripHeader(p.stdout) - - p = GitCommand(None, - ['cherry-pick', sha1], - capture_stdout=True, - capture_stderr=True) - status = p.Wait() - - if p.stdout: - print(p.stdout.strip(), file=sys.stdout) - if p.stderr: - print(p.stderr.strip(), file=sys.stderr) - - if status == 0: - # The cherry-pick was applied correctly. We just need to edit the - # commit message. - new_msg = self._Reformat(old_msg, sha1) - - p = GitCommand(None, ['commit', '--amend', '-F', '-'], - input=new_msg, - capture_stdout=True, - capture_stderr=True) - if p.Wait() != 0: - print("error: Failed to update commit message", file=sys.stderr) - sys.exit(1) - - else: - print('NOTE: When committing (please see above) and editing the commit ' - 'message, please remove the old Change-Id-line and add:') - print(self._GetReference(sha1), file=sys.stderr) - print(file=sys.stderr) - - def _IsChangeId(self, line): - return CHANGE_ID_RE.match(line) - - def _GetReference(self, sha1): - return "(cherry picked from commit %s)" % sha1 - - def _StripHeader(self, commit_msg): - lines = commit_msg.splitlines() - return "\n".join(lines[lines.index("") + 1:]) - - def _Reformat(self, old_msg, sha1): - new_msg = [] - - for line in old_msg.splitlines(): - if not self._IsChangeId(line): - new_msg.append(line) - - # Add a blank line between the message and the change id/reference - try: - if new_msg[-1].strip() != "": - new_msg.append("") - except IndexError: - pass - - new_msg.append(self._GetReference(sha1)) - return "\n".join(new_msg) + def ValidateOptions(self, opt, args): + if len(args) != 1: + self.Usage() + + def Execute(self, opt, args): + reference = args[0] + + p = GitCommand( + None, + ["rev-parse", "--verify", reference], + capture_stdout=True, + capture_stderr=True, + ) + if p.Wait() != 0: + print(p.stderr, file=sys.stderr) + sys.exit(1) + sha1 = p.stdout.strip() + + p = GitCommand(None, ["cat-file", "commit", sha1], capture_stdout=True) + if p.Wait() != 0: + print( + "error: Failed to retrieve old commit message", file=sys.stderr + ) + sys.exit(1) + old_msg = self._StripHeader(p.stdout) + + p = GitCommand( + None, + ["cherry-pick", sha1], + capture_stdout=True, + capture_stderr=True, + ) + status = p.Wait() + + if p.stdout: + print(p.stdout.strip(), file=sys.stdout) + if p.stderr: + print(p.stderr.strip(), file=sys.stderr) + + if status == 0: + # The cherry-pick was applied correctly. We just need to edit the + # commit message. + new_msg = self._Reformat(old_msg, sha1) + + p = GitCommand( + None, + ["commit", "--amend", "-F", "-"], + input=new_msg, + capture_stdout=True, + capture_stderr=True, + ) + if p.Wait() != 0: + print("error: Failed to update commit message", file=sys.stderr) + sys.exit(1) + + else: + print( + "NOTE: When committing (please see above) and editing the " + "commit message, please remove the old Change-Id-line and add:" + ) + print(self._GetReference(sha1), file=sys.stderr) + print(file=sys.stderr) + + def _IsChangeId(self, line): + return CHANGE_ID_RE.match(line) + + def _GetReference(self, sha1): + return "(cherry picked from commit %s)" % sha1 + + def _StripHeader(self, commit_msg): + lines = commit_msg.splitlines() + return "\n".join(lines[lines.index("") + 1 :]) + + def _Reformat(self, old_msg, sha1): + new_msg = [] + + for line in old_msg.splitlines(): + if not self._IsChangeId(line): + new_msg.append(line) + + # Add a blank line between the message and the change id/reference. + try: + if new_msg[-1].strip() != "": + new_msg.append("") + except IndexError: + pass + + new_msg.append(self._GetReference(sha1)) + return "\n".join(new_msg) diff --git a/subcmds/diff.py b/subcmds/diff.py index a606ee9a..5c627c0c 100644 --- a/subcmds/diff.py +++ b/subcmds/diff.py @@ -19,54 +19,63 @@ from command import DEFAULT_LOCAL_JOBS, PagedCommand class Diff(PagedCommand): - COMMON = True - helpSummary = "Show changes between commit and working tree" - helpUsage = """ + COMMON = True + helpSummary = "Show changes between commit and working tree" + helpUsage = """ %prog [...] The -u option causes '%prog' to generate diff output with file paths relative to the repository root, so the output can be applied to the Unix 'patch' command. """ - PARALLEL_JOBS = DEFAULT_LOCAL_JOBS + PARALLEL_JOBS = DEFAULT_LOCAL_JOBS - def _Options(self, p): - p.add_option('-u', '--absolute', - dest='absolute', action='store_true', - help='paths are relative to the repository root') + def _Options(self, p): + p.add_option( + "-u", + "--absolute", + dest="absolute", + action="store_true", + help="paths are relative to the repository root", + ) - def _ExecuteOne(self, absolute, local, project): - """Obtains the diff for a specific project. + def _ExecuteOne(self, absolute, local, project): + """Obtains the diff for a specific project. - Args: - absolute: Paths are relative to the root. - local: a boolean, if True, the path is relative to the local - (sub)manifest. If false, the path is relative to the - outermost manifest. - project: Project to get status of. + Args: + absolute: Paths are relative to the root. + local: a boolean, if True, the path is relative to the local + (sub)manifest. If false, the path is relative to the outermost + manifest. + project: Project to get status of. - Returns: - The status of the project. - """ - buf = io.StringIO() - ret = project.PrintWorkTreeDiff(absolute, output_redir=buf, local=local) - return (ret, buf.getvalue()) + Returns: + The status of the project. + """ + buf = io.StringIO() + ret = project.PrintWorkTreeDiff(absolute, output_redir=buf, local=local) + return (ret, buf.getvalue()) - def Execute(self, opt, args): - all_projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only) + def Execute(self, opt, args): + all_projects = self.GetProjects( + args, all_manifests=not opt.this_manifest_only + ) - def _ProcessResults(_pool, _output, results): - ret = 0 - for (state, output) in results: - if output: - print(output, end='') - if not state: - ret = 1 - return ret + def _ProcessResults(_pool, _output, results): + ret = 0 + for state, output in results: + if output: + print(output, end="") + if not state: + ret = 1 + return ret - return self.ExecuteInParallel( - opt.jobs, - functools.partial(self._ExecuteOne, opt.absolute, opt.this_manifest_only), - all_projects, - callback=_ProcessResults, - ordered=True) + return self.ExecuteInParallel( + opt.jobs, + functools.partial( + self._ExecuteOne, opt.absolute, opt.this_manifest_only + ), + all_projects, + callback=_ProcessResults, + ordered=True, + ) diff --git a/subcmds/diffmanifests.py b/subcmds/diffmanifests.py index 4f9f5b0f..b446dbd8 100644 --- a/subcmds/diffmanifests.py +++ b/subcmds/diffmanifests.py @@ -18,24 +18,24 @@ from manifest_xml import RepoClient class _Coloring(Coloring): - def __init__(self, config): - Coloring.__init__(self, config, "status") + def __init__(self, config): + Coloring.__init__(self, config, "status") class Diffmanifests(PagedCommand): - """ A command to see logs in projects represented by manifests + """A command to see logs in projects represented by manifests - This is used to see deeper differences between manifests. Where a simple - diff would only show a diff of sha1s for example, this command will display - the logs of the project between both sha1s, allowing user to see diff at a - deeper level. - """ + This is used to see deeper differences between manifests. Where a simple + diff would only show a diff of sha1s for example, this command will display + the logs of the project between both sha1s, allowing user to see diff at a + deeper level. + """ - COMMON = True - helpSummary = "Manifest diff utility" - helpUsage = """%prog manifest1.xml [manifest2.xml] [options]""" + COMMON = True + helpSummary = "Manifest diff utility" + helpUsage = """%prog manifest1.xml [manifest2.xml] [options]""" - helpDescription = """ + helpDescription = """ The %prog command shows differences between project revisions of manifest1 and manifest2. if manifest2 is not specified, current manifest.xml will be used instead. Both absolute and relative paths may be used for manifests. Relative @@ -65,159 +65,209 @@ synced and their revisions won't be found. """ - def _Options(self, p): - p.add_option('--raw', - dest='raw', action='store_true', - help='display raw diff') - p.add_option('--no-color', - dest='color', action='store_false', default=True, - help='does not display the diff in color') - p.add_option('--pretty-format', - dest='pretty_format', action='store', - metavar='', - help='print the log using a custom git pretty format string') - - def _printRawDiff(self, diff, pretty_format=None, local=False): - _RelPath = lambda p: p.RelPath(local=local) - for project in diff['added']: - self.printText("A %s %s" % (_RelPath(project), project.revisionExpr)) - self.out.nl() - - for project in diff['removed']: - self.printText("R %s %s" % (_RelPath(project), project.revisionExpr)) - self.out.nl() - - for project, otherProject in diff['changed']: - self.printText("C %s %s %s" % (_RelPath(project), project.revisionExpr, - otherProject.revisionExpr)) - self.out.nl() - self._printLogs(project, otherProject, raw=True, color=False, pretty_format=pretty_format) - - for project, otherProject in diff['unreachable']: - self.printText("U %s %s %s" % (_RelPath(project), project.revisionExpr, - otherProject.revisionExpr)) - self.out.nl() - - def _printDiff(self, diff, color=True, pretty_format=None, local=False): - _RelPath = lambda p: p.RelPath(local=local) - if diff['added']: - self.out.nl() - self.printText('added projects : \n') - self.out.nl() - for project in diff['added']: - self.printProject('\t%s' % (_RelPath(project))) - self.printText(' at revision ') - self.printRevision(project.revisionExpr) - self.out.nl() - - if diff['removed']: - self.out.nl() - self.printText('removed projects : \n') - self.out.nl() - for project in diff['removed']: - self.printProject('\t%s' % (_RelPath(project))) - self.printText(' at revision ') - self.printRevision(project.revisionExpr) - self.out.nl() - - if diff['missing']: - self.out.nl() - self.printText('missing projects : \n') - self.out.nl() - for project in diff['missing']: - self.printProject('\t%s' % (_RelPath(project))) - self.printText(' at revision ') - self.printRevision(project.revisionExpr) - self.out.nl() - - if diff['changed']: - self.out.nl() - self.printText('changed projects : \n') - self.out.nl() - for project, otherProject in diff['changed']: - self.printProject('\t%s' % (_RelPath(project))) - self.printText(' changed from ') - self.printRevision(project.revisionExpr) - self.printText(' to ') - self.printRevision(otherProject.revisionExpr) - self.out.nl() - self._printLogs(project, otherProject, raw=False, color=color, - pretty_format=pretty_format) - self.out.nl() - - if diff['unreachable']: - self.out.nl() - self.printText('projects with unreachable revisions : \n') - self.out.nl() - for project, otherProject in diff['unreachable']: - self.printProject('\t%s ' % (_RelPath(project))) - self.printRevision(project.revisionExpr) - self.printText(' or ') - self.printRevision(otherProject.revisionExpr) - self.printText(' not found') - self.out.nl() - - def _printLogs(self, project, otherProject, raw=False, color=True, - pretty_format=None): - - logs = project.getAddedAndRemovedLogs(otherProject, - oneline=(pretty_format is None), - color=color, - pretty_format=pretty_format) - if logs['removed']: - removedLogs = logs['removed'].split('\n') - for log in removedLogs: - if log.strip(): - if raw: - self.printText(' R ' + log) + def _Options(self, p): + p.add_option( + "--raw", dest="raw", action="store_true", help="display raw diff" + ) + p.add_option( + "--no-color", + dest="color", + action="store_false", + default=True, + help="does not display the diff in color", + ) + p.add_option( + "--pretty-format", + dest="pretty_format", + action="store", + metavar="", + help="print the log using a custom git pretty format string", + ) + + def _printRawDiff(self, diff, pretty_format=None, local=False): + _RelPath = lambda p: p.RelPath(local=local) + for project in diff["added"]: + self.printText( + "A %s %s" % (_RelPath(project), project.revisionExpr) + ) self.out.nl() - else: - self.printRemoved('\t\t[-] ') - self.printText(log) + + for project in diff["removed"]: + self.printText( + "R %s %s" % (_RelPath(project), project.revisionExpr) + ) + self.out.nl() + + for project, otherProject in diff["changed"]: + self.printText( + "C %s %s %s" + % ( + _RelPath(project), + project.revisionExpr, + otherProject.revisionExpr, + ) + ) + self.out.nl() + self._printLogs( + project, + otherProject, + raw=True, + color=False, + pretty_format=pretty_format, + ) + + for project, otherProject in diff["unreachable"]: + self.printText( + "U %s %s %s" + % ( + _RelPath(project), + project.revisionExpr, + otherProject.revisionExpr, + ) + ) + self.out.nl() + + def _printDiff(self, diff, color=True, pretty_format=None, local=False): + _RelPath = lambda p: p.RelPath(local=local) + if diff["added"]: + self.out.nl() + self.printText("added projects : \n") self.out.nl() + for project in diff["added"]: + self.printProject("\t%s" % (_RelPath(project))) + self.printText(" at revision ") + self.printRevision(project.revisionExpr) + self.out.nl() - if logs['added']: - addedLogs = logs['added'].split('\n') - for log in addedLogs: - if log.strip(): - if raw: - self.printText(' A ' + log) + if diff["removed"]: self.out.nl() - else: - self.printAdded('\t\t[+] ') - self.printText(log) + self.printText("removed projects : \n") self.out.nl() + for project in diff["removed"]: + self.printProject("\t%s" % (_RelPath(project))) + self.printText(" at revision ") + self.printRevision(project.revisionExpr) + self.out.nl() - def ValidateOptions(self, opt, args): - if not args or len(args) > 2: - self.OptionParser.error('missing manifests to diff') - if opt.this_manifest_only is False: - raise self.OptionParser.error( - '`diffmanifest` only supports the current tree') - - def Execute(self, opt, args): - self.out = _Coloring(self.client.globalConfig) - self.printText = self.out.nofmt_printer('text') - if opt.color: - self.printProject = self.out.nofmt_printer('project', attr='bold') - self.printAdded = self.out.nofmt_printer('green', fg='green', attr='bold') - self.printRemoved = self.out.nofmt_printer('red', fg='red', attr='bold') - self.printRevision = self.out.nofmt_printer('revision', fg='yellow') - else: - self.printProject = self.printAdded = self.printRemoved = self.printRevision = self.printText - - manifest1 = RepoClient(self.repodir) - manifest1.Override(args[0], load_local_manifests=False) - if len(args) == 1: - manifest2 = self.manifest - else: - manifest2 = RepoClient(self.repodir) - manifest2.Override(args[1], load_local_manifests=False) - - diff = manifest1.projectsDiff(manifest2) - if opt.raw: - self._printRawDiff(diff, pretty_format=opt.pretty_format, - local=opt.this_manifest_only) - else: - self._printDiff(diff, color=opt.color, pretty_format=opt.pretty_format, - local=opt.this_manifest_only) + if diff["missing"]: + self.out.nl() + self.printText("missing projects : \n") + self.out.nl() + for project in diff["missing"]: + self.printProject("\t%s" % (_RelPath(project))) + self.printText(" at revision ") + self.printRevision(project.revisionExpr) + self.out.nl() + + if diff["changed"]: + self.out.nl() + self.printText("changed projects : \n") + self.out.nl() + for project, otherProject in diff["changed"]: + self.printProject("\t%s" % (_RelPath(project))) + self.printText(" changed from ") + self.printRevision(project.revisionExpr) + self.printText(" to ") + self.printRevision(otherProject.revisionExpr) + self.out.nl() + self._printLogs( + project, + otherProject, + raw=False, + color=color, + pretty_format=pretty_format, + ) + self.out.nl() + + if diff["unreachable"]: + self.out.nl() + self.printText("projects with unreachable revisions : \n") + self.out.nl() + for project, otherProject in diff["unreachable"]: + self.printProject("\t%s " % (_RelPath(project))) + self.printRevision(project.revisionExpr) + self.printText(" or ") + self.printRevision(otherProject.revisionExpr) + self.printText(" not found") + self.out.nl() + + def _printLogs( + self, project, otherProject, raw=False, color=True, pretty_format=None + ): + logs = project.getAddedAndRemovedLogs( + otherProject, + oneline=(pretty_format is None), + color=color, + pretty_format=pretty_format, + ) + if logs["removed"]: + removedLogs = logs["removed"].split("\n") + for log in removedLogs: + if log.strip(): + if raw: + self.printText(" R " + log) + self.out.nl() + else: + self.printRemoved("\t\t[-] ") + self.printText(log) + self.out.nl() + + if logs["added"]: + addedLogs = logs["added"].split("\n") + for log in addedLogs: + if log.strip(): + if raw: + self.printText(" A " + log) + self.out.nl() + else: + self.printAdded("\t\t[+] ") + self.printText(log) + self.out.nl() + + def ValidateOptions(self, opt, args): + if not args or len(args) > 2: + self.OptionParser.error("missing manifests to diff") + if opt.this_manifest_only is False: + raise self.OptionParser.error( + "`diffmanifest` only supports the current tree" + ) + + def Execute(self, opt, args): + self.out = _Coloring(self.client.globalConfig) + self.printText = self.out.nofmt_printer("text") + if opt.color: + self.printProject = self.out.nofmt_printer("project", attr="bold") + self.printAdded = self.out.nofmt_printer( + "green", fg="green", attr="bold" + ) + self.printRemoved = self.out.nofmt_printer( + "red", fg="red", attr="bold" + ) + self.printRevision = self.out.nofmt_printer("revision", fg="yellow") + else: + self.printProject = ( + self.printAdded + ) = self.printRemoved = self.printRevision = self.printText + + manifest1 = RepoClient(self.repodir) + manifest1.Override(args[0], load_local_manifests=False) + if len(args) == 1: + manifest2 = self.manifest + else: + manifest2 = RepoClient(self.repodir) + manifest2.Override(args[1], load_local_manifests=False) + + diff = manifest1.projectsDiff(manifest2) + if opt.raw: + self._printRawDiff( + diff, + pretty_format=opt.pretty_format, + local=opt.this_manifest_only, + ) + else: + self._printDiff( + diff, + color=opt.color, + pretty_format=opt.pretty_format, + local=opt.this_manifest_only, + ) diff --git a/subcmds/download.py b/subcmds/download.py index 15824843..d81d1f8c 100644 --- a/subcmds/download.py +++ b/subcmds/download.py @@ -18,143 +18,187 @@ import sys from command import Command from error import GitError, NoSuchProjectError -CHANGE_RE = re.compile(r'^([1-9][0-9]*)(?:[/\.-]([1-9][0-9]*))?$') +CHANGE_RE = re.compile(r"^([1-9][0-9]*)(?:[/\.-]([1-9][0-9]*))?$") class Download(Command): - COMMON = True - helpSummary = "Download and checkout a change" - helpUsage = """ + COMMON = True + helpSummary = "Download and checkout a change" + helpUsage = """ %prog {[project] change[/patchset]}... """ - helpDescription = """ + helpDescription = """ The '%prog' command downloads a change from the review system and makes it available in your project's local working directory. If no project is specified try to use current directory as a project. """ - def _Options(self, p): - p.add_option('-b', '--branch', - help='create a new branch first') - p.add_option('-c', '--cherry-pick', - dest='cherrypick', action='store_true', - help="cherry-pick instead of checkout") - p.add_option('-x', '--record-origin', action='store_true', - help='pass -x when cherry-picking') - p.add_option('-r', '--revert', - dest='revert', action='store_true', - help="revert instead of checkout") - p.add_option('-f', '--ff-only', - dest='ffonly', action='store_true', - help="force fast-forward merge") - - def _ParseChangeIds(self, opt, args): - if not args: - self.Usage() - - to_get = [] - project = None - - for a in args: - m = CHANGE_RE.match(a) - if m: - if not project: - project = self.GetProjects(".")[0] - print('Defaulting to cwd project', project.name) - chg_id = int(m.group(1)) - if m.group(2): - ps_id = int(m.group(2)) - else: - ps_id = 1 - refs = 'refs/changes/%2.2d/%d/' % (chg_id % 100, chg_id) - output = project._LsRemote(refs + '*') - if output: - regex = refs + r'(\d+)' - rcomp = re.compile(regex, re.I) - for line in output.splitlines(): - match = rcomp.search(line) - if match: - ps_id = max(int(match.group(1)), ps_id) - to_get.append((project, chg_id, ps_id)) - else: - projects = self.GetProjects([a], all_manifests=not opt.this_manifest_only) - if len(projects) > 1: - # If the cwd is one of the projects, assume they want that. - try: - project = self.GetProjects('.')[0] - except NoSuchProjectError: - project = None - if project not in projects: - print('error: %s matches too many projects; please re-run inside ' - 'the project checkout.' % (a,), file=sys.stderr) - for project in projects: - print(' %s/ @ %s' % (project.RelPath(local=opt.this_manifest_only), - project.revisionExpr), file=sys.stderr) - sys.exit(1) - else: - project = projects[0] - print('Defaulting to cwd project', project.name) - return to_get - - def ValidateOptions(self, opt, args): - if opt.record_origin: - if not opt.cherrypick: - self.OptionParser.error('-x only makes sense with --cherry-pick') - - if opt.ffonly: - self.OptionParser.error('-x and --ff are mutually exclusive options') - - def Execute(self, opt, args): - for project, change_id, ps_id in self._ParseChangeIds(opt, args): - dl = project.DownloadPatchSet(change_id, ps_id) - if not dl: - print('[%s] change %d/%d not found' - % (project.name, change_id, ps_id), - file=sys.stderr) - sys.exit(1) - - if not opt.revert and not dl.commits: - print('[%s] change %d/%d has already been merged' - % (project.name, change_id, ps_id), - file=sys.stderr) - continue - - if len(dl.commits) > 1: - print('[%s] %d/%d depends on %d unmerged changes:' - % (project.name, change_id, ps_id, len(dl.commits)), - file=sys.stderr) - for c in dl.commits: - print(' %s' % (c), file=sys.stderr) - - if opt.cherrypick: - mode = 'cherry-pick' - elif opt.revert: - mode = 'revert' - elif opt.ffonly: - mode = 'fast-forward merge' - else: - mode = 'checkout' - - # We'll combine the branch+checkout operation, but all the rest need a - # dedicated branch start. - if opt.branch and mode != 'checkout': - project.StartBranch(opt.branch) - - try: - if opt.cherrypick: - project._CherryPick(dl.commit, ffonly=opt.ffonly, - record_origin=opt.record_origin) - elif opt.revert: - project._Revert(dl.commit) - elif opt.ffonly: - project._FastForward(dl.commit, ffonly=True) - else: - if opt.branch: - project.StartBranch(opt.branch, revision=dl.commit) - else: - project._Checkout(dl.commit) - - except GitError: - print('[%s] Could not complete the %s of %s' - % (project.name, mode, dl.commit), file=sys.stderr) - sys.exit(1) + def _Options(self, p): + p.add_option("-b", "--branch", help="create a new branch first") + p.add_option( + "-c", + "--cherry-pick", + dest="cherrypick", + action="store_true", + help="cherry-pick instead of checkout", + ) + p.add_option( + "-x", + "--record-origin", + action="store_true", + help="pass -x when cherry-picking", + ) + p.add_option( + "-r", + "--revert", + dest="revert", + action="store_true", + help="revert instead of checkout", + ) + p.add_option( + "-f", + "--ff-only", + dest="ffonly", + action="store_true", + help="force fast-forward merge", + ) + + def _ParseChangeIds(self, opt, args): + if not args: + self.Usage() + + to_get = [] + project = None + + for a in args: + m = CHANGE_RE.match(a) + if m: + if not project: + project = self.GetProjects(".")[0] + print("Defaulting to cwd project", project.name) + chg_id = int(m.group(1)) + if m.group(2): + ps_id = int(m.group(2)) + else: + ps_id = 1 + refs = "refs/changes/%2.2d/%d/" % (chg_id % 100, chg_id) + output = project._LsRemote(refs + "*") + if output: + regex = refs + r"(\d+)" + rcomp = re.compile(regex, re.I) + for line in output.splitlines(): + match = rcomp.search(line) + if match: + ps_id = max(int(match.group(1)), ps_id) + to_get.append((project, chg_id, ps_id)) + else: + projects = self.GetProjects( + [a], all_manifests=not opt.this_manifest_only + ) + if len(projects) > 1: + # If the cwd is one of the projects, assume they want that. + try: + project = self.GetProjects(".")[0] + except NoSuchProjectError: + project = None + if project not in projects: + print( + "error: %s matches too many projects; please " + "re-run inside the project checkout." % (a,), + file=sys.stderr, + ) + for project in projects: + print( + " %s/ @ %s" + % ( + project.RelPath( + local=opt.this_manifest_only + ), + project.revisionExpr, + ), + file=sys.stderr, + ) + sys.exit(1) + else: + project = projects[0] + print("Defaulting to cwd project", project.name) + return to_get + + def ValidateOptions(self, opt, args): + if opt.record_origin: + if not opt.cherrypick: + self.OptionParser.error( + "-x only makes sense with --cherry-pick" + ) + + if opt.ffonly: + self.OptionParser.error( + "-x and --ff are mutually exclusive options" + ) + + def Execute(self, opt, args): + for project, change_id, ps_id in self._ParseChangeIds(opt, args): + dl = project.DownloadPatchSet(change_id, ps_id) + if not dl: + print( + "[%s] change %d/%d not found" + % (project.name, change_id, ps_id), + file=sys.stderr, + ) + sys.exit(1) + + if not opt.revert and not dl.commits: + print( + "[%s] change %d/%d has already been merged" + % (project.name, change_id, ps_id), + file=sys.stderr, + ) + continue + + if len(dl.commits) > 1: + print( + "[%s] %d/%d depends on %d unmerged changes:" + % (project.name, change_id, ps_id, len(dl.commits)), + file=sys.stderr, + ) + for c in dl.commits: + print(" %s" % (c), file=sys.stderr) + + if opt.cherrypick: + mode = "cherry-pick" + elif opt.revert: + mode = "revert" + elif opt.ffonly: + mode = "fast-forward merge" + else: + mode = "checkout" + + # We'll combine the branch+checkout operation, but all the rest need + # a dedicated branch start. + if opt.branch and mode != "checkout": + project.StartBranch(opt.branch) + + try: + if opt.cherrypick: + project._CherryPick( + dl.commit, + ffonly=opt.ffonly, + record_origin=opt.record_origin, + ) + elif opt.revert: + project._Revert(dl.commit) + elif opt.ffonly: + project._FastForward(dl.commit, ffonly=True) + else: + if opt.branch: + project.StartBranch(opt.branch, revision=dl.commit) + else: + project._Checkout(dl.commit) + + except GitError: + print( + "[%s] Could not complete the %s of %s" + % (project.name, mode, dl.commit), + file=sys.stderr, + ) + sys.exit(1) diff --git a/subcmds/forall.py b/subcmds/forall.py index f9f34e33..0a897357 100644 --- a/subcmds/forall.py +++ b/subcmds/forall.py @@ -23,31 +23,36 @@ import sys import subprocess from color import Coloring -from command import DEFAULT_LOCAL_JOBS, Command, MirrorSafeCommand, WORKER_BATCH_SIZE +from command import ( + DEFAULT_LOCAL_JOBS, + Command, + MirrorSafeCommand, + WORKER_BATCH_SIZE, +) from error import ManifestInvalidRevisionError _CAN_COLOR = [ - 'branch', - 'diff', - 'grep', - 'log', + "branch", + "diff", + "grep", + "log", ] class ForallColoring(Coloring): - def __init__(self, config): - Coloring.__init__(self, config, 'forall') - self.project = self.printer('project', attr='bold') + def __init__(self, config): + Coloring.__init__(self, config, "forall") + self.project = self.printer("project", attr="bold") class Forall(Command, MirrorSafeCommand): - COMMON = False - helpSummary = "Run a shell command in each project" - helpUsage = """ + COMMON = False + helpSummary = "Run a shell command in each project" + helpUsage = """ %prog [...] -c [...] %prog -r str1 [str2] ... -c [...] """ - helpDescription = """ + helpDescription = """ Executes the same shell command in each project. The -r option allows running the command only on projects matching @@ -125,236 +130,285 @@ terminal and are not redirected. If -e is used, when a command exits unsuccessfully, '%prog' will abort without iterating through the remaining projects. """ - PARALLEL_JOBS = DEFAULT_LOCAL_JOBS - - @staticmethod - def _cmd_option(option, _opt_str, _value, parser): - setattr(parser.values, option.dest, list(parser.rargs)) - while parser.rargs: - del parser.rargs[0] - - def _Options(self, p): - p.add_option('-r', '--regex', - dest='regex', action='store_true', - help='execute the command only on projects matching regex or wildcard expression') - p.add_option('-i', '--inverse-regex', - dest='inverse_regex', action='store_true', - help='execute the command only on projects not matching regex or ' - 'wildcard expression') - p.add_option('-g', '--groups', - dest='groups', - help='execute the command only on projects matching the specified groups') - p.add_option('-c', '--command', - help='command (and arguments) to execute', - dest='command', - action='callback', - callback=self._cmd_option) - p.add_option('-e', '--abort-on-errors', - dest='abort_on_errors', action='store_true', - help='abort if a command exits unsuccessfully') - p.add_option('--ignore-missing', action='store_true', - help='silently skip & do not exit non-zero due missing ' - 'checkouts') - - g = p.get_option_group('--quiet') - g.add_option('-p', - dest='project_header', action='store_true', - help='show project headers before output') - p.add_option('--interactive', - action='store_true', - help='force interactive usage') - - def WantPager(self, opt): - return opt.project_header and opt.jobs == 1 - - def ValidateOptions(self, opt, args): - if not opt.command: - self.Usage() - - def Execute(self, opt, args): - cmd = [opt.command[0]] - all_trees = not opt.this_manifest_only - - shell = True - if re.compile(r'^[a-z0-9A-Z_/\.-]+$').match(cmd[0]): - shell = False - - if shell: - cmd.append(cmd[0]) - cmd.extend(opt.command[1:]) - - # Historically, forall operated interactively, and in serial. If the user - # has selected 1 job, then default to interacive mode. - if opt.jobs == 1: - opt.interactive = True - - if opt.project_header \ - and not shell \ - and cmd[0] == 'git': - # If this is a direct git command that can enable colorized - # output and the user prefers coloring, add --color into the - # command line because we are going to wrap the command into - # a pipe and git won't know coloring should activate. - # - for cn in cmd[1:]: - if not cn.startswith('-'): - break - else: - cn = None - if cn and cn in _CAN_COLOR: - class ColorCmd(Coloring): - def __init__(self, config, cmd): - Coloring.__init__(self, config, cmd) - if ColorCmd(self.manifest.manifestProject.config, cn).is_on: - cmd.insert(cmd.index(cn) + 1, '--color') - - mirror = self.manifest.IsMirror - rc = 0 - - smart_sync_manifest_name = "smart_sync_override.xml" - smart_sync_manifest_path = os.path.join( - self.manifest.manifestProject.worktree, smart_sync_manifest_name) - - if os.path.isfile(smart_sync_manifest_path): - self.manifest.Override(smart_sync_manifest_path) - - if opt.regex: - projects = self.FindProjects(args, all_manifests=all_trees) - elif opt.inverse_regex: - projects = self.FindProjects(args, inverse=True, all_manifests=all_trees) - else: - projects = self.GetProjects(args, groups=opt.groups, all_manifests=all_trees) - - os.environ['REPO_COUNT'] = str(len(projects)) - - try: - config = self.manifest.manifestProject.config - with multiprocessing.Pool(opt.jobs, InitWorker) as pool: - results_it = pool.imap( - functools.partial(DoWorkWrapper, mirror, opt, cmd, shell, config), - enumerate(projects), - chunksize=WORKER_BATCH_SIZE) - first = True - for (r, output) in results_it: - if output: - if first: - first = False - elif opt.project_header: - print() - # To simplify the DoWorkWrapper, take care of automatic newlines. - end = '\n' - if output[-1] == '\n': - end = '' - print(output, end=end) - rc = rc or r - if r != 0 and opt.abort_on_errors: - raise Exception('Aborting due to previous error') - except (KeyboardInterrupt, WorkerKeyboardInterrupt): - # Catch KeyboardInterrupt raised inside and outside of workers - rc = rc or errno.EINTR - except Exception as e: - # Catch any other exceptions raised - print('forall: unhandled error, terminating the pool: %s: %s' % - (type(e).__name__, e), - file=sys.stderr) - rc = rc or getattr(e, 'errno', 1) - if rc != 0: - sys.exit(rc) + PARALLEL_JOBS = DEFAULT_LOCAL_JOBS + + @staticmethod + def _cmd_option(option, _opt_str, _value, parser): + setattr(parser.values, option.dest, list(parser.rargs)) + while parser.rargs: + del parser.rargs[0] + + def _Options(self, p): + p.add_option( + "-r", + "--regex", + dest="regex", + action="store_true", + help="execute the command only on projects matching regex or " + "wildcard expression", + ) + p.add_option( + "-i", + "--inverse-regex", + dest="inverse_regex", + action="store_true", + help="execute the command only on projects not matching regex or " + "wildcard expression", + ) + p.add_option( + "-g", + "--groups", + dest="groups", + help="execute the command only on projects matching the specified " + "groups", + ) + p.add_option( + "-c", + "--command", + help="command (and arguments) to execute", + dest="command", + action="callback", + callback=self._cmd_option, + ) + p.add_option( + "-e", + "--abort-on-errors", + dest="abort_on_errors", + action="store_true", + help="abort if a command exits unsuccessfully", + ) + p.add_option( + "--ignore-missing", + action="store_true", + help="silently skip & do not exit non-zero due missing " + "checkouts", + ) + + g = p.get_option_group("--quiet") + g.add_option( + "-p", + dest="project_header", + action="store_true", + help="show project headers before output", + ) + p.add_option( + "--interactive", action="store_true", help="force interactive usage" + ) + + def WantPager(self, opt): + return opt.project_header and opt.jobs == 1 + + def ValidateOptions(self, opt, args): + if not opt.command: + self.Usage() + + def Execute(self, opt, args): + cmd = [opt.command[0]] + all_trees = not opt.this_manifest_only + + shell = True + if re.compile(r"^[a-z0-9A-Z_/\.-]+$").match(cmd[0]): + shell = False + + if shell: + cmd.append(cmd[0]) + cmd.extend(opt.command[1:]) + + # Historically, forall operated interactively, and in serial. If the + # user has selected 1 job, then default to interacive mode. + if opt.jobs == 1: + opt.interactive = True + + if opt.project_header and not shell and cmd[0] == "git": + # If this is a direct git command that can enable colorized + # output and the user prefers coloring, add --color into the + # command line because we are going to wrap the command into + # a pipe and git won't know coloring should activate. + # + for cn in cmd[1:]: + if not cn.startswith("-"): + break + else: + cn = None + if cn and cn in _CAN_COLOR: + + class ColorCmd(Coloring): + def __init__(self, config, cmd): + Coloring.__init__(self, config, cmd) + + if ColorCmd(self.manifest.manifestProject.config, cn).is_on: + cmd.insert(cmd.index(cn) + 1, "--color") + + mirror = self.manifest.IsMirror + rc = 0 + + smart_sync_manifest_name = "smart_sync_override.xml" + smart_sync_manifest_path = os.path.join( + self.manifest.manifestProject.worktree, smart_sync_manifest_name + ) + + if os.path.isfile(smart_sync_manifest_path): + self.manifest.Override(smart_sync_manifest_path) + + if opt.regex: + projects = self.FindProjects(args, all_manifests=all_trees) + elif opt.inverse_regex: + projects = self.FindProjects( + args, inverse=True, all_manifests=all_trees + ) + else: + projects = self.GetProjects( + args, groups=opt.groups, all_manifests=all_trees + ) + + os.environ["REPO_COUNT"] = str(len(projects)) + + try: + config = self.manifest.manifestProject.config + with multiprocessing.Pool(opt.jobs, InitWorker) as pool: + results_it = pool.imap( + functools.partial( + DoWorkWrapper, mirror, opt, cmd, shell, config + ), + enumerate(projects), + chunksize=WORKER_BATCH_SIZE, + ) + first = True + for r, output in results_it: + if output: + if first: + first = False + elif opt.project_header: + print() + # To simplify the DoWorkWrapper, take care of automatic + # newlines. + end = "\n" + if output[-1] == "\n": + end = "" + print(output, end=end) + rc = rc or r + if r != 0 and opt.abort_on_errors: + raise Exception("Aborting due to previous error") + except (KeyboardInterrupt, WorkerKeyboardInterrupt): + # Catch KeyboardInterrupt raised inside and outside of workers + rc = rc or errno.EINTR + except Exception as e: + # Catch any other exceptions raised + print( + "forall: unhandled error, terminating the pool: %s: %s" + % (type(e).__name__, e), + file=sys.stderr, + ) + rc = rc or getattr(e, "errno", 1) + if rc != 0: + sys.exit(rc) class WorkerKeyboardInterrupt(Exception): - """ Keyboard interrupt exception for worker processes. """ + """Keyboard interrupt exception for worker processes.""" def InitWorker(): - signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGINT, signal.SIG_IGN) def DoWorkWrapper(mirror, opt, cmd, shell, config, args): - """ A wrapper around the DoWork() method. + """A wrapper around the DoWork() method. - Catch the KeyboardInterrupt exceptions here and re-raise them as a different, - ``Exception``-based exception to stop it flooding the console with stacktraces - and making the parent hang indefinitely. + Catch the KeyboardInterrupt exceptions here and re-raise them as a + different, ``Exception``-based exception to stop it flooding the console + with stacktraces and making the parent hang indefinitely. - """ - cnt, project = args - try: - return DoWork(project, mirror, opt, cmd, shell, cnt, config) - except KeyboardInterrupt: - print('%s: Worker interrupted' % project.name) - raise WorkerKeyboardInterrupt() + """ + cnt, project = args + try: + return DoWork(project, mirror, opt, cmd, shell, cnt, config) + except KeyboardInterrupt: + print("%s: Worker interrupted" % project.name) + raise WorkerKeyboardInterrupt() def DoWork(project, mirror, opt, cmd, shell, cnt, config): - env = os.environ.copy() - - def setenv(name, val): - if val is None: - val = '' - env[name] = val - - setenv('REPO_PROJECT', project.name) - setenv('REPO_OUTERPATH', project.manifest.path_prefix) - setenv('REPO_INNERPATH', project.relpath) - setenv('REPO_PATH', project.RelPath(local=opt.this_manifest_only)) - setenv('REPO_REMOTE', project.remote.name) - try: - # If we aren't in a fully synced state and we don't have the ref the manifest - # wants, then this will fail. Ignore it for the purposes of this code. - lrev = '' if mirror else project.GetRevisionId() - except ManifestInvalidRevisionError: - lrev = '' - setenv('REPO_LREV', lrev) - setenv('REPO_RREV', project.revisionExpr) - setenv('REPO_UPSTREAM', project.upstream) - setenv('REPO_DEST_BRANCH', project.dest_branch) - setenv('REPO_I', str(cnt + 1)) - for annotation in project.annotations: - setenv("REPO__%s" % (annotation.name), annotation.value) - - if mirror: - setenv('GIT_DIR', project.gitdir) - cwd = project.gitdir - else: - cwd = project.worktree - - if not os.path.exists(cwd): - # Allow the user to silently ignore missing checkouts so they can run on - # partial checkouts (good for infra recovery tools). - if opt.ignore_missing: - return (0, '') - - output = '' - if ((opt.project_header and opt.verbose) - or not opt.project_header): - output = 'skipping %s/' % project.RelPath(local=opt.this_manifest_only) - return (1, output) - - if opt.verbose: - stderr = subprocess.STDOUT - else: - stderr = subprocess.DEVNULL - - stdin = None if opt.interactive else subprocess.DEVNULL - - result = subprocess.run( - cmd, cwd=cwd, shell=shell, env=env, check=False, - encoding='utf-8', errors='replace', - stdin=stdin, stdout=subprocess.PIPE, stderr=stderr) - - output = result.stdout - if opt.project_header: - if output: - buf = io.StringIO() - out = ForallColoring(config) - out.redirect(buf) - if mirror: - project_header_path = project.name - else: - project_header_path = project.RelPath(local=opt.this_manifest_only) - out.project('project %s/' % project_header_path) - out.nl() - buf.write(output) - output = buf.getvalue() - return (result.returncode, output) + env = os.environ.copy() + + def setenv(name, val): + if val is None: + val = "" + env[name] = val + + setenv("REPO_PROJECT", project.name) + setenv("REPO_OUTERPATH", project.manifest.path_prefix) + setenv("REPO_INNERPATH", project.relpath) + setenv("REPO_PATH", project.RelPath(local=opt.this_manifest_only)) + setenv("REPO_REMOTE", project.remote.name) + try: + # If we aren't in a fully synced state and we don't have the ref the + # manifest wants, then this will fail. Ignore it for the purposes of + # this code. + lrev = "" if mirror else project.GetRevisionId() + except ManifestInvalidRevisionError: + lrev = "" + setenv("REPO_LREV", lrev) + setenv("REPO_RREV", project.revisionExpr) + setenv("REPO_UPSTREAM", project.upstream) + setenv("REPO_DEST_BRANCH", project.dest_branch) + setenv("REPO_I", str(cnt + 1)) + for annotation in project.annotations: + setenv("REPO__%s" % (annotation.name), annotation.value) + + if mirror: + setenv("GIT_DIR", project.gitdir) + cwd = project.gitdir + else: + cwd = project.worktree + + if not os.path.exists(cwd): + # Allow the user to silently ignore missing checkouts so they can run on + # partial checkouts (good for infra recovery tools). + if opt.ignore_missing: + return (0, "") + + output = "" + if (opt.project_header and opt.verbose) or not opt.project_header: + output = "skipping %s/" % project.RelPath( + local=opt.this_manifest_only + ) + return (1, output) + + if opt.verbose: + stderr = subprocess.STDOUT + else: + stderr = subprocess.DEVNULL + + stdin = None if opt.interactive else subprocess.DEVNULL + + result = subprocess.run( + cmd, + cwd=cwd, + shell=shell, + env=env, + check=False, + encoding="utf-8", + errors="replace", + stdin=stdin, + stdout=subprocess.PIPE, + stderr=stderr, + ) + + output = result.stdout + if opt.project_header: + if output: + buf = io.StringIO() + out = ForallColoring(config) + out.redirect(buf) + if mirror: + project_header_path = project.name + else: + project_header_path = project.RelPath( + local=opt.this_manifest_only + ) + out.project("project %s/" % project_header_path) + out.nl() + buf.write(output) + output = buf.getvalue() + return (result.returncode, output) diff --git a/subcmds/gitc_delete.py b/subcmds/gitc_delete.py index df749469..ae9d4d1f 100644 --- a/subcmds/gitc_delete.py +++ b/subcmds/gitc_delete.py @@ -19,28 +19,34 @@ import platform_utils class GitcDelete(Command, GitcClientCommand): - COMMON = True - visible_everywhere = False - helpSummary = "Delete a GITC Client." - helpUsage = """ + COMMON = True + visible_everywhere = False + helpSummary = "Delete a GITC Client." + helpUsage = """ %prog """ - helpDescription = """ + helpDescription = """ This subcommand deletes the current GITC client, deleting the GITC manifest and all locally downloaded sources. """ - def _Options(self, p): - p.add_option('-f', '--force', - dest='force', action='store_true', - help='force the deletion (no prompt)') + def _Options(self, p): + p.add_option( + "-f", + "--force", + dest="force", + action="store_true", + help="force the deletion (no prompt)", + ) - def Execute(self, opt, args): - if not opt.force: - prompt = ('This will delete GITC client: %s\nAre you sure? (yes/no) ' % - self.gitc_manifest.gitc_client_name) - response = input(prompt).lower() - if not response == 'yes': - print('Response was not "yes"\n Exiting...') - sys.exit(1) - platform_utils.rmtree(self.gitc_manifest.gitc_client_dir) + def Execute(self, opt, args): + if not opt.force: + prompt = ( + "This will delete GITC client: %s\nAre you sure? (yes/no) " + % self.gitc_manifest.gitc_client_name + ) + response = input(prompt).lower() + if not response == "yes": + print('Response was not "yes"\n Exiting...') + sys.exit(1) + platform_utils.rmtree(self.gitc_manifest.gitc_client_dir) diff --git a/subcmds/gitc_init.py b/subcmds/gitc_init.py index e3a5813d..54791d58 100644 --- a/subcmds/gitc_init.py +++ b/subcmds/gitc_init.py @@ -23,13 +23,13 @@ import wrapper class GitcInit(init.Init, GitcAvailableCommand): - COMMON = True - MULTI_MANIFEST_SUPPORT = False - helpSummary = "Initialize a GITC Client." - helpUsage = """ + COMMON = True + MULTI_MANIFEST_SUPPORT = False + helpSummary = "Initialize a GITC Client." + helpUsage = """ %prog [options] [client name] """ - helpDescription = """ + helpDescription = """ The '%prog' command is ran to initialize a new GITC client for use with the GITC file system. @@ -47,30 +47,41 @@ The optional -f argument can be used to specify the manifest file to use for this GITC client. """ - def _Options(self, p): - super()._Options(p, gitc_init=True) + def _Options(self, p): + super()._Options(p, gitc_init=True) - def Execute(self, opt, args): - gitc_client = gitc_utils.parse_clientdir(os.getcwd()) - if not gitc_client or (opt.gitc_client and gitc_client != opt.gitc_client): - print('fatal: Please update your repo command. See go/gitc for instructions.', - file=sys.stderr) - sys.exit(1) - self.client_dir = os.path.join(gitc_utils.get_gitc_manifest_dir(), - gitc_client) - super().Execute(opt, args) + def Execute(self, opt, args): + gitc_client = gitc_utils.parse_clientdir(os.getcwd()) + if not gitc_client or ( + opt.gitc_client and gitc_client != opt.gitc_client + ): + print( + "fatal: Please update your repo command. See go/gitc for " + "instructions.", + file=sys.stderr, + ) + sys.exit(1) + self.client_dir = os.path.join( + gitc_utils.get_gitc_manifest_dir(), gitc_client + ) + super().Execute(opt, args) - manifest_file = self.manifest.manifestFile - if opt.manifest_file: - if not os.path.exists(opt.manifest_file): - print('fatal: Specified manifest file %s does not exist.' % - opt.manifest_file) - sys.exit(1) - manifest_file = opt.manifest_file + manifest_file = self.manifest.manifestFile + if opt.manifest_file: + if not os.path.exists(opt.manifest_file): + print( + "fatal: Specified manifest file %s does not exist." + % opt.manifest_file + ) + sys.exit(1) + manifest_file = opt.manifest_file - manifest = GitcManifest(self.repodir, os.path.join(self.client_dir, - '.manifest')) - manifest.Override(manifest_file) - gitc_utils.generate_gitc_manifest(None, manifest) - print('Please run `cd %s` to view your GITC client.' % - os.path.join(wrapper.Wrapper().GITC_FS_ROOT_DIR, gitc_client)) + manifest = GitcManifest( + self.repodir, os.path.join(self.client_dir, ".manifest") + ) + manifest.Override(manifest_file) + gitc_utils.generate_gitc_manifest(None, manifest) + print( + "Please run `cd %s` to view your GITC client." + % os.path.join(wrapper.Wrapper().GITC_FS_ROOT_DIR, gitc_client) + ) diff --git a/subcmds/grep.py b/subcmds/grep.py index 93c9ae51..5cd33763 100644 --- a/subcmds/grep.py +++ b/subcmds/grep.py @@ -22,19 +22,19 @@ from git_command import GitCommand class GrepColoring(Coloring): - def __init__(self, config): - Coloring.__init__(self, config, 'grep') - self.project = self.printer('project', attr='bold') - self.fail = self.printer('fail', fg='red') + def __init__(self, config): + Coloring.__init__(self, config, "grep") + self.project = self.printer("project", attr="bold") + self.fail = self.printer("fail", fg="red") class Grep(PagedCommand): - COMMON = True - helpSummary = "Print lines matching a pattern" - helpUsage = """ + COMMON = True + helpSummary = "Print lines matching a pattern" + helpUsage = """ %prog {pattern | -e pattern} [...] """ - helpDescription = """ + helpDescription = """ Search for the specified patterns in all project files. # Boolean Options @@ -62,215 +62,304 @@ contain a line that matches both expressions: repo grep --all-match -e NODE -e Unexpected """ - PARALLEL_JOBS = DEFAULT_LOCAL_JOBS - - @staticmethod - def _carry_option(_option, opt_str, value, parser): - pt = getattr(parser.values, 'cmd_argv', None) - if pt is None: - pt = [] - setattr(parser.values, 'cmd_argv', pt) - - if opt_str == '-(': - pt.append('(') - elif opt_str == '-)': - pt.append(')') - else: - pt.append(opt_str) - - if value is not None: - pt.append(value) - - def _CommonOptions(self, p): - """Override common options slightly.""" - super()._CommonOptions(p, opt_v=False) - - def _Options(self, p): - g = p.add_option_group('Sources') - g.add_option('--cached', - action='callback', callback=self._carry_option, - help='Search the index, instead of the work tree') - g.add_option('-r', '--revision', - dest='revision', action='append', metavar='TREEish', - help='Search TREEish, instead of the work tree') - - g = p.add_option_group('Pattern') - g.add_option('-e', - action='callback', callback=self._carry_option, - metavar='PATTERN', type='str', - help='Pattern to search for') - g.add_option('-i', '--ignore-case', - action='callback', callback=self._carry_option, - help='Ignore case differences') - g.add_option('-a', '--text', - action='callback', callback=self._carry_option, - help="Process binary files as if they were text") - g.add_option('-I', - action='callback', callback=self._carry_option, - help="Don't match the pattern in binary files") - g.add_option('-w', '--word-regexp', - action='callback', callback=self._carry_option, - help='Match the pattern only at word boundaries') - g.add_option('-v', '--invert-match', - action='callback', callback=self._carry_option, - help='Select non-matching lines') - g.add_option('-G', '--basic-regexp', - action='callback', callback=self._carry_option, - help='Use POSIX basic regexp for patterns (default)') - g.add_option('-E', '--extended-regexp', - action='callback', callback=self._carry_option, - help='Use POSIX extended regexp for patterns') - g.add_option('-F', '--fixed-strings', - action='callback', callback=self._carry_option, - help='Use fixed strings (not regexp) for pattern') - - g = p.add_option_group('Pattern Grouping') - g.add_option('--all-match', - action='callback', callback=self._carry_option, - help='Limit match to lines that have all patterns') - g.add_option('--and', '--or', '--not', - action='callback', callback=self._carry_option, - help='Boolean operators to combine patterns') - g.add_option('-(', '-)', - action='callback', callback=self._carry_option, - help='Boolean operator grouping') - - g = p.add_option_group('Output') - g.add_option('-n', - action='callback', callback=self._carry_option, - help='Prefix the line number to matching lines') - g.add_option('-C', - action='callback', callback=self._carry_option, - metavar='CONTEXT', type='str', - help='Show CONTEXT lines around match') - g.add_option('-B', - action='callback', callback=self._carry_option, - metavar='CONTEXT', type='str', - help='Show CONTEXT lines before match') - g.add_option('-A', - action='callback', callback=self._carry_option, - metavar='CONTEXT', type='str', - help='Show CONTEXT lines after match') - g.add_option('-l', '--name-only', '--files-with-matches', - action='callback', callback=self._carry_option, - help='Show only file names containing matching lines') - g.add_option('-L', '--files-without-match', - action='callback', callback=self._carry_option, - help='Show only file names not containing matching lines') - - def _ExecuteOne(self, cmd_argv, project): - """Process one project.""" - try: - p = GitCommand(project, - cmd_argv, - bare=False, - capture_stdout=True, - capture_stderr=True) - except GitError as e: - return (project, -1, None, str(e)) - - return (project, p.Wait(), p.stdout, p.stderr) - - @staticmethod - def _ProcessResults(full_name, have_rev, opt, _pool, out, results): - git_failed = False - bad_rev = False - have_match = False - _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only) - - for project, rc, stdout, stderr in results: - if rc < 0: - git_failed = True - out.project('--- project %s ---' % _RelPath(project)) - out.nl() - out.fail('%s', stderr) - out.nl() - continue - - if rc: - # no results - if stderr: - if have_rev and 'fatal: ambiguous argument' in stderr: - bad_rev = True - else: - out.project('--- project %s ---' % _RelPath(project)) - out.nl() - out.fail('%s', stderr.strip()) - out.nl() - continue - have_match = True - - # We cut the last element, to avoid a blank line. - r = stdout.split('\n') - r = r[0:-1] - - if have_rev and full_name: - for line in r: - rev, line = line.split(':', 1) - out.write("%s", rev) - out.write(':') - out.project(_RelPath(project)) - out.write('/') - out.write("%s", line) - out.nl() - elif full_name: - for line in r: - out.project(_RelPath(project)) - out.write('/') - out.write("%s", line) - out.nl() - else: - for line in r: - print(line) - - return (git_failed, bad_rev, have_match) - - def Execute(self, opt, args): - out = GrepColoring(self.manifest.manifestProject.config) - - cmd_argv = ['grep'] - if out.is_on: - cmd_argv.append('--color') - cmd_argv.extend(getattr(opt, 'cmd_argv', [])) - - if '-e' not in cmd_argv: - if not args: - self.Usage() - cmd_argv.append('-e') - cmd_argv.append(args[0]) - args = args[1:] - - projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only) - - full_name = False - if len(projects) > 1: - cmd_argv.append('--full-name') - full_name = True - - have_rev = False - if opt.revision: - if '--cached' in cmd_argv: - print('fatal: cannot combine --cached and --revision', file=sys.stderr) - sys.exit(1) - have_rev = True - cmd_argv.extend(opt.revision) - cmd_argv.append('--') - - git_failed, bad_rev, have_match = self.ExecuteInParallel( - opt.jobs, - functools.partial(self._ExecuteOne, cmd_argv), - projects, - callback=functools.partial(self._ProcessResults, full_name, have_rev, opt), - output=out, - ordered=True) - - if git_failed: - sys.exit(1) - elif have_match: - sys.exit(0) - elif have_rev and bad_rev: - for r in opt.revision: - print("error: can't search revision %s" % r, file=sys.stderr) - sys.exit(1) - else: - sys.exit(1) + PARALLEL_JOBS = DEFAULT_LOCAL_JOBS + + @staticmethod + def _carry_option(_option, opt_str, value, parser): + pt = getattr(parser.values, "cmd_argv", None) + if pt is None: + pt = [] + setattr(parser.values, "cmd_argv", pt) + + if opt_str == "-(": + pt.append("(") + elif opt_str == "-)": + pt.append(")") + else: + pt.append(opt_str) + + if value is not None: + pt.append(value) + + def _CommonOptions(self, p): + """Override common options slightly.""" + super()._CommonOptions(p, opt_v=False) + + def _Options(self, p): + g = p.add_option_group("Sources") + g.add_option( + "--cached", + action="callback", + callback=self._carry_option, + help="Search the index, instead of the work tree", + ) + g.add_option( + "-r", + "--revision", + dest="revision", + action="append", + metavar="TREEish", + help="Search TREEish, instead of the work tree", + ) + + g = p.add_option_group("Pattern") + g.add_option( + "-e", + action="callback", + callback=self._carry_option, + metavar="PATTERN", + type="str", + help="Pattern to search for", + ) + g.add_option( + "-i", + "--ignore-case", + action="callback", + callback=self._carry_option, + help="Ignore case differences", + ) + g.add_option( + "-a", + "--text", + action="callback", + callback=self._carry_option, + help="Process binary files as if they were text", + ) + g.add_option( + "-I", + action="callback", + callback=self._carry_option, + help="Don't match the pattern in binary files", + ) + g.add_option( + "-w", + "--word-regexp", + action="callback", + callback=self._carry_option, + help="Match the pattern only at word boundaries", + ) + g.add_option( + "-v", + "--invert-match", + action="callback", + callback=self._carry_option, + help="Select non-matching lines", + ) + g.add_option( + "-G", + "--basic-regexp", + action="callback", + callback=self._carry_option, + help="Use POSIX basic regexp for patterns (default)", + ) + g.add_option( + "-E", + "--extended-regexp", + action="callback", + callback=self._carry_option, + help="Use POSIX extended regexp for patterns", + ) + g.add_option( + "-F", + "--fixed-strings", + action="callback", + callback=self._carry_option, + help="Use fixed strings (not regexp) for pattern", + ) + + g = p.add_option_group("Pattern Grouping") + g.add_option( + "--all-match", + action="callback", + callback=self._carry_option, + help="Limit match to lines that have all patterns", + ) + g.add_option( + "--and", + "--or", + "--not", + action="callback", + callback=self._carry_option, + help="Boolean operators to combine patterns", + ) + g.add_option( + "-(", + "-)", + action="callback", + callback=self._carry_option, + help="Boolean operator grouping", + ) + + g = p.add_option_group("Output") + g.add_option( + "-n", + action="callback", + callback=self._carry_option, + help="Prefix the line number to matching lines", + ) + g.add_option( + "-C", + action="callback", + callback=self._carry_option, + metavar="CONTEXT", + type="str", + help="Show CONTEXT lines around match", + ) + g.add_option( + "-B", + action="callback", + callback=self._carry_option, + metavar="CONTEXT", + type="str", + help="Show CONTEXT lines before match", + ) + g.add_option( + "-A", + action="callback", + callback=self._carry_option, + metavar="CONTEXT", + type="str", + help="Show CONTEXT lines after match", + ) + g.add_option( + "-l", + "--name-only", + "--files-with-matches", + action="callback", + callback=self._carry_option, + help="Show only file names containing matching lines", + ) + g.add_option( + "-L", + "--files-without-match", + action="callback", + callback=self._carry_option, + help="Show only file names not containing matching lines", + ) + + def _ExecuteOne(self, cmd_argv, project): + """Process one project.""" + try: + p = GitCommand( + project, + cmd_argv, + bare=False, + capture_stdout=True, + capture_stderr=True, + ) + except GitError as e: + return (project, -1, None, str(e)) + + return (project, p.Wait(), p.stdout, p.stderr) + + @staticmethod + def _ProcessResults(full_name, have_rev, opt, _pool, out, results): + git_failed = False + bad_rev = False + have_match = False + _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only) + + for project, rc, stdout, stderr in results: + if rc < 0: + git_failed = True + out.project("--- project %s ---" % _RelPath(project)) + out.nl() + out.fail("%s", stderr) + out.nl() + continue + + if rc: + # no results + if stderr: + if have_rev and "fatal: ambiguous argument" in stderr: + bad_rev = True + else: + out.project("--- project %s ---" % _RelPath(project)) + out.nl() + out.fail("%s", stderr.strip()) + out.nl() + continue + have_match = True + + # We cut the last element, to avoid a blank line. + r = stdout.split("\n") + r = r[0:-1] + + if have_rev and full_name: + for line in r: + rev, line = line.split(":", 1) + out.write("%s", rev) + out.write(":") + out.project(_RelPath(project)) + out.write("/") + out.write("%s", line) + out.nl() + elif full_name: + for line in r: + out.project(_RelPath(project)) + out.write("/") + out.write("%s", line) + out.nl() + else: + for line in r: + print(line) + + return (git_failed, bad_rev, have_match) + + def Execute(self, opt, args): + out = GrepColoring(self.manifest.manifestProject.config) + + cmd_argv = ["grep"] + if out.is_on: + cmd_argv.append("--color") + cmd_argv.extend(getattr(opt, "cmd_argv", [])) + + if "-e" not in cmd_argv: + if not args: + self.Usage() + cmd_argv.append("-e") + cmd_argv.append(args[0]) + args = args[1:] + + projects = self.GetProjects( + args, all_manifests=not opt.this_manifest_only + ) + + full_name = False + if len(projects) > 1: + cmd_argv.append("--full-name") + full_name = True + + have_rev = False + if opt.revision: + if "--cached" in cmd_argv: + print( + "fatal: cannot combine --cached and --revision", + file=sys.stderr, + ) + sys.exit(1) + have_rev = True + cmd_argv.extend(opt.revision) + cmd_argv.append("--") + + git_failed, bad_rev, have_match = self.ExecuteInParallel( + opt.jobs, + functools.partial(self._ExecuteOne, cmd_argv), + projects, + callback=functools.partial( + self._ProcessResults, full_name, have_rev, opt + ), + output=out, + ordered=True, + ) + + if git_failed: + sys.exit(1) + elif have_match: + sys.exit(0) + elif have_rev and bad_rev: + for r in opt.revision: + print("error: can't search revision %s" % r, file=sys.stderr) + sys.exit(1) + else: + sys.exit(1) diff --git a/subcmds/help.py b/subcmds/help.py index 1ad391db..50a48047 100644 --- a/subcmds/help.py +++ b/subcmds/help.py @@ -18,163 +18,193 @@ import textwrap from subcmds import all_commands from color import Coloring -from command import PagedCommand, MirrorSafeCommand, GitcAvailableCommand, GitcClientCommand +from command import ( + PagedCommand, + MirrorSafeCommand, + GitcAvailableCommand, + GitcClientCommand, +) import gitc_utils from wrapper import Wrapper class Help(PagedCommand, MirrorSafeCommand): - COMMON = False - helpSummary = "Display detailed help on a command" - helpUsage = """ + COMMON = False + helpSummary = "Display detailed help on a command" + helpUsage = """ %prog [--all|command] """ - helpDescription = """ + helpDescription = """ Displays detailed usage information about a command. """ - def _PrintCommands(self, commandNames): - """Helper to display |commandNames| summaries.""" - maxlen = 0 - for name in commandNames: - maxlen = max(maxlen, len(name)) - fmt = ' %%-%ds %%s' % maxlen - - for name in commandNames: - command = all_commands[name]() - try: - summary = command.helpSummary.strip() - except AttributeError: - summary = '' - print(fmt % (name, summary)) - - def _PrintAllCommands(self): - print('usage: repo COMMAND [ARGS]') - self.PrintAllCommandsBody() - - def PrintAllCommandsBody(self): - print('The complete list of recognized repo commands is:') - commandNames = list(sorted(all_commands)) - self._PrintCommands(commandNames) - print("See 'repo help ' for more information on a " - 'specific command.') - print('Bug reports:', Wrapper().BUG_URL) - - def _PrintCommonCommands(self): - print('usage: repo COMMAND [ARGS]') - self.PrintCommonCommandsBody() - - def PrintCommonCommandsBody(self): - print('The most commonly used repo commands are:') - - def gitc_supported(cmd): - if not isinstance(cmd, GitcAvailableCommand) and not isinstance(cmd, GitcClientCommand): - return True - if self.client.isGitcClient: - return True - if isinstance(cmd, GitcClientCommand): - return False - if gitc_utils.get_gitc_manifest_dir(): - return True - return False - - commandNames = list(sorted([name - for name, command in all_commands.items() - if command.COMMON and gitc_supported(command)])) - self._PrintCommands(commandNames) - - print( - "See 'repo help ' for more information on a specific command.\n" - "See 'repo help --all' for a complete list of recognized commands.") - print('Bug reports:', Wrapper().BUG_URL) - - def _PrintCommandHelp(self, cmd, header_prefix=''): - class _Out(Coloring): - def __init__(self, gc): - Coloring.__init__(self, gc, 'help') - self.heading = self.printer('heading', attr='bold') - self._first = True - - def _PrintSection(self, heading, bodyAttr): - try: - body = getattr(cmd, bodyAttr) - except AttributeError: - return - if body == '' or body is None: - return - - if not self._first: - self.nl() - self._first = False - - self.heading('%s%s', header_prefix, heading) - self.nl() - self.nl() - - me = 'repo %s' % cmd.NAME - body = body.strip() - body = body.replace('%prog', me) - - # Extract the title, but skip any trailing {#anchors}. - asciidoc_hdr = re.compile(r'^\n?#+ ([^{]+)(\{#.+\})?$') - for para in body.split("\n\n"): - if para.startswith(' '): - self.write('%s', para) - self.nl() - self.nl() - continue - - m = asciidoc_hdr.match(para) - if m: - self.heading('%s%s', header_prefix, m.group(1)) - self.nl() - self.nl() - continue - - lines = textwrap.wrap(para.replace(' ', ' '), width=80, - break_long_words=False, break_on_hyphens=False) - for line in lines: - self.write('%s', line) - self.nl() - self.nl() - - out = _Out(self.client.globalConfig) - out._PrintSection('Summary', 'helpSummary') - cmd.OptionParser.print_help() - out._PrintSection('Description', 'helpDescription') - - def _PrintAllCommandHelp(self): - for name in sorted(all_commands): - cmd = all_commands[name](manifest=self.manifest) - self._PrintCommandHelp(cmd, header_prefix='[%s] ' % (name,)) - - def _Options(self, p): - p.add_option('-a', '--all', - dest='show_all', action='store_true', - help='show the complete list of commands') - p.add_option('--help-all', - dest='show_all_help', action='store_true', - help='show the --help of all commands') - - def Execute(self, opt, args): - if len(args) == 0: - if opt.show_all_help: - self._PrintAllCommandHelp() - elif opt.show_all: - self._PrintAllCommands() - else: - self._PrintCommonCommands() - - elif len(args) == 1: - name = args[0] - - try: - cmd = all_commands[name](manifest=self.manifest) - except KeyError: - print("repo: '%s' is not a repo command." % name, file=sys.stderr) - sys.exit(1) - - self._PrintCommandHelp(cmd) - - else: - self._PrintCommandHelp(self) + def _PrintCommands(self, commandNames): + """Helper to display |commandNames| summaries.""" + maxlen = 0 + for name in commandNames: + maxlen = max(maxlen, len(name)) + fmt = " %%-%ds %%s" % maxlen + + for name in commandNames: + command = all_commands[name]() + try: + summary = command.helpSummary.strip() + except AttributeError: + summary = "" + print(fmt % (name, summary)) + + def _PrintAllCommands(self): + print("usage: repo COMMAND [ARGS]") + self.PrintAllCommandsBody() + + def PrintAllCommandsBody(self): + print("The complete list of recognized repo commands is:") + commandNames = list(sorted(all_commands)) + self._PrintCommands(commandNames) + print( + "See 'repo help ' for more information on a " + "specific command." + ) + print("Bug reports:", Wrapper().BUG_URL) + + def _PrintCommonCommands(self): + print("usage: repo COMMAND [ARGS]") + self.PrintCommonCommandsBody() + + def PrintCommonCommandsBody(self): + print("The most commonly used repo commands are:") + + def gitc_supported(cmd): + if not isinstance(cmd, GitcAvailableCommand) and not isinstance( + cmd, GitcClientCommand + ): + return True + if self.client.isGitcClient: + return True + if isinstance(cmd, GitcClientCommand): + return False + if gitc_utils.get_gitc_manifest_dir(): + return True + return False + + commandNames = list( + sorted( + [ + name + for name, command in all_commands.items() + if command.COMMON and gitc_supported(command) + ] + ) + ) + self._PrintCommands(commandNames) + + print( + "See 'repo help ' for more information on a specific " + "command.\nSee 'repo help --all' for a complete list of recognized " + "commands." + ) + print("Bug reports:", Wrapper().BUG_URL) + + def _PrintCommandHelp(self, cmd, header_prefix=""): + class _Out(Coloring): + def __init__(self, gc): + Coloring.__init__(self, gc, "help") + self.heading = self.printer("heading", attr="bold") + self._first = True + + def _PrintSection(self, heading, bodyAttr): + try: + body = getattr(cmd, bodyAttr) + except AttributeError: + return + if body == "" or body is None: + return + + if not self._first: + self.nl() + self._first = False + + self.heading("%s%s", header_prefix, heading) + self.nl() + self.nl() + + me = "repo %s" % cmd.NAME + body = body.strip() + body = body.replace("%prog", me) + + # Extract the title, but skip any trailing {#anchors}. + asciidoc_hdr = re.compile(r"^\n?#+ ([^{]+)(\{#.+\})?$") + for para in body.split("\n\n"): + if para.startswith(" "): + self.write("%s", para) + self.nl() + self.nl() + continue + + m = asciidoc_hdr.match(para) + if m: + self.heading("%s%s", header_prefix, m.group(1)) + self.nl() + self.nl() + continue + + lines = textwrap.wrap( + para.replace(" ", " "), + width=80, + break_long_words=False, + break_on_hyphens=False, + ) + for line in lines: + self.write("%s", line) + self.nl() + self.nl() + + out = _Out(self.client.globalConfig) + out._PrintSection("Summary", "helpSummary") + cmd.OptionParser.print_help() + out._PrintSection("Description", "helpDescription") + + def _PrintAllCommandHelp(self): + for name in sorted(all_commands): + cmd = all_commands[name](manifest=self.manifest) + self._PrintCommandHelp(cmd, header_prefix="[%s] " % (name,)) + + def _Options(self, p): + p.add_option( + "-a", + "--all", + dest="show_all", + action="store_true", + help="show the complete list of commands", + ) + p.add_option( + "--help-all", + dest="show_all_help", + action="store_true", + help="show the --help of all commands", + ) + + def Execute(self, opt, args): + if len(args) == 0: + if opt.show_all_help: + self._PrintAllCommandHelp() + elif opt.show_all: + self._PrintAllCommands() + else: + self._PrintCommonCommands() + + elif len(args) == 1: + name = args[0] + + try: + cmd = all_commands[name](manifest=self.manifest) + except KeyError: + print( + "repo: '%s' is not a repo command." % name, file=sys.stderr + ) + sys.exit(1) + + self._PrintCommandHelp(cmd) + + else: + self._PrintCommandHelp(self) diff --git a/subcmds/info.py b/subcmds/info.py index baa4c5b1..6e7f3ed2 100644 --- a/subcmds/info.py +++ b/subcmds/info.py @@ -20,203 +20,234 @@ from git_refs import R_M, R_HEADS class _Coloring(Coloring): - def __init__(self, config): - Coloring.__init__(self, config, "status") + def __init__(self, config): + Coloring.__init__(self, config, "status") class Info(PagedCommand): - COMMON = True - helpSummary = "Get info on the manifest branch, current branch or unmerged branches" - helpUsage = "%prog [-dl] [-o [-c]] [...]" - - def _Options(self, p): - p.add_option('-d', '--diff', - dest='all', action='store_true', - help="show full info and commit diff including remote branches") - p.add_option('-o', '--overview', - dest='overview', action='store_true', - help='show overview of all local commits') - p.add_option('-c', '--current-branch', - dest="current_branch", action="store_true", - help="consider only checked out branches") - p.add_option('--no-current-branch', - dest='current_branch', action='store_false', - help='consider all local branches') - # Turn this into a warning & remove this someday. - p.add_option('-b', - dest='current_branch', action='store_true', - help=optparse.SUPPRESS_HELP) - p.add_option('-l', '--local-only', - dest="local", action="store_true", - help="disable all remote operations") - - def Execute(self, opt, args): - self.out = _Coloring(self.client.globalConfig) - self.heading = self.out.printer('heading', attr='bold') - self.headtext = self.out.nofmt_printer('headtext', fg='yellow') - self.redtext = self.out.printer('redtext', fg='red') - self.sha = self.out.printer("sha", fg='yellow') - self.text = self.out.nofmt_printer('text') - self.dimtext = self.out.printer('dimtext', attr='dim') - - self.opt = opt - - if not opt.this_manifest_only: - self.manifest = self.manifest.outer_client - manifestConfig = self.manifest.manifestProject.config - mergeBranch = manifestConfig.GetBranch("default").merge - manifestGroups = self.manifest.GetGroupsStr() - - self.heading("Manifest branch: ") - if self.manifest.default.revisionExpr: - self.headtext(self.manifest.default.revisionExpr) - self.out.nl() - self.heading("Manifest merge branch: ") - self.headtext(mergeBranch) - self.out.nl() - self.heading("Manifest groups: ") - self.headtext(manifestGroups) - self.out.nl() - - self.printSeparator() - - if not opt.overview: - self._printDiffInfo(opt, args) - else: - self._printCommitOverview(opt, args) - - def printSeparator(self): - self.text("----------------------------") - self.out.nl() - - def _printDiffInfo(self, opt, args): - # We let exceptions bubble up to main as they'll be well structured. - projs = self.GetProjects(args, all_manifests=not opt.this_manifest_only) - - for p in projs: - self.heading("Project: ") - self.headtext(p.name) - self.out.nl() - - self.heading("Mount path: ") - self.headtext(p.worktree) - self.out.nl() - - self.heading("Current revision: ") - self.headtext(p.GetRevisionId()) - self.out.nl() - - currentBranch = p.CurrentBranch - if currentBranch: - self.heading('Current branch: ') - self.headtext(currentBranch) + COMMON = True + helpSummary = ( + "Get info on the manifest branch, current branch or unmerged branches" + ) + helpUsage = "%prog [-dl] [-o [-c]] [...]" + + def _Options(self, p): + p.add_option( + "-d", + "--diff", + dest="all", + action="store_true", + help="show full info and commit diff including remote branches", + ) + p.add_option( + "-o", + "--overview", + dest="overview", + action="store_true", + help="show overview of all local commits", + ) + p.add_option( + "-c", + "--current-branch", + dest="current_branch", + action="store_true", + help="consider only checked out branches", + ) + p.add_option( + "--no-current-branch", + dest="current_branch", + action="store_false", + help="consider all local branches", + ) + # Turn this into a warning & remove this someday. + p.add_option( + "-b", + dest="current_branch", + action="store_true", + help=optparse.SUPPRESS_HELP, + ) + p.add_option( + "-l", + "--local-only", + dest="local", + action="store_true", + help="disable all remote operations", + ) + + def Execute(self, opt, args): + self.out = _Coloring(self.client.globalConfig) + self.heading = self.out.printer("heading", attr="bold") + self.headtext = self.out.nofmt_printer("headtext", fg="yellow") + self.redtext = self.out.printer("redtext", fg="red") + self.sha = self.out.printer("sha", fg="yellow") + self.text = self.out.nofmt_printer("text") + self.dimtext = self.out.printer("dimtext", attr="dim") + + self.opt = opt + + if not opt.this_manifest_only: + self.manifest = self.manifest.outer_client + manifestConfig = self.manifest.manifestProject.config + mergeBranch = manifestConfig.GetBranch("default").merge + manifestGroups = self.manifest.GetGroupsStr() + + self.heading("Manifest branch: ") + if self.manifest.default.revisionExpr: + self.headtext(self.manifest.default.revisionExpr) + self.out.nl() + self.heading("Manifest merge branch: ") + self.headtext(mergeBranch) + self.out.nl() + self.heading("Manifest groups: ") + self.headtext(manifestGroups) + self.out.nl() + + self.printSeparator() + + if not opt.overview: + self._printDiffInfo(opt, args) + else: + self._printCommitOverview(opt, args) + + def printSeparator(self): + self.text("----------------------------") self.out.nl() - self.heading("Manifest revision: ") - self.headtext(p.revisionExpr) - self.out.nl() - - localBranches = list(p.GetBranches().keys()) - self.heading("Local Branches: ") - self.redtext(str(len(localBranches))) - if localBranches: - self.text(" [") - self.text(", ".join(localBranches)) - self.text("]") - self.out.nl() - - if self.opt.all: - self.findRemoteLocalDiff(p) - - self.printSeparator() - - def findRemoteLocalDiff(self, project): - # Fetch all the latest commits. - if not self.opt.local: - project.Sync_NetworkHalf(quiet=True, current_branch_only=True) - - branch = self.manifest.manifestProject.config.GetBranch('default').merge - if branch.startswith(R_HEADS): - branch = branch[len(R_HEADS):] - logTarget = R_M + branch - - bareTmp = project.bare_git._bare - project.bare_git._bare = False - localCommits = project.bare_git.rev_list( - '--abbrev=8', - '--abbrev-commit', - '--pretty=oneline', - logTarget + "..", - '--') - - originCommits = project.bare_git.rev_list( - '--abbrev=8', - '--abbrev-commit', - '--pretty=oneline', - ".." + logTarget, - '--') - project.bare_git._bare = bareTmp - - self.heading("Local Commits: ") - self.redtext(str(len(localCommits))) - self.dimtext(" (on current branch)") - self.out.nl() - - for c in localCommits: - split = c.split() - self.sha(split[0] + " ") - self.text(" ".join(split[1:])) - self.out.nl() - - self.printSeparator() - - self.heading("Remote Commits: ") - self.redtext(str(len(originCommits))) - self.out.nl() - - for c in originCommits: - split = c.split() - self.sha(split[0] + " ") - self.text(" ".join(split[1:])) - self.out.nl() - - def _printCommitOverview(self, opt, args): - all_branches = [] - for project in self.GetProjects(args, all_manifests=not opt.this_manifest_only): - br = [project.GetUploadableBranch(x) - for x in project.GetBranches()] - br = [x for x in br if x] - if self.opt.current_branch: - br = [x for x in br if x.name == project.CurrentBranch] - all_branches.extend(br) - - if not all_branches: - return - - self.out.nl() - self.heading('Projects Overview') - project = None - - for branch in all_branches: - if project != branch.project: - project = branch.project + def _printDiffInfo(self, opt, args): + # We let exceptions bubble up to main as they'll be well structured. + projs = self.GetProjects(args, all_manifests=not opt.this_manifest_only) + + for p in projs: + self.heading("Project: ") + self.headtext(p.name) + self.out.nl() + + self.heading("Mount path: ") + self.headtext(p.worktree) + self.out.nl() + + self.heading("Current revision: ") + self.headtext(p.GetRevisionId()) + self.out.nl() + + currentBranch = p.CurrentBranch + if currentBranch: + self.heading("Current branch: ") + self.headtext(currentBranch) + self.out.nl() + + self.heading("Manifest revision: ") + self.headtext(p.revisionExpr) + self.out.nl() + + localBranches = list(p.GetBranches().keys()) + self.heading("Local Branches: ") + self.redtext(str(len(localBranches))) + if localBranches: + self.text(" [") + self.text(", ".join(localBranches)) + self.text("]") + self.out.nl() + + if self.opt.all: + self.findRemoteLocalDiff(p) + + self.printSeparator() + + def findRemoteLocalDiff(self, project): + # Fetch all the latest commits. + if not self.opt.local: + project.Sync_NetworkHalf(quiet=True, current_branch_only=True) + + branch = self.manifest.manifestProject.config.GetBranch("default").merge + if branch.startswith(R_HEADS): + branch = branch[len(R_HEADS) :] + logTarget = R_M + branch + + bareTmp = project.bare_git._bare + project.bare_git._bare = False + localCommits = project.bare_git.rev_list( + "--abbrev=8", + "--abbrev-commit", + "--pretty=oneline", + logTarget + "..", + "--", + ) + + originCommits = project.bare_git.rev_list( + "--abbrev=8", + "--abbrev-commit", + "--pretty=oneline", + ".." + logTarget, + "--", + ) + project.bare_git._bare = bareTmp + + self.heading("Local Commits: ") + self.redtext(str(len(localCommits))) + self.dimtext(" (on current branch)") self.out.nl() - self.headtext(project.RelPath(local=opt.this_manifest_only)) + + for c in localCommits: + split = c.split() + self.sha(split[0] + " ") + self.text(" ".join(split[1:])) + self.out.nl() + + self.printSeparator() + + self.heading("Remote Commits: ") + self.redtext(str(len(originCommits))) self.out.nl() - commits = branch.commits - date = branch.date - self.text('%s %-33s (%2d commit%s, %s)' % ( - branch.name == project.CurrentBranch and '*' or ' ', - branch.name, - len(commits), - len(commits) != 1 and 's' or '', - date)) - self.out.nl() - - for commit in commits: - split = commit.split() - self.text('{0:38}{1} '.format('', '-')) - self.sha(split[0] + " ") - self.text(" ".join(split[1:])) + for c in originCommits: + split = c.split() + self.sha(split[0] + " ") + self.text(" ".join(split[1:])) + self.out.nl() + + def _printCommitOverview(self, opt, args): + all_branches = [] + for project in self.GetProjects( + args, all_manifests=not opt.this_manifest_only + ): + br = [project.GetUploadableBranch(x) for x in project.GetBranches()] + br = [x for x in br if x] + if self.opt.current_branch: + br = [x for x in br if x.name == project.CurrentBranch] + all_branches.extend(br) + + if not all_branches: + return + self.out.nl() + self.heading("Projects Overview") + project = None + + for branch in all_branches: + if project != branch.project: + project = branch.project + self.out.nl() + self.headtext(project.RelPath(local=opt.this_manifest_only)) + self.out.nl() + + commits = branch.commits + date = branch.date + self.text( + "%s %-33s (%2d commit%s, %s)" + % ( + branch.name == project.CurrentBranch and "*" or " ", + branch.name, + len(commits), + len(commits) != 1 and "s" or "", + date, + ) + ) + self.out.nl() + + for commit in commits: + split = commit.split() + self.text("{0:38}{1} ".format("", "-")) + self.sha(split[0] + " ") + self.text(" ".join(split[1:])) + self.out.nl() diff --git a/subcmds/init.py b/subcmds/init.py index 813fa590..b5c2e3b5 100644 --- a/subcmds/init.py +++ b/subcmds/init.py @@ -22,13 +22,13 @@ from wrapper import Wrapper class Init(InteractiveCommand, MirrorSafeCommand): - COMMON = True - MULTI_MANIFEST_SUPPORT = True - helpSummary = "Initialize a repo client checkout in the current directory" - helpUsage = """ + COMMON = True + MULTI_MANIFEST_SUPPORT = True + helpSummary = "Initialize a repo client checkout in the current directory" + helpUsage = """ %prog [options] [manifest url] """ - helpDescription = """ + helpDescription = """ The '%prog' command is run once to install and initialize repo. The latest repo source code and manifest collection is downloaded from the server and is installed in the .repo/ directory in the @@ -77,243 +77,303 @@ manifest, a subsequent `repo sync` (or `repo sync -d`) is necessary to update the working directory files. """ - def _CommonOptions(self, p): - """Disable due to re-use of Wrapper().""" - - def _Options(self, p, gitc_init=False): - Wrapper().InitParser(p, gitc_init=gitc_init) - m = p.add_option_group('Multi-manifest') - m.add_option('--outer-manifest', action='store_true', default=True, - help='operate starting at the outermost manifest') - m.add_option('--no-outer-manifest', dest='outer_manifest', - action='store_false', help='do not operate on outer manifests') - m.add_option('--this-manifest-only', action='store_true', default=None, - help='only operate on this (sub)manifest') - m.add_option('--no-this-manifest-only', '--all-manifests', - dest='this_manifest_only', action='store_false', - help='operate on this manifest and its submanifests') - - def _RegisteredEnvironmentOptions(self): - return {'REPO_MANIFEST_URL': 'manifest_url', - 'REPO_MIRROR_LOCATION': 'reference'} - - def _SyncManifest(self, opt): - """Call manifestProject.Sync with arguments from opt. - - Args: - opt: options from optparse. - """ - # Normally this value is set when instantiating the project, but the - # manifest project is special and is created when instantiating the - # manifest which happens before we parse options. - self.manifest.manifestProject.clone_depth = opt.manifest_depth - if not self.manifest.manifestProject.Sync( - manifest_url=opt.manifest_url, - manifest_branch=opt.manifest_branch, - standalone_manifest=opt.standalone_manifest, - groups=opt.groups, - platform=opt.platform, - mirror=opt.mirror, - dissociate=opt.dissociate, - reference=opt.reference, - worktree=opt.worktree, - submodules=opt.submodules, - archive=opt.archive, - partial_clone=opt.partial_clone, - clone_filter=opt.clone_filter, - partial_clone_exclude=opt.partial_clone_exclude, - clone_bundle=opt.clone_bundle, - git_lfs=opt.git_lfs, - use_superproject=opt.use_superproject, - verbose=opt.verbose, - current_branch_only=opt.current_branch_only, - tags=opt.tags, - depth=opt.depth, - git_event_log=self.git_event_log, - manifest_name=opt.manifest_name): - sys.exit(1) - - def _Prompt(self, prompt, value): - print('%-10s [%s]: ' % (prompt, value), end='', flush=True) - a = sys.stdin.readline().strip() - if a == '': - return value - return a - - def _ShouldConfigureUser(self, opt, existing_checkout): - gc = self.client.globalConfig - mp = self.manifest.manifestProject - - # If we don't have local settings, get from global. - if not mp.config.Has('user.name') or not mp.config.Has('user.email'): - if not gc.Has('user.name') or not gc.Has('user.email'): - return True - - mp.config.SetString('user.name', gc.GetString('user.name')) - mp.config.SetString('user.email', gc.GetString('user.email')) - - if not opt.quiet and not existing_checkout or opt.verbose: - print() - print('Your identity is: %s <%s>' % (mp.config.GetString('user.name'), - mp.config.GetString('user.email'))) - print("If you want to change this, please re-run 'repo init' with --config-name") - return False - - def _ConfigureUser(self, opt): - mp = self.manifest.manifestProject - - while True: - if not opt.quiet: + def _CommonOptions(self, p): + """Disable due to re-use of Wrapper().""" + + def _Options(self, p, gitc_init=False): + Wrapper().InitParser(p, gitc_init=gitc_init) + m = p.add_option_group("Multi-manifest") + m.add_option( + "--outer-manifest", + action="store_true", + default=True, + help="operate starting at the outermost manifest", + ) + m.add_option( + "--no-outer-manifest", + dest="outer_manifest", + action="store_false", + help="do not operate on outer manifests", + ) + m.add_option( + "--this-manifest-only", + action="store_true", + default=None, + help="only operate on this (sub)manifest", + ) + m.add_option( + "--no-this-manifest-only", + "--all-manifests", + dest="this_manifest_only", + action="store_false", + help="operate on this manifest and its submanifests", + ) + + def _RegisteredEnvironmentOptions(self): + return { + "REPO_MANIFEST_URL": "manifest_url", + "REPO_MIRROR_LOCATION": "reference", + } + + def _SyncManifest(self, opt): + """Call manifestProject.Sync with arguments from opt. + + Args: + opt: options from optparse. + """ + # Normally this value is set when instantiating the project, but the + # manifest project is special and is created when instantiating the + # manifest which happens before we parse options. + self.manifest.manifestProject.clone_depth = opt.manifest_depth + if not self.manifest.manifestProject.Sync( + manifest_url=opt.manifest_url, + manifest_branch=opt.manifest_branch, + standalone_manifest=opt.standalone_manifest, + groups=opt.groups, + platform=opt.platform, + mirror=opt.mirror, + dissociate=opt.dissociate, + reference=opt.reference, + worktree=opt.worktree, + submodules=opt.submodules, + archive=opt.archive, + partial_clone=opt.partial_clone, + clone_filter=opt.clone_filter, + partial_clone_exclude=opt.partial_clone_exclude, + clone_bundle=opt.clone_bundle, + git_lfs=opt.git_lfs, + use_superproject=opt.use_superproject, + verbose=opt.verbose, + current_branch_only=opt.current_branch_only, + tags=opt.tags, + depth=opt.depth, + git_event_log=self.git_event_log, + manifest_name=opt.manifest_name, + ): + sys.exit(1) + + def _Prompt(self, prompt, value): + print("%-10s [%s]: " % (prompt, value), end="", flush=True) + a = sys.stdin.readline().strip() + if a == "": + return value + return a + + def _ShouldConfigureUser(self, opt, existing_checkout): + gc = self.client.globalConfig + mp = self.manifest.manifestProject + + # If we don't have local settings, get from global. + if not mp.config.Has("user.name") or not mp.config.Has("user.email"): + if not gc.Has("user.name") or not gc.Has("user.email"): + return True + + mp.config.SetString("user.name", gc.GetString("user.name")) + mp.config.SetString("user.email", gc.GetString("user.email")) + + if not opt.quiet and not existing_checkout or opt.verbose: + print() + print( + "Your identity is: %s <%s>" + % ( + mp.config.GetString("user.name"), + mp.config.GetString("user.email"), + ) + ) + print( + "If you want to change this, please re-run 'repo init' with " + "--config-name" + ) + return False + + def _ConfigureUser(self, opt): + mp = self.manifest.manifestProject + + while True: + if not opt.quiet: + print() + name = self._Prompt("Your Name", mp.UserName) + email = self._Prompt("Your Email", mp.UserEmail) + + if not opt.quiet: + print() + print("Your identity is: %s <%s>" % (name, email)) + print("is this correct [y/N]? ", end="", flush=True) + a = sys.stdin.readline().strip().lower() + if a in ("yes", "y", "t", "true"): + break + + if name != mp.UserName: + mp.config.SetString("user.name", name) + if email != mp.UserEmail: + mp.config.SetString("user.email", email) + + def _HasColorSet(self, gc): + for n in ["ui", "diff", "status"]: + if gc.Has("color.%s" % n): + return True + return False + + def _ConfigureColor(self): + gc = self.client.globalConfig + if self._HasColorSet(gc): + return + + class _Test(Coloring): + def __init__(self): + Coloring.__init__(self, gc, "test color display") + self._on = True + + out = _Test() + print() - name = self._Prompt('Your Name', mp.UserName) - email = self._Prompt('Your Email', mp.UserEmail) + print("Testing colorized output (for 'repo diff', 'repo status'):") + + for c in ["black", "red", "green", "yellow", "blue", "magenta", "cyan"]: + out.write(" ") + out.printer(fg=c)(" %-6s ", c) + out.write(" ") + out.printer(fg="white", bg="black")(" %s " % "white") + out.nl() + + for c in ["bold", "dim", "ul", "reverse"]: + out.write(" ") + out.printer(fg="black", attr=c)(" %-6s ", c) + out.nl() + + print( + "Enable color display in this user account (y/N)? ", + end="", + flush=True, + ) + a = sys.stdin.readline().strip().lower() + if a in ("y", "yes", "t", "true", "on"): + gc.SetString("color.ui", "auto") + + def _DisplayResult(self): + if self.manifest.IsMirror: + init_type = "mirror " + else: + init_type = "" - if not opt.quiet: print() - print('Your identity is: %s <%s>' % (name, email)) - print('is this correct [y/N]? ', end='', flush=True) - a = sys.stdin.readline().strip().lower() - if a in ('yes', 'y', 't', 'true'): - break - - if name != mp.UserName: - mp.config.SetString('user.name', name) - if email != mp.UserEmail: - mp.config.SetString('user.email', email) - - def _HasColorSet(self, gc): - for n in ['ui', 'diff', 'status']: - if gc.Has('color.%s' % n): - return True - return False - - def _ConfigureColor(self): - gc = self.client.globalConfig - if self._HasColorSet(gc): - return - - class _Test(Coloring): - def __init__(self): - Coloring.__init__(self, gc, 'test color display') - self._on = True - out = _Test() - - print() - print("Testing colorized output (for 'repo diff', 'repo status'):") - - for c in ['black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan']: - out.write(' ') - out.printer(fg=c)(' %-6s ', c) - out.write(' ') - out.printer(fg='white', bg='black')(' %s ' % 'white') - out.nl() - - for c in ['bold', 'dim', 'ul', 'reverse']: - out.write(' ') - out.printer(fg='black', attr=c)(' %-6s ', c) - out.nl() - - print('Enable color display in this user account (y/N)? ', end='', flush=True) - a = sys.stdin.readline().strip().lower() - if a in ('y', 'yes', 't', 'true', 'on'): - gc.SetString('color.ui', 'auto') - - def _DisplayResult(self): - if self.manifest.IsMirror: - init_type = 'mirror ' - else: - init_type = '' - - print() - print('repo %shas been initialized in %s' % (init_type, self.manifest.topdir)) - - current_dir = os.getcwd() - if current_dir != self.manifest.topdir: - print('If this is not the directory in which you want to initialize ' - 'repo, please run:') - print(' rm -r %s' % os.path.join(self.manifest.topdir, '.repo')) - print('and try again.') - - def ValidateOptions(self, opt, args): - if opt.reference: - opt.reference = os.path.expanduser(opt.reference) - - # Check this here, else manifest will be tagged "not new" and init won't be - # possible anymore without removing the .repo/manifests directory. - if opt.mirror: - if opt.archive: - self.OptionParser.error('--mirror and --archive cannot be used ' - 'together.') - if opt.use_superproject is not None: - self.OptionParser.error('--mirror and --use-superproject cannot be ' - 'used together.') - if opt.archive and opt.use_superproject is not None: - self.OptionParser.error('--archive and --use-superproject cannot be used ' - 'together.') - - if opt.standalone_manifest and (opt.manifest_branch or - opt.manifest_name != 'default.xml'): - self.OptionParser.error('--manifest-branch and --manifest-name cannot' - ' be used with --standalone-manifest.') - - if args: - if opt.manifest_url: - self.OptionParser.error( - '--manifest-url option and URL argument both specified: only use ' - 'one to select the manifest URL.') - - opt.manifest_url = args.pop(0) - - if args: - self.OptionParser.error('too many arguments to init') - - def Execute(self, opt, args): - git_require(MIN_GIT_VERSION_HARD, fail=True) - if not git_require(MIN_GIT_VERSION_SOFT): - print('repo: warning: git-%s+ will soon be required; please upgrade your ' - 'version of git to maintain support.' - % ('.'.join(str(x) for x in MIN_GIT_VERSION_SOFT),), - file=sys.stderr) - - rp = self.manifest.repoProject - - # Handle new --repo-url requests. - if opt.repo_url: - remote = rp.GetRemote('origin') - remote.url = opt.repo_url - remote.Save() - - # Handle new --repo-rev requests. - if opt.repo_rev: - wrapper = Wrapper() - try: - remote_ref, rev = wrapper.check_repo_rev( - rp.gitdir, opt.repo_rev, repo_verify=opt.repo_verify, quiet=opt.quiet) - except wrapper.CloneFailure: - print('fatal: double check your --repo-rev setting.', file=sys.stderr) - sys.exit(1) - branch = rp.GetBranch('default') - branch.merge = remote_ref - rp.work_git.reset('--hard', rev) - branch.Save() - - if opt.worktree: - # Older versions of git supported worktree, but had dangerous gc bugs. - git_require((2, 15, 0), fail=True, msg='git gc worktree corruption') - - # Provide a short notice that we're reinitializing an existing checkout. - # Sometimes developers might not realize that they're in one, or that - # repo doesn't do nested checkouts. - existing_checkout = self.manifest.manifestProject.Exists - if not opt.quiet and existing_checkout: - print('repo: reusing existing repo client checkout in', self.manifest.topdir) - - self._SyncManifest(opt) - - if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror: - if opt.config_name or self._ShouldConfigureUser(opt, existing_checkout): - self._ConfigureUser(opt) - self._ConfigureColor() - - if not opt.quiet: - self._DisplayResult() + print( + "repo %shas been initialized in %s" + % (init_type, self.manifest.topdir) + ) + + current_dir = os.getcwd() + if current_dir != self.manifest.topdir: + print( + "If this is not the directory in which you want to initialize " + "repo, please run:" + ) + print(" rm -r %s" % os.path.join(self.manifest.topdir, ".repo")) + print("and try again.") + + def ValidateOptions(self, opt, args): + if opt.reference: + opt.reference = os.path.expanduser(opt.reference) + + # Check this here, else manifest will be tagged "not new" and init won't + # be possible anymore without removing the .repo/manifests directory. + if opt.mirror: + if opt.archive: + self.OptionParser.error( + "--mirror and --archive cannot be used " "together." + ) + if opt.use_superproject is not None: + self.OptionParser.error( + "--mirror and --use-superproject cannot be " + "used together." + ) + if opt.archive and opt.use_superproject is not None: + self.OptionParser.error( + "--archive and --use-superproject cannot be used " "together." + ) + + if opt.standalone_manifest and ( + opt.manifest_branch or opt.manifest_name != "default.xml" + ): + self.OptionParser.error( + "--manifest-branch and --manifest-name cannot" + " be used with --standalone-manifest." + ) + + if args: + if opt.manifest_url: + self.OptionParser.error( + "--manifest-url option and URL argument both specified: " + "only use one to select the manifest URL." + ) + + opt.manifest_url = args.pop(0) + + if args: + self.OptionParser.error("too many arguments to init") + + def Execute(self, opt, args): + git_require(MIN_GIT_VERSION_HARD, fail=True) + if not git_require(MIN_GIT_VERSION_SOFT): + print( + "repo: warning: git-%s+ will soon be required; please upgrade " + "your version of git to maintain support." + % (".".join(str(x) for x in MIN_GIT_VERSION_SOFT),), + file=sys.stderr, + ) + + rp = self.manifest.repoProject + + # Handle new --repo-url requests. + if opt.repo_url: + remote = rp.GetRemote("origin") + remote.url = opt.repo_url + remote.Save() + + # Handle new --repo-rev requests. + if opt.repo_rev: + wrapper = Wrapper() + try: + remote_ref, rev = wrapper.check_repo_rev( + rp.gitdir, + opt.repo_rev, + repo_verify=opt.repo_verify, + quiet=opt.quiet, + ) + except wrapper.CloneFailure: + print( + "fatal: double check your --repo-rev setting.", + file=sys.stderr, + ) + sys.exit(1) + branch = rp.GetBranch("default") + branch.merge = remote_ref + rp.work_git.reset("--hard", rev) + branch.Save() + + if opt.worktree: + # Older versions of git supported worktree, but had dangerous gc + # bugs. + git_require((2, 15, 0), fail=True, msg="git gc worktree corruption") + + # Provide a short notice that we're reinitializing an existing checkout. + # Sometimes developers might not realize that they're in one, or that + # repo doesn't do nested checkouts. + existing_checkout = self.manifest.manifestProject.Exists + if not opt.quiet and existing_checkout: + print( + "repo: reusing existing repo client checkout in", + self.manifest.topdir, + ) + + self._SyncManifest(opt) + + if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror: + if opt.config_name or self._ShouldConfigureUser( + opt, existing_checkout + ): + self._ConfigureUser(opt) + self._ConfigureColor() + + if not opt.quiet: + self._DisplayResult() diff --git a/subcmds/list.py b/subcmds/list.py index ad8036ee..24e3e1fc 100644 --- a/subcmds/list.py +++ b/subcmds/list.py @@ -18,13 +18,13 @@ from command import Command, MirrorSafeCommand class List(Command, MirrorSafeCommand): - COMMON = True - helpSummary = "List projects and their associated directories" - helpUsage = """ + COMMON = True + helpSummary = "List projects and their associated directories" + helpUsage = """ %prog [-f] [...] %prog [-f] -r str1 [str2]... """ - helpDescription = """ + helpDescription = """ List all projects; pass '.' to list the project for the cwd. By default, only projects that currently exist in the checkout are shown. If @@ -35,69 +35,103 @@ groups, then also pass --groups all. This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'. """ - def _Options(self, p): - p.add_option('-r', '--regex', - dest='regex', action='store_true', - help='filter the project list based on regex or wildcard matching of strings') - p.add_option('-g', '--groups', - dest='groups', - help='filter the project list based on the groups the project is in') - p.add_option('-a', '--all', - action='store_true', - help='show projects regardless of checkout state') - p.add_option('-n', '--name-only', - dest='name_only', action='store_true', - help='display only the name of the repository') - p.add_option('-p', '--path-only', - dest='path_only', action='store_true', - help='display only the path of the repository') - p.add_option('-f', '--fullpath', - dest='fullpath', action='store_true', - help='display the full work tree path instead of the relative path') - p.add_option('--relative-to', metavar='PATH', - help='display paths relative to this one (default: top of repo client checkout)') + def _Options(self, p): + p.add_option( + "-r", + "--regex", + dest="regex", + action="store_true", + help="filter the project list based on regex or wildcard matching " + "of strings", + ) + p.add_option( + "-g", + "--groups", + dest="groups", + help="filter the project list based on the groups the project is " + "in", + ) + p.add_option( + "-a", + "--all", + action="store_true", + help="show projects regardless of checkout state", + ) + p.add_option( + "-n", + "--name-only", + dest="name_only", + action="store_true", + help="display only the name of the repository", + ) + p.add_option( + "-p", + "--path-only", + dest="path_only", + action="store_true", + help="display only the path of the repository", + ) + p.add_option( + "-f", + "--fullpath", + dest="fullpath", + action="store_true", + help="display the full work tree path instead of the relative path", + ) + p.add_option( + "--relative-to", + metavar="PATH", + help="display paths relative to this one (default: top of repo " + "client checkout)", + ) - def ValidateOptions(self, opt, args): - if opt.fullpath and opt.name_only: - self.OptionParser.error('cannot combine -f and -n') + def ValidateOptions(self, opt, args): + if opt.fullpath and opt.name_only: + self.OptionParser.error("cannot combine -f and -n") - # Resolve any symlinks so the output is stable. - if opt.relative_to: - opt.relative_to = os.path.realpath(opt.relative_to) + # Resolve any symlinks so the output is stable. + if opt.relative_to: + opt.relative_to = os.path.realpath(opt.relative_to) - def Execute(self, opt, args): - """List all projects and the associated directories. + def Execute(self, opt, args): + """List all projects and the associated directories. - This may be possible to do with 'repo forall', but repo newbies have - trouble figuring that out. The idea here is that it should be more - discoverable. + This may be possible to do with 'repo forall', but repo newbies have + trouble figuring that out. The idea here is that it should be more + discoverable. - Args: - opt: The options. - args: Positional args. Can be a list of projects to list, or empty. - """ - if not opt.regex: - projects = self.GetProjects(args, groups=opt.groups, missing_ok=opt.all, - all_manifests=not opt.this_manifest_only) - else: - projects = self.FindProjects(args, all_manifests=not opt.this_manifest_only) + Args: + opt: The options. + args: Positional args. Can be a list of projects to list, or empty. + """ + if not opt.regex: + projects = self.GetProjects( + args, + groups=opt.groups, + missing_ok=opt.all, + all_manifests=not opt.this_manifest_only, + ) + else: + projects = self.FindProjects( + args, all_manifests=not opt.this_manifest_only + ) - def _getpath(x): - if opt.fullpath: - return x.worktree - if opt.relative_to: - return os.path.relpath(x.worktree, opt.relative_to) - return x.RelPath(local=opt.this_manifest_only) + def _getpath(x): + if opt.fullpath: + return x.worktree + if opt.relative_to: + return os.path.relpath(x.worktree, opt.relative_to) + return x.RelPath(local=opt.this_manifest_only) - lines = [] - for project in projects: - if opt.name_only and not opt.path_only: - lines.append("%s" % (project.name)) - elif opt.path_only and not opt.name_only: - lines.append("%s" % (_getpath(project))) - else: - lines.append("%s : %s" % (_getpath(project), project.name)) + lines = [] + for project in projects: + if opt.name_only and not opt.path_only: + lines.append("%s" % (project.name)) + elif opt.path_only and not opt.name_only: + lines.append("%s" % (_getpath(project))) + else: + lines.append("%s : %s" % (_getpath(project), project.name)) - if lines: - lines.sort() - print('\n'.join(lines)) + if lines: + lines.sort() + print("\n".join(lines)) diff --git a/subcmds/manifest.py b/subcmds/manifest.py index f4602a59..f72df348 100644 --- a/subcmds/manifest.py +++ b/subcmds/manifest.py @@ -20,12 +20,12 @@ from command import PagedCommand class Manifest(PagedCommand): - COMMON = False - helpSummary = "Manifest inspection utility" - helpUsage = """ + COMMON = False + helpSummary = "Manifest inspection utility" + helpUsage = """ %prog [-o {-|NAME.xml}] [-m MANIFEST.xml] [-r] """ - _helpDescription = """ + _helpDescription = """ With the -o option, exports the current manifest for inspection. The manifest and (if present) local_manifests/ are combined @@ -40,92 +40,136 @@ when the manifest was generated. The 'dest-branch' attribute is set to indicate the remote ref to push changes to via 'repo upload'. """ - @property - def helpDescription(self): - helptext = self._helpDescription + '\n' - r = os.path.dirname(__file__) - r = os.path.dirname(r) - with open(os.path.join(r, 'docs', 'manifest-format.md')) as fd: - for line in fd: - helptext += line - return helptext - - def _Options(self, p): - p.add_option('-r', '--revision-as-HEAD', - dest='peg_rev', action='store_true', - help='save revisions as current HEAD') - p.add_option('-m', '--manifest-name', - help='temporary manifest to use for this sync', metavar='NAME.xml') - p.add_option('--suppress-upstream-revision', dest='peg_rev_upstream', - default=True, action='store_false', - help='if in -r mode, do not write the upstream field ' - '(only of use if the branch names for a sha1 manifest are ' - 'sensitive)') - p.add_option('--suppress-dest-branch', dest='peg_rev_dest_branch', - default=True, action='store_false', - help='if in -r mode, do not write the dest-branch field ' - '(only of use if the branch names for a sha1 manifest are ' - 'sensitive)') - p.add_option('--json', default=False, action='store_true', - help='output manifest in JSON format (experimental)') - p.add_option('--pretty', default=False, action='store_true', - help='format output for humans to read') - p.add_option('--no-local-manifests', default=False, action='store_true', - dest='ignore_local_manifests', help='ignore local manifests') - p.add_option('-o', '--output-file', - dest='output_file', - default='-', - help='file to save the manifest to. (Filename prefix for multi-tree.)', - metavar='-|NAME.xml') - - def _Output(self, opt): - # If alternate manifest is specified, override the manifest file that we're using. - if opt.manifest_name: - self.manifest.Override(opt.manifest_name, False) - - for manifest in self.ManifestList(opt): - output_file = opt.output_file - if output_file == '-': - fd = sys.stdout - else: - if manifest.path_prefix: - output_file = f'{opt.output_file}:{manifest.path_prefix.replace("/", "%2f")}' - fd = open(output_file, 'w') - - manifest.SetUseLocalManifests(not opt.ignore_local_manifests) - - if opt.json: - print('warning: --json is experimental!', file=sys.stderr) - doc = manifest.ToDict(peg_rev=opt.peg_rev, - peg_rev_upstream=opt.peg_rev_upstream, - peg_rev_dest_branch=opt.peg_rev_dest_branch) - - json_settings = { - # JSON style guide says Uunicode characters are fully allowed. - 'ensure_ascii': False, - # We use 2 space indent to match JSON style guide. - 'indent': 2 if opt.pretty else None, - 'separators': (',', ': ') if opt.pretty else (',', ':'), - 'sort_keys': True, - } - fd.write(json.dumps(doc, **json_settings)) - else: - manifest.Save(fd, - peg_rev=opt.peg_rev, - peg_rev_upstream=opt.peg_rev_upstream, - peg_rev_dest_branch=opt.peg_rev_dest_branch) - if output_file != '-': - fd.close() - if manifest.path_prefix: - print(f'Saved {manifest.path_prefix} submanifest to {output_file}', - file=sys.stderr) - else: - print(f'Saved manifest to {output_file}', file=sys.stderr) - - - def ValidateOptions(self, opt, args): - if args: - self.Usage() - - def Execute(self, opt, args): - self._Output(opt) + @property + def helpDescription(self): + helptext = self._helpDescription + "\n" + r = os.path.dirname(__file__) + r = os.path.dirname(r) + with open(os.path.join(r, "docs", "manifest-format.md")) as fd: + for line in fd: + helptext += line + return helptext + + def _Options(self, p): + p.add_option( + "-r", + "--revision-as-HEAD", + dest="peg_rev", + action="store_true", + help="save revisions as current HEAD", + ) + p.add_option( + "-m", + "--manifest-name", + help="temporary manifest to use for this sync", + metavar="NAME.xml", + ) + p.add_option( + "--suppress-upstream-revision", + dest="peg_rev_upstream", + default=True, + action="store_false", + help="if in -r mode, do not write the upstream field " + "(only of use if the branch names for a sha1 manifest are " + "sensitive)", + ) + p.add_option( + "--suppress-dest-branch", + dest="peg_rev_dest_branch", + default=True, + action="store_false", + help="if in -r mode, do not write the dest-branch field " + "(only of use if the branch names for a sha1 manifest are " + "sensitive)", + ) + p.add_option( + "--json", + default=False, + action="store_true", + help="output manifest in JSON format (experimental)", + ) + p.add_option( + "--pretty", + default=False, + action="store_true", + help="format output for humans to read", + ) + p.add_option( + "--no-local-manifests", + default=False, + action="store_true", + dest="ignore_local_manifests", + help="ignore local manifests", + ) + p.add_option( + "-o", + "--output-file", + dest="output_file", + default="-", + help="file to save the manifest to. (Filename prefix for " + "multi-tree.)", + metavar="-|NAME.xml", + ) + + def _Output(self, opt): + # If alternate manifest is specified, override the manifest file that + # we're using. + if opt.manifest_name: + self.manifest.Override(opt.manifest_name, False) + + for manifest in self.ManifestList(opt): + output_file = opt.output_file + if output_file == "-": + fd = sys.stdout + else: + if manifest.path_prefix: + output_file = ( + f"{opt.output_file}:" + f'{manifest.path_prefix.replace("/", "%2f")}' + ) + fd = open(output_file, "w") + + manifest.SetUseLocalManifests(not opt.ignore_local_manifests) + + if opt.json: + print("warning: --json is experimental!", file=sys.stderr) + doc = manifest.ToDict( + peg_rev=opt.peg_rev, + peg_rev_upstream=opt.peg_rev_upstream, + peg_rev_dest_branch=opt.peg_rev_dest_branch, + ) + + json_settings = { + # JSON style guide says Unicode characters are fully + # allowed. + "ensure_ascii": False, + # We use 2 space indent to match JSON style guide. + "indent": 2 if opt.pretty else None, + "separators": (",", ": ") if opt.pretty else (",", ":"), + "sort_keys": True, + } + fd.write(json.dumps(doc, **json_settings)) + else: + manifest.Save( + fd, + peg_rev=opt.peg_rev, + peg_rev_upstream=opt.peg_rev_upstream, + peg_rev_dest_branch=opt.peg_rev_dest_branch, + ) + if output_file != "-": + fd.close() + if manifest.path_prefix: + print( + f"Saved {manifest.path_prefix} submanifest to " + f"{output_file}", + file=sys.stderr, + ) + else: + print(f"Saved manifest to {output_file}", file=sys.stderr) + + def ValidateOptions(self, opt, args): + if args: + self.Usage() + + def Execute(self, opt, args): + self._Output(opt) diff --git a/subcmds/overview.py b/subcmds/overview.py index 11dba95f..8ccad611 100644 --- a/subcmds/overview.py +++ b/subcmds/overview.py @@ -19,12 +19,12 @@ from command import PagedCommand class Overview(PagedCommand): - COMMON = True - helpSummary = "Display overview of unmerged project branches" - helpUsage = """ + COMMON = True + helpSummary = "Display overview of unmerged project branches" + helpUsage = """ %prog [--current-branch] [...] """ - helpDescription = """ + helpDescription = """ The '%prog' command is used to display an overview of the projects branches, and list any local commits that have not yet been merged into the project. @@ -33,59 +33,77 @@ branches currently checked out in each project. By default, all branches are displayed. """ - def _Options(self, p): - p.add_option('-c', '--current-branch', - dest="current_branch", action="store_true", - help="consider only checked out branches") - p.add_option('--no-current-branch', - dest='current_branch', action='store_false', - help='consider all local branches') - # Turn this into a warning & remove this someday. - p.add_option('-b', - dest='current_branch', action='store_true', - help=optparse.SUPPRESS_HELP) + def _Options(self, p): + p.add_option( + "-c", + "--current-branch", + dest="current_branch", + action="store_true", + help="consider only checked out branches", + ) + p.add_option( + "--no-current-branch", + dest="current_branch", + action="store_false", + help="consider all local branches", + ) + # Turn this into a warning & remove this someday. + p.add_option( + "-b", + dest="current_branch", + action="store_true", + help=optparse.SUPPRESS_HELP, + ) - def Execute(self, opt, args): - all_branches = [] - for project in self.GetProjects(args, all_manifests=not opt.this_manifest_only): - br = [project.GetUploadableBranch(x) - for x in project.GetBranches()] - br = [x for x in br if x] - if opt.current_branch: - br = [x for x in br if x.name == project.CurrentBranch] - all_branches.extend(br) + def Execute(self, opt, args): + all_branches = [] + for project in self.GetProjects( + args, all_manifests=not opt.this_manifest_only + ): + br = [project.GetUploadableBranch(x) for x in project.GetBranches()] + br = [x for x in br if x] + if opt.current_branch: + br = [x for x in br if x.name == project.CurrentBranch] + all_branches.extend(br) - if not all_branches: - return + if not all_branches: + return - class Report(Coloring): - def __init__(self, config): - Coloring.__init__(self, config, 'status') - self.project = self.printer('header', attr='bold') - self.text = self.printer('text') + class Report(Coloring): + def __init__(self, config): + Coloring.__init__(self, config, "status") + self.project = self.printer("header", attr="bold") + self.text = self.printer("text") - out = Report(all_branches[0].project.config) - out.text("Deprecated. See repo info -o.") - out.nl() - out.project('Projects Overview') - out.nl() - - project = None - - for branch in all_branches: - if project != branch.project: - project = branch.project + out = Report(all_branches[0].project.config) + out.text("Deprecated. See repo info -o.") out.nl() - out.project('project %s/' % project.RelPath(local=opt.this_manifest_only)) + out.project("Projects Overview") out.nl() - commits = branch.commits - date = branch.date - print('%s %-33s (%2d commit%s, %s)' % ( - branch.name == project.CurrentBranch and '*' or ' ', - branch.name, - len(commits), - len(commits) != 1 and 's' or ' ', - date)) - for commit in commits: - print('%-35s - %s' % ('', commit)) + project = None + + for branch in all_branches: + if project != branch.project: + project = branch.project + out.nl() + out.project( + "project %s/" + % project.RelPath(local=opt.this_manifest_only) + ) + out.nl() + + commits = branch.commits + date = branch.date + print( + "%s %-33s (%2d commit%s, %s)" + % ( + branch.name == project.CurrentBranch and "*" or " ", + branch.name, + len(commits), + len(commits) != 1 and "s" or " ", + date, + ) + ) + for commit in commits: + print("%-35s - %s" % ("", commit)) diff --git a/subcmds/prune.py b/subcmds/prune.py index 251accaa..5a68c14a 100644 --- a/subcmds/prune.py +++ b/subcmds/prune.py @@ -19,63 +19,76 @@ from command import DEFAULT_LOCAL_JOBS, PagedCommand class Prune(PagedCommand): - COMMON = True - helpSummary = "Prune (delete) already merged topics" - helpUsage = """ + COMMON = True + helpSummary = "Prune (delete) already merged topics" + helpUsage = """ %prog [...] """ - PARALLEL_JOBS = DEFAULT_LOCAL_JOBS + PARALLEL_JOBS = DEFAULT_LOCAL_JOBS - def _ExecuteOne(self, project): - """Process one project.""" - return project.PruneHeads() + def _ExecuteOne(self, project): + """Process one project.""" + return project.PruneHeads() - def Execute(self, opt, args): - projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only) + def Execute(self, opt, args): + projects = self.GetProjects( + args, all_manifests=not opt.this_manifest_only + ) - # NB: Should be able to refactor this module to display summary as results - # come back from children. - def _ProcessResults(_pool, _output, results): - return list(itertools.chain.from_iterable(results)) + # NB: Should be able to refactor this module to display summary as + # results come back from children. + def _ProcessResults(_pool, _output, results): + return list(itertools.chain.from_iterable(results)) - all_branches = self.ExecuteInParallel( - opt.jobs, - self._ExecuteOne, - projects, - callback=_ProcessResults, - ordered=True) + all_branches = self.ExecuteInParallel( + opt.jobs, + self._ExecuteOne, + projects, + callback=_ProcessResults, + ordered=True, + ) - if not all_branches: - return + if not all_branches: + return - class Report(Coloring): - def __init__(self, config): - Coloring.__init__(self, config, 'status') - self.project = self.printer('header', attr='bold') + class Report(Coloring): + def __init__(self, config): + Coloring.__init__(self, config, "status") + self.project = self.printer("header", attr="bold") - out = Report(all_branches[0].project.config) - out.project('Pending Branches') - out.nl() + out = Report(all_branches[0].project.config) + out.project("Pending Branches") + out.nl() - project = None + project = None - for branch in all_branches: - if project != branch.project: - project = branch.project - out.nl() - out.project('project %s/' % project.RelPath(local=opt.this_manifest_only)) - out.nl() + for branch in all_branches: + if project != branch.project: + project = branch.project + out.nl() + out.project( + "project %s/" + % project.RelPath(local=opt.this_manifest_only) + ) + out.nl() - print('%s %-33s ' % ( - branch.name == project.CurrentBranch and '*' or ' ', - branch.name), end='') + print( + "%s %-33s " + % ( + branch.name == project.CurrentBranch and "*" or " ", + branch.name, + ), + end="", + ) - if not branch.base_exists: - print('(ignoring: tracking branch is gone: %s)' % (branch.base,)) - else: - commits = branch.commits - date = branch.date - print('(%2d commit%s, %s)' % ( - len(commits), - len(commits) != 1 and 's' or ' ', - date)) + if not branch.base_exists: + print( + "(ignoring: tracking branch is gone: %s)" % (branch.base,) + ) + else: + commits = branch.commits + date = branch.date + print( + "(%2d commit%s, %s)" + % (len(commits), len(commits) != 1 and "s" or " ", date) + ) diff --git a/subcmds/rebase.py b/subcmds/rebase.py index 3d1a63e4..dc4f5805 100644 --- a/subcmds/rebase.py +++ b/subcmds/rebase.py @@ -20,146 +20,193 @@ from git_command import GitCommand class RebaseColoring(Coloring): - def __init__(self, config): - Coloring.__init__(self, config, 'rebase') - self.project = self.printer('project', attr='bold') - self.fail = self.printer('fail', fg='red') + def __init__(self, config): + Coloring.__init__(self, config, "rebase") + self.project = self.printer("project", attr="bold") + self.fail = self.printer("fail", fg="red") class Rebase(Command): - COMMON = True - helpSummary = "Rebase local branches on upstream branch" - helpUsage = """ + COMMON = True + helpSummary = "Rebase local branches on upstream branch" + helpUsage = """ %prog {[...] | -i ...} """ - helpDescription = """ + helpDescription = """ '%prog' uses git rebase to move local changes in the current topic branch to the HEAD of the upstream history, useful when you have made commits in a topic branch but need to incorporate new upstream changes "underneath" them. """ - def _Options(self, p): - g = p.get_option_group('--quiet') - g.add_option('-i', '--interactive', - dest="interactive", action="store_true", - help="interactive rebase (single project only)") - - p.add_option('--fail-fast', - dest='fail_fast', action='store_true', - help='stop rebasing after first error is hit') - p.add_option('-f', '--force-rebase', - dest='force_rebase', action='store_true', - help='pass --force-rebase to git rebase') - p.add_option('--no-ff', - dest='ff', default=True, action='store_false', - help='pass --no-ff to git rebase') - p.add_option('--autosquash', - dest='autosquash', action='store_true', - help='pass --autosquash to git rebase') - p.add_option('--whitespace', - dest='whitespace', action='store', metavar='WS', - help='pass --whitespace to git rebase') - p.add_option('--auto-stash', - dest='auto_stash', action='store_true', - help='stash local modifications before starting') - p.add_option('-m', '--onto-manifest', - dest='onto_manifest', action='store_true', - help='rebase onto the manifest version instead of upstream ' - 'HEAD (this helps to make sure the local tree stays ' - 'consistent if you previously synced to a manifest)') - - def Execute(self, opt, args): - all_projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only) - one_project = len(all_projects) == 1 - - if opt.interactive and not one_project: - print('error: interactive rebase not supported with multiple projects', - file=sys.stderr) - if len(args) == 1: - print('note: project %s is mapped to more than one path' % (args[0],), - file=sys.stderr) - return 1 - - # Setup the common git rebase args that we use for all projects. - common_args = ['rebase'] - if opt.whitespace: - common_args.append('--whitespace=%s' % opt.whitespace) - if opt.quiet: - common_args.append('--quiet') - if opt.force_rebase: - common_args.append('--force-rebase') - if not opt.ff: - common_args.append('--no-ff') - if opt.autosquash: - common_args.append('--autosquash') - if opt.interactive: - common_args.append('-i') - - config = self.manifest.manifestProject.config - out = RebaseColoring(config) - out.redirect(sys.stdout) - _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only) - - ret = 0 - for project in all_projects: - if ret and opt.fail_fast: - break - - cb = project.CurrentBranch - if not cb: - if one_project: - print("error: project %s has a detached HEAD" % _RelPath(project), - file=sys.stderr) - return 1 - # ignore branches with detatched HEADs - continue - - upbranch = project.GetBranch(cb) - if not upbranch.LocalMerge: - if one_project: - print("error: project %s does not track any remote branches" - % _RelPath(project), file=sys.stderr) - return 1 - # ignore branches without remotes - continue - - args = common_args[:] - if opt.onto_manifest: - args.append('--onto') - args.append(project.revisionExpr) - - args.append(upbranch.LocalMerge) - - out.project('project %s: rebasing %s -> %s', - _RelPath(project), cb, upbranch.LocalMerge) - out.nl() - out.flush() - - needs_stash = False - if opt.auto_stash: - stash_args = ["update-index", "--refresh", "-q"] - - if GitCommand(project, stash_args).Wait() != 0: - needs_stash = True - # Dirty index, requires stash... - stash_args = ["stash"] - - if GitCommand(project, stash_args).Wait() != 0: - ret += 1 - continue - - if GitCommand(project, args).Wait() != 0: - ret += 1 - continue - - if needs_stash: - stash_args.append('pop') - stash_args.append('--quiet') - if GitCommand(project, stash_args).Wait() != 0: - ret += 1 - - if ret: - out.fail('%i projects had errors', ret) - out.nl() - - return ret + def _Options(self, p): + g = p.get_option_group("--quiet") + g.add_option( + "-i", + "--interactive", + dest="interactive", + action="store_true", + help="interactive rebase (single project only)", + ) + + p.add_option( + "--fail-fast", + dest="fail_fast", + action="store_true", + help="stop rebasing after first error is hit", + ) + p.add_option( + "-f", + "--force-rebase", + dest="force_rebase", + action="store_true", + help="pass --force-rebase to git rebase", + ) + p.add_option( + "--no-ff", + dest="ff", + default=True, + action="store_false", + help="pass --no-ff to git rebase", + ) + p.add_option( + "--autosquash", + dest="autosquash", + action="store_true", + help="pass --autosquash to git rebase", + ) + p.add_option( + "--whitespace", + dest="whitespace", + action="store", + metavar="WS", + help="pass --whitespace to git rebase", + ) + p.add_option( + "--auto-stash", + dest="auto_stash", + action="store_true", + help="stash local modifications before starting", + ) + p.add_option( + "-m", + "--onto-manifest", + dest="onto_manifest", + action="store_true", + help="rebase onto the manifest version instead of upstream " + "HEAD (this helps to make sure the local tree stays " + "consistent if you previously synced to a manifest)", + ) + + def Execute(self, opt, args): + all_projects = self.GetProjects( + args, all_manifests=not opt.this_manifest_only + ) + one_project = len(all_projects) == 1 + + if opt.interactive and not one_project: + print( + "error: interactive rebase not supported with multiple " + "projects", + file=sys.stderr, + ) + if len(args) == 1: + print( + "note: project %s is mapped to more than one path" + % (args[0],), + file=sys.stderr, + ) + return 1 + + # Setup the common git rebase args that we use for all projects. + common_args = ["rebase"] + if opt.whitespace: + common_args.append("--whitespace=%s" % opt.whitespace) + if opt.quiet: + common_args.append("--quiet") + if opt.force_rebase: + common_args.append("--force-rebase") + if not opt.ff: + common_args.append("--no-ff") + if opt.autosquash: + common_args.append("--autosquash") + if opt.interactive: + common_args.append("-i") + + config = self.manifest.manifestProject.config + out = RebaseColoring(config) + out.redirect(sys.stdout) + _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only) + + ret = 0 + for project in all_projects: + if ret and opt.fail_fast: + break + + cb = project.CurrentBranch + if not cb: + if one_project: + print( + "error: project %s has a detached HEAD" + % _RelPath(project), + file=sys.stderr, + ) + return 1 + # Ignore branches with detached HEADs. + continue + + upbranch = project.GetBranch(cb) + if not upbranch.LocalMerge: + if one_project: + print( + "error: project %s does not track any remote branches" + % _RelPath(project), + file=sys.stderr, + ) + return 1 + # Ignore branches without remotes. + continue + + args = common_args[:] + if opt.onto_manifest: + args.append("--onto") + args.append(project.revisionExpr) + + args.append(upbranch.LocalMerge) + + out.project( + "project %s: rebasing %s -> %s", + _RelPath(project), + cb, + upbranch.LocalMerge, + ) + out.nl() + out.flush() + + needs_stash = False + if opt.auto_stash: + stash_args = ["update-index", "--refresh", "-q"] + + if GitCommand(project, stash_args).Wait() != 0: + needs_stash = True + # Dirty index, requires stash... + stash_args = ["stash"] + + if GitCommand(project, stash_args).Wait() != 0: + ret += 1 + continue + + if GitCommand(project, args).Wait() != 0: + ret += 1 + continue + + if needs_stash: + stash_args.append("pop") + stash_args.append("--quiet") + if GitCommand(project, stash_args).Wait() != 0: + ret += 1 + + if ret: + out.fail("%i projects had errors", ret) + out.nl() + + return ret diff --git a/subcmds/selfupdate.py b/subcmds/selfupdate.py index 898bc3f2..d5d0a838 100644 --- a/subcmds/selfupdate.py +++ b/subcmds/selfupdate.py @@ -21,12 +21,12 @@ from subcmds.sync import _PostRepoFetch class Selfupdate(Command, MirrorSafeCommand): - COMMON = False - helpSummary = "Update repo to the latest version" - helpUsage = """ + COMMON = False + helpSummary = "Update repo to the latest version" + helpUsage = """ %prog """ - helpDescription = """ + helpDescription = """ The '%prog' command upgrades repo to the latest version, if a newer version is available. @@ -34,28 +34,33 @@ Normally this is done automatically by 'repo sync' and does not need to be performed by an end-user. """ - def _Options(self, p): - g = p.add_option_group('repo Version options') - g.add_option('--no-repo-verify', - dest='repo_verify', default=True, action='store_false', - help='do not verify repo source code') - g.add_option('--repo-upgraded', - dest='repo_upgraded', action='store_true', - help=SUPPRESS_HELP) - - def Execute(self, opt, args): - rp = self.manifest.repoProject - rp.PreSync() - - if opt.repo_upgraded: - _PostRepoUpgrade(self.manifest) - - else: - if not rp.Sync_NetworkHalf().success: - print("error: can't update repo", file=sys.stderr) - sys.exit(1) - - rp.bare_git.gc('--auto') - _PostRepoFetch(rp, - repo_verify=opt.repo_verify, - verbose=True) + def _Options(self, p): + g = p.add_option_group("repo Version options") + g.add_option( + "--no-repo-verify", + dest="repo_verify", + default=True, + action="store_false", + help="do not verify repo source code", + ) + g.add_option( + "--repo-upgraded", + dest="repo_upgraded", + action="store_true", + help=SUPPRESS_HELP, + ) + + def Execute(self, opt, args): + rp = self.manifest.repoProject + rp.PreSync() + + if opt.repo_upgraded: + _PostRepoUpgrade(self.manifest) + + else: + if not rp.Sync_NetworkHalf().success: + print("error: can't update repo", file=sys.stderr) + sys.exit(1) + + rp.bare_git.gc("--auto") + _PostRepoFetch(rp, repo_verify=opt.repo_verify, verbose=True) diff --git a/subcmds/smartsync.py b/subcmds/smartsync.py index d91d59c6..49d09972 100644 --- a/subcmds/smartsync.py +++ b/subcmds/smartsync.py @@ -16,18 +16,18 @@ from subcmds.sync import Sync class Smartsync(Sync): - COMMON = True - helpSummary = "Update working tree to the latest known good revision" - helpUsage = """ + COMMON = True + helpSummary = "Update working tree to the latest known good revision" + helpUsage = """ %prog [...] """ - helpDescription = """ + helpDescription = """ The '%prog' command is a shortcut for sync -s. """ - def _Options(self, p): - Sync._Options(self, p, show_smart=False) + def _Options(self, p): + Sync._Options(self, p, show_smart=False) - def Execute(self, opt, args): - opt.smart_sync = True - Sync.Execute(self, opt, args) + def Execute(self, opt, args): + opt.smart_sync = True + Sync.Execute(self, opt, args) diff --git a/subcmds/stage.py b/subcmds/stage.py index bdb72012..4d54eb19 100644 --- a/subcmds/stage.py +++ b/subcmds/stage.py @@ -20,98 +20,111 @@ from git_command import GitCommand class _ProjectList(Coloring): - def __init__(self, gc): - Coloring.__init__(self, gc, 'interactive') - self.prompt = self.printer('prompt', fg='blue', attr='bold') - self.header = self.printer('header', attr='bold') - self.help = self.printer('help', fg='red', attr='bold') + def __init__(self, gc): + Coloring.__init__(self, gc, "interactive") + self.prompt = self.printer("prompt", fg="blue", attr="bold") + self.header = self.printer("header", attr="bold") + self.help = self.printer("help", fg="red", attr="bold") class Stage(InteractiveCommand): - COMMON = True - helpSummary = "Stage file(s) for commit" - helpUsage = """ + COMMON = True + helpSummary = "Stage file(s) for commit" + helpUsage = """ %prog -i [...] """ - helpDescription = """ + helpDescription = """ The '%prog' command stages files to prepare the next commit. """ - def _Options(self, p): - g = p.get_option_group('--quiet') - g.add_option('-i', '--interactive', - dest='interactive', action='store_true', - help='use interactive staging') - - def Execute(self, opt, args): - if opt.interactive: - self._Interactive(opt, args) - else: - self.Usage() - - def _Interactive(self, opt, args): - all_projects = [ - p for p in self.GetProjects(args, all_manifests=not opt.this_manifest_only) - if p.IsDirty()] - if not all_projects: - print('no projects have uncommitted modifications', file=sys.stderr) - return - - out = _ProjectList(self.manifest.manifestProject.config) - while True: - out.header(' %s', 'project') - out.nl() - - for i in range(len(all_projects)): - project = all_projects[i] - out.write('%3d: %s', i + 1, - project.RelPath(local=opt.this_manifest_only) + '/') - out.nl() - out.nl() - - out.write('%3d: (', 0) - out.prompt('q') - out.write('uit)') - out.nl() - - out.prompt('project> ') - out.flush() - try: - a = sys.stdin.readline() - except KeyboardInterrupt: - out.nl() - break - if a == '': - out.nl() - break - - a = a.strip() - if a.lower() in ('q', 'quit', 'exit'): - break - if not a: - continue - - try: - a_index = int(a) - except ValueError: - a_index = None - - if a_index is not None: - if a_index == 0: - break - if 0 < a_index and a_index <= len(all_projects): - _AddI(all_projects[a_index - 1]) - continue - - projects = [ - p for p in all_projects - if a in [p.name, p.RelPath(local=opt.this_manifest_only)]] - if len(projects) == 1: - _AddI(projects[0]) - continue - print('Bye.') + def _Options(self, p): + g = p.get_option_group("--quiet") + g.add_option( + "-i", + "--interactive", + dest="interactive", + action="store_true", + help="use interactive staging", + ) + + def Execute(self, opt, args): + if opt.interactive: + self._Interactive(opt, args) + else: + self.Usage() + + def _Interactive(self, opt, args): + all_projects = [ + p + for p in self.GetProjects( + args, all_manifests=not opt.this_manifest_only + ) + if p.IsDirty() + ] + if not all_projects: + print("no projects have uncommitted modifications", file=sys.stderr) + return + + out = _ProjectList(self.manifest.manifestProject.config) + while True: + out.header(" %s", "project") + out.nl() + + for i in range(len(all_projects)): + project = all_projects[i] + out.write( + "%3d: %s", + i + 1, + project.RelPath(local=opt.this_manifest_only) + "/", + ) + out.nl() + out.nl() + + out.write("%3d: (", 0) + out.prompt("q") + out.write("uit)") + out.nl() + + out.prompt("project> ") + out.flush() + try: + a = sys.stdin.readline() + except KeyboardInterrupt: + out.nl() + break + if a == "": + out.nl() + break + + a = a.strip() + if a.lower() in ("q", "quit", "exit"): + break + if not a: + continue + + try: + a_index = int(a) + except ValueError: + a_index = None + + if a_index is not None: + if a_index == 0: + break + if 0 < a_index and a_index <= len(all_projects): + _AddI(all_projects[a_index - 1]) + continue + + projects = [ + p + for p in all_projects + if a in [p.name, p.RelPath(local=opt.this_manifest_only)] + ] + if len(projects) == 1: + _AddI(projects[0]) + continue + print("Bye.") def _AddI(project): - p = GitCommand(project, ['add', '--interactive'], bare=False) - p.Wait() + p = GitCommand(project, ["add", "--interactive"], bare=False) + p.Wait() diff --git a/subcmds/start.py b/subcmds/start.py index 809df963..d7772b33 100644 --- a/subcmds/start.py +++ b/subcmds/start.py @@ -25,119 +25,147 @@ from project import SyncBuffer class Start(Command): - COMMON = True - helpSummary = "Start a new branch for development" - helpUsage = """ + COMMON = True + helpSummary = "Start a new branch for development" + helpUsage = """ %prog [--all | ...] """ - helpDescription = """ + helpDescription = """ '%prog' begins a new branch of development, starting from the revision specified in the manifest. """ - PARALLEL_JOBS = DEFAULT_LOCAL_JOBS - - def _Options(self, p): - p.add_option('--all', - dest='all', action='store_true', - help='begin branch in all projects') - p.add_option('-r', '--rev', '--revision', dest='revision', - help='point branch at this revision instead of upstream') - p.add_option('--head', '--HEAD', - dest='revision', action='store_const', const='HEAD', - help='abbreviation for --rev HEAD') - - def ValidateOptions(self, opt, args): - if not args: - self.Usage() - - nb = args[0] - if not git.check_ref_format('heads/%s' % nb): - self.OptionParser.error("'%s' is not a valid name" % nb) - - def _ExecuteOne(self, revision, nb, project): - """Start one project.""" - # If the current revision is immutable, such as a SHA1, a tag or - # a change, then we can't push back to it. Substitute with - # dest_branch, if defined; or with manifest default revision instead. - branch_merge = '' - if IsImmutable(project.revisionExpr): - if project.dest_branch: - branch_merge = project.dest_branch - else: - branch_merge = self.manifest.default.revisionExpr - - try: - ret = project.StartBranch( - nb, branch_merge=branch_merge, revision=revision) - except Exception as e: - print('error: unable to checkout %s: %s' % (project.name, e), file=sys.stderr) - ret = False - return (ret, project) - - def Execute(self, opt, args): - nb = args[0] - err = [] - projects = [] - if not opt.all: - projects = args[1:] - if len(projects) < 1: - projects = ['.'] # start it in the local project by default - - all_projects = self.GetProjects(projects, - missing_ok=bool(self.gitc_manifest), - all_manifests=not opt.this_manifest_only) - - # This must happen after we find all_projects, since GetProjects may need - # the local directory, which will disappear once we save the GITC manifest. - if self.gitc_manifest: - gitc_projects = self.GetProjects(projects, manifest=self.gitc_manifest, - missing_ok=True) - for project in gitc_projects: - if project.old_revision: - project.already_synced = True - else: - project.already_synced = False - project.old_revision = project.revisionExpr - project.revisionExpr = None - # Save the GITC manifest. - gitc_utils.save_manifest(self.gitc_manifest) - - # Make sure we have a valid CWD - if not os.path.exists(os.getcwd()): - os.chdir(self.manifest.topdir) - - pm = Progress('Syncing %s' % nb, len(all_projects), quiet=opt.quiet) - for project in all_projects: - gitc_project = self.gitc_manifest.paths[project.relpath] - # Sync projects that have not been opened. - if not gitc_project.already_synced: - proj_localdir = os.path.join(self.gitc_manifest.gitc_client_dir, - project.relpath) - project.worktree = proj_localdir - if not os.path.exists(proj_localdir): - os.makedirs(proj_localdir) - project.Sync_NetworkHalf() - sync_buf = SyncBuffer(self.manifest.manifestProject.config) - project.Sync_LocalHalf(sync_buf) - project.revisionId = gitc_project.old_revision - pm.update() - pm.end() - - def _ProcessResults(_pool, pm, results): - for (result, project) in results: - if not result: - err.append(project) - pm.update() - - self.ExecuteInParallel( - opt.jobs, - functools.partial(self._ExecuteOne, opt.revision, nb), - all_projects, - callback=_ProcessResults, - output=Progress('Starting %s' % (nb,), len(all_projects), quiet=opt.quiet)) - - if err: - for p in err: - print("error: %s/: cannot start %s" % (p.RelPath(local=opt.this_manifest_only), nb), - file=sys.stderr) - sys.exit(1) + PARALLEL_JOBS = DEFAULT_LOCAL_JOBS + + def _Options(self, p): + p.add_option( + "--all", + dest="all", + action="store_true", + help="begin branch in all projects", + ) + p.add_option( + "-r", + "--rev", + "--revision", + dest="revision", + help="point branch at this revision instead of upstream", + ) + p.add_option( + "--head", + "--HEAD", + dest="revision", + action="store_const", + const="HEAD", + help="abbreviation for --rev HEAD", + ) + + def ValidateOptions(self, opt, args): + if not args: + self.Usage() + + nb = args[0] + if not git.check_ref_format("heads/%s" % nb): + self.OptionParser.error("'%s' is not a valid name" % nb) + + def _ExecuteOne(self, revision, nb, project): + """Start one project.""" + # If the current revision is immutable, such as a SHA1, a tag or + # a change, then we can't push back to it. Substitute with + # dest_branch, if defined; or with manifest default revision instead. + branch_merge = "" + if IsImmutable(project.revisionExpr): + if project.dest_branch: + branch_merge = project.dest_branch + else: + branch_merge = self.manifest.default.revisionExpr + + try: + ret = project.StartBranch( + nb, branch_merge=branch_merge, revision=revision + ) + except Exception as e: + print( + "error: unable to checkout %s: %s" % (project.name, e), + file=sys.stderr, + ) + ret = False + return (ret, project) + + def Execute(self, opt, args): + nb = args[0] + err = [] + projects = [] + if not opt.all: + projects = args[1:] + if len(projects) < 1: + projects = ["."] # start it in the local project by default + + all_projects = self.GetProjects( + projects, + missing_ok=bool(self.gitc_manifest), + all_manifests=not opt.this_manifest_only, + ) + + # This must happen after we find all_projects, since GetProjects may + # need the local directory, which will disappear once we save the GITC + # manifest. + if self.gitc_manifest: + gitc_projects = self.GetProjects( + projects, manifest=self.gitc_manifest, missing_ok=True + ) + for project in gitc_projects: + if project.old_revision: + project.already_synced = True + else: + project.already_synced = False + project.old_revision = project.revisionExpr + project.revisionExpr = None + # Save the GITC manifest. + gitc_utils.save_manifest(self.gitc_manifest) + + # Make sure we have a valid CWD. + if not os.path.exists(os.getcwd()): + os.chdir(self.manifest.topdir) + + pm = Progress("Syncing %s" % nb, len(all_projects), quiet=opt.quiet) + for project in all_projects: + gitc_project = self.gitc_manifest.paths[project.relpath] + # Sync projects that have not been opened. + if not gitc_project.already_synced: + proj_localdir = os.path.join( + self.gitc_manifest.gitc_client_dir, project.relpath + ) + project.worktree = proj_localdir + if not os.path.exists(proj_localdir): + os.makedirs(proj_localdir) + project.Sync_NetworkHalf() + sync_buf = SyncBuffer(self.manifest.manifestProject.config) + project.Sync_LocalHalf(sync_buf) + project.revisionId = gitc_project.old_revision + pm.update() + pm.end() + + def _ProcessResults(_pool, pm, results): + for result, project in results: + if not result: + err.append(project) + pm.update() + + self.ExecuteInParallel( + opt.jobs, + functools.partial(self._ExecuteOne, opt.revision, nb), + all_projects, + callback=_ProcessResults, + output=Progress( + "Starting %s" % (nb,), len(all_projects), quiet=opt.quiet + ), + ) + + if err: + for p in err: + print( + "error: %s/: cannot start %s" + % (p.RelPath(local=opt.this_manifest_only), nb), + file=sys.stderr, + ) + sys.exit(1) diff --git a/subcmds/status.py b/subcmds/status.py index 572c72f7..6e0026f9 100644 --- a/subcmds/status.py +++ b/subcmds/status.py @@ -24,12 +24,12 @@ import platform_utils class Status(PagedCommand): - COMMON = True - helpSummary = "Show the working tree status" - helpUsage = """ + COMMON = True + helpSummary = "Show the working tree status" + helpUsage = """ %prog [...] """ - helpDescription = """ + helpDescription = """ '%prog' compares the working tree to the staging area (aka index), and the most recent commit on this branch (HEAD), in each project specified. A summary is displayed, one line per file where there @@ -76,109 +76,128 @@ the following meanings: d: deleted ( in index, not in work tree ) """ - PARALLEL_JOBS = DEFAULT_LOCAL_JOBS - - def _Options(self, p): - p.add_option('-o', '--orphans', - dest='orphans', action='store_true', - help="include objects in working directory outside of repo projects") - - def _StatusHelper(self, quiet, local, project): - """Obtains the status for a specific project. - - Obtains the status for a project, redirecting the output to - the specified object. - - Args: - quiet: Where to output the status. - local: a boolean, if True, the path is relative to the local - (sub)manifest. If false, the path is relative to the - outermost manifest. - project: Project to get status of. - - Returns: - The status of the project. - """ - buf = io.StringIO() - ret = project.PrintWorkTreeStatus(quiet=quiet, output_redir=buf, - local=local) - return (ret, buf.getvalue()) - - def _FindOrphans(self, dirs, proj_dirs, proj_dirs_parents, outstring): - """find 'dirs' that are present in 'proj_dirs_parents' but not in 'proj_dirs'""" - status_header = ' --\t' - for item in dirs: - if not platform_utils.isdir(item): - outstring.append(''.join([status_header, item])) - continue - if item in proj_dirs: - continue - if item in proj_dirs_parents: - self._FindOrphans(glob.glob('%s/.*' % item) + - glob.glob('%s/*' % item), - proj_dirs, proj_dirs_parents, outstring) - continue - outstring.append(''.join([status_header, item, '/'])) - - def Execute(self, opt, args): - all_projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only) - - def _ProcessResults(_pool, _output, results): - ret = 0 - for (state, output) in results: - if output: - print(output, end='') - if state == 'CLEAN': - ret += 1 - return ret - - counter = self.ExecuteInParallel( - opt.jobs, - functools.partial(self._StatusHelper, opt.quiet, opt.this_manifest_only), - all_projects, - callback=_ProcessResults, - ordered=True) - - if not opt.quiet and len(all_projects) == counter: - print('nothing to commit (working directory clean)') - - if opt.orphans: - proj_dirs = set() - proj_dirs_parents = set() - for project in self.GetProjects(None, missing_ok=True, all_manifests=not opt.this_manifest_only): - relpath = project.RelPath(local=opt.this_manifest_only) - proj_dirs.add(relpath) - (head, _tail) = os.path.split(relpath) - while head != "": - proj_dirs_parents.add(head) - (head, _tail) = os.path.split(head) - proj_dirs.add('.repo') - - class StatusColoring(Coloring): - def __init__(self, config): - Coloring.__init__(self, config, 'status') - self.project = self.printer('header', attr='bold') - self.untracked = self.printer('untracked', fg='red') - - orig_path = os.getcwd() - try: - os.chdir(self.manifest.topdir) - - outstring = [] - self._FindOrphans(glob.glob('.*') + - glob.glob('*'), - proj_dirs, proj_dirs_parents, outstring) - - if outstring: - output = StatusColoring(self.client.globalConfig) - output.project('Objects not within a project (orphans)') - output.nl() - for entry in outstring: - output.untracked(entry) - output.nl() - else: - print('No orphan files or directories') - - finally: - # Restore CWD. - os.chdir(orig_path) + PARALLEL_JOBS = DEFAULT_LOCAL_JOBS + + def _Options(self, p): + p.add_option( + "-o", + "--orphans", + dest="orphans", + action="store_true", + help="include objects in working directory outside of repo " + "projects", + ) + + def _StatusHelper(self, quiet, local, project): + """Obtains the status for a specific project. + + Obtains the status for a project, redirecting the output to + the specified object. + + Args: + quiet: Where to output the status. + local: a boolean, if True, the path is relative to the local + (sub)manifest. If false, the path is relative to the outermost + manifest. + project: Project to get status of. + + Returns: + The status of the project. + """ + buf = io.StringIO() + ret = project.PrintWorkTreeStatus( + quiet=quiet, output_redir=buf, local=local + ) + return (ret, buf.getvalue()) + + def _FindOrphans(self, dirs, proj_dirs, proj_dirs_parents, outstring): + """find 'dirs' that are present in 'proj_dirs_parents' but not in 'proj_dirs'""" # noqa: E501 + status_header = " --\t" + for item in dirs: + if not platform_utils.isdir(item): + outstring.append("".join([status_header, item])) + continue + if item in proj_dirs: + continue + if item in proj_dirs_parents: + self._FindOrphans( + glob.glob("%s/.*" % item) + glob.glob("%s/*" % item), + proj_dirs, + proj_dirs_parents, + outstring, + ) + continue + outstring.append("".join([status_header, item, "/"])) + + def Execute(self, opt, args): + all_projects = self.GetProjects( + args, all_manifests=not opt.this_manifest_only + ) + + def _ProcessResults(_pool, _output, results): + ret = 0 + for state, output in results: + if output: + print(output, end="") + if state == "CLEAN": + ret += 1 + return ret + + counter = self.ExecuteInParallel( + opt.jobs, + functools.partial( + self._StatusHelper, opt.quiet, opt.this_manifest_only + ), + all_projects, + callback=_ProcessResults, + ordered=True, + ) + + if not opt.quiet and len(all_projects) == counter: + print("nothing to commit (working directory clean)") + + if opt.orphans: + proj_dirs = set() + proj_dirs_parents = set() + for project in self.GetProjects( + None, missing_ok=True, all_manifests=not opt.this_manifest_only + ): + relpath = project.RelPath(local=opt.this_manifest_only) + proj_dirs.add(relpath) + (head, _tail) = os.path.split(relpath) + while head != "": + proj_dirs_parents.add(head) + (head, _tail) = os.path.split(head) + proj_dirs.add(".repo") + + class StatusColoring(Coloring): + def __init__(self, config): + Coloring.__init__(self, config, "status") + self.project = self.printer("header", attr="bold") + self.untracked = self.printer("untracked", fg="red") + + orig_path = os.getcwd() + try: + os.chdir(self.manifest.topdir) + + outstring = [] + self._FindOrphans( + glob.glob(".*") + glob.glob("*"), + proj_dirs, + proj_dirs_parents, + outstring, + ) + + if outstring: + output = StatusColoring(self.client.globalConfig) + output.project("Objects not within a project (orphans)") + output.nl() + for entry in outstring: + output.untracked(entry) + output.nl() + else: + print("No orphan files or directories") + + finally: + # Restore CWD. + os.chdir(orig_path) diff --git a/subcmds/sync.py b/subcmds/sync.py index 9a8ca8f7..eabaa68b 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -33,18 +33,21 @@ import xml.parsers.expat import xmlrpc.client try: - import threading as _threading + import threading as _threading except ImportError: - import dummy_threading as _threading + import dummy_threading as _threading try: - import resource + import resource + + def _rlimit_nofile(): + return resource.getrlimit(resource.RLIMIT_NOFILE) - def _rlimit_nofile(): - return resource.getrlimit(resource.RLIMIT_NOFILE) except ImportError: - def _rlimit_nofile(): - return (256, 256) + + def _rlimit_nofile(): + return (256, 256) + import event_log from git_command import git_require @@ -54,7 +57,12 @@ import git_superproject import gitc_utils from project import Project from project import RemoteSpec -from command import Command, DEFAULT_LOCAL_JOBS, MirrorSafeCommand, WORKER_BATCH_SIZE +from command import ( + Command, + DEFAULT_LOCAL_JOBS, + MirrorSafeCommand, + WORKER_BATCH_SIZE, +) from error import RepoChangedException, GitError import platform_utils from project import SyncBuffer @@ -68,70 +76,74 @@ _ONE_DAY_S = 24 * 60 * 60 # Env var to implicitly turn auto-gc back on. This was added to allow a user to # revert a change in default behavior in v2.29.9. Remove after 2023-04-01. -_REPO_AUTO_GC = 'REPO_AUTO_GC' -_AUTO_GC = os.environ.get(_REPO_AUTO_GC) == '1' +_REPO_AUTO_GC = "REPO_AUTO_GC" +_AUTO_GC = os.environ.get(_REPO_AUTO_GC) == "1" class _FetchOneResult(NamedTuple): - """_FetchOne return value. - - Attributes: - success (bool): True if successful. - project (Project): The fetched project. - start (float): The starting time.time(). - finish (float): The ending time.time(). - remote_fetched (bool): True if the remote was actually queried. - """ - success: bool - project: Project - start: float - finish: float - remote_fetched: bool + """_FetchOne return value. + + Attributes: + success (bool): True if successful. + project (Project): The fetched project. + start (float): The starting time.time(). + finish (float): The ending time.time(). + remote_fetched (bool): True if the remote was actually queried. + """ + + success: bool + project: Project + start: float + finish: float + remote_fetched: bool class _FetchResult(NamedTuple): - """_Fetch return value. + """_Fetch return value. + + Attributes: + success (bool): True if successful. + projects (Set[str]): The names of the git directories of fetched projects. + """ - Attributes: - success (bool): True if successful. - projects (Set[str]): The names of the git directories of fetched projects. - """ - success: bool - projects: Set[str] + success: bool + projects: Set[str] class _FetchMainResult(NamedTuple): - """_FetchMain return value. + """_FetchMain return value. + + Attributes: + all_projects (List[Project]): The fetched projects. + """ - Attributes: - all_projects (List[Project]): The fetched projects. - """ - all_projects: List[Project] + all_projects: List[Project] class _CheckoutOneResult(NamedTuple): - """_CheckoutOne return value. + """_CheckoutOne return value. + + Attributes: + success (bool): True if successful. + project (Project): The project. + start (float): The starting time.time(). + finish (float): The ending time.time(). + """ - Attributes: - success (bool): True if successful. - project (Project): The project. - start (float): The starting time.time(). - finish (float): The ending time.time(). - """ - success: bool - project: Project - start: float - finish: float + success: bool + project: Project + start: float + finish: float class Sync(Command, MirrorSafeCommand): - COMMON = True - MULTI_MANIFEST_SUPPORT = True - helpSummary = "Update working tree to the latest revision" - helpUsage = """ + COMMON = True + MULTI_MANIFEST_SUPPORT = True + helpSummary = "Update working tree to the latest revision" + helpUsage = """ %prog [...] """ - helpDescription = """ + helpDescription = """ The '%prog' command synchronizes local project directories with the remote repositories specified in the manifest. If a local project does not yet exist, it will clone a new local directory from @@ -230,1293 +242,1604 @@ If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or later is required to fix a server side protocol bug. """ - # A value of 0 means we want parallel jobs, but we'll determine the default - # value later on. - PARALLEL_JOBS = 0 - - def _Options(self, p, show_smart=True): - p.add_option('--jobs-network', default=None, type=int, metavar='JOBS', - help='number of network jobs to run in parallel (defaults to --jobs or 1)') - p.add_option('--jobs-checkout', default=None, type=int, metavar='JOBS', - help='number of local checkout jobs to run in parallel (defaults to --jobs or ' - f'{DEFAULT_LOCAL_JOBS})') - - p.add_option('-f', '--force-broken', - dest='force_broken', action='store_true', - help='obsolete option (to be deleted in the future)') - p.add_option('--fail-fast', - dest='fail_fast', action='store_true', - help='stop syncing after first error is hit') - p.add_option('--force-sync', - dest='force_sync', action='store_true', - help="overwrite an existing git directory if it needs to " - "point to a different object directory. WARNING: this " - "may cause loss of data") - p.add_option('--force-remove-dirty', - dest='force_remove_dirty', action='store_true', - help="force remove projects with uncommitted modifications if " - "projects no longer exist in the manifest. " - "WARNING: this may cause loss of data") - p.add_option('-l', '--local-only', - dest='local_only', action='store_true', - help="only update working tree, don't fetch") - p.add_option('--no-manifest-update', '--nmu', - dest='mp_update', action='store_false', default='true', - help='use the existing manifest checkout as-is. ' - '(do not update to the latest revision)') - p.add_option('-n', '--network-only', - dest='network_only', action='store_true', - help="fetch only, don't update working tree") - p.add_option('-d', '--detach', - dest='detach_head', action='store_true', - help='detach projects back to manifest revision') - p.add_option('-c', '--current-branch', - dest='current_branch_only', action='store_true', - help='fetch only current branch from server') - p.add_option('--no-current-branch', - dest='current_branch_only', action='store_false', - help='fetch all branches from server') - p.add_option('-m', '--manifest-name', - dest='manifest_name', - help='temporary manifest to use for this sync', metavar='NAME.xml') - p.add_option('--clone-bundle', action='store_true', - help='enable use of /clone.bundle on HTTP/HTTPS') - p.add_option('--no-clone-bundle', dest='clone_bundle', action='store_false', - help='disable use of /clone.bundle on HTTP/HTTPS') - p.add_option('-u', '--manifest-server-username', action='store', - dest='manifest_server_username', - help='username to authenticate with the manifest server') - p.add_option('-p', '--manifest-server-password', action='store', - dest='manifest_server_password', - help='password to authenticate with the manifest server') - p.add_option('--fetch-submodules', - dest='fetch_submodules', action='store_true', - help='fetch submodules from server') - p.add_option('--use-superproject', action='store_true', - help='use the manifest superproject to sync projects; implies -c') - p.add_option('--no-use-superproject', action='store_false', - dest='use_superproject', - help='disable use of manifest superprojects') - p.add_option('--tags', action='store_true', - help='fetch tags') - p.add_option('--no-tags', - dest='tags', action='store_false', - help="don't fetch tags (default)") - p.add_option('--optimized-fetch', - dest='optimized_fetch', action='store_true', - help='only fetch projects fixed to sha1 if revision does not exist locally') - p.add_option('--retry-fetches', - default=0, action='store', type='int', - help='number of times to retry fetches on transient errors') - p.add_option('--prune', action='store_true', - help='delete refs that no longer exist on the remote (default)') - p.add_option('--no-prune', dest='prune', action='store_false', - help='do not delete refs that no longer exist on the remote') - p.add_option('--auto-gc', action='store_true', default=None, - help='run garbage collection on all synced projects') - p.add_option('--no-auto-gc', dest='auto_gc', action='store_false', - help='do not run garbage collection on any projects (default)') - if show_smart: - p.add_option('-s', '--smart-sync', - dest='smart_sync', action='store_true', - help='smart sync using manifest from the latest known good build') - p.add_option('-t', '--smart-tag', - dest='smart_tag', action='store', - help='smart sync using manifest from a known tag') - - g = p.add_option_group('repo Version options') - g.add_option('--no-repo-verify', - dest='repo_verify', default=True, action='store_false', - help='do not verify repo source code') - g.add_option('--repo-upgraded', - dest='repo_upgraded', action='store_true', - help=SUPPRESS_HELP) - - def _GetBranch(self, manifest_project): - """Returns the branch name for getting the approved smartsync manifest. - - Args: - manifest_project: the manifestProject to query. - """ - b = manifest_project.GetBranch(manifest_project.CurrentBranch) - branch = b.merge - if branch.startswith(R_HEADS): - branch = branch[len(R_HEADS):] - return branch - - def _GetCurrentBranchOnly(self, opt, manifest): - """Returns whether current-branch or use-superproject options are enabled. - - Args: - opt: Program options returned from optparse. See _Options(). - manifest: The manifest to use. - - Returns: - True if a superproject is requested, otherwise the value of the - current_branch option (True, False or None). - """ - return git_superproject.UseSuperproject(opt.use_superproject, manifest) or opt.current_branch_only - - def _UpdateProjectsRevisionId(self, opt, args, superproject_logging_data, - manifest): - """Update revisionId of projects with the commit hash from the superproject. - - This function updates each project's revisionId with the commit hash from - the superproject. It writes the updated manifest into a file and reloads - the manifest from it. When appropriate, sub manifests are also processed. - - Args: - opt: Program options returned from optparse. See _Options(). - args: Arguments to pass to GetProjects. See the GetProjects - docstring for details. - superproject_logging_data: A dictionary of superproject data to log. - manifest: The manifest to use. - """ - have_superproject = manifest.superproject or any( - m.superproject for m in manifest.all_children) - if not have_superproject: - return - - if opt.local_only and manifest.superproject: - manifest_path = manifest.superproject.manifest_path - if manifest_path: - self._ReloadManifest(manifest_path, manifest) - return - - all_projects = self.GetProjects(args, - missing_ok=True, - submodules_ok=opt.fetch_submodules, - manifest=manifest, - all_manifests=not opt.this_manifest_only) - - per_manifest = collections.defaultdict(list) - manifest_paths = {} - if opt.this_manifest_only: - per_manifest[manifest.path_prefix] = all_projects - else: - for p in all_projects: - per_manifest[p.manifest.path_prefix].append(p) - - superproject_logging_data = {} - need_unload = False - for m in self.ManifestList(opt): - if not m.path_prefix in per_manifest: - continue - use_super = git_superproject.UseSuperproject(opt.use_superproject, m) - if superproject_logging_data: - superproject_logging_data['multimanifest'] = True - superproject_logging_data.update( - superproject=use_super, - haslocalmanifests=bool(m.HasLocalManifests), - hassuperprojecttag=bool(m.superproject), - ) - if use_super and (m.IsMirror or m.IsArchive): - # Don't use superproject, because we have no working tree. - use_super = False - superproject_logging_data['superproject'] = False - superproject_logging_data['noworktree'] = True - if opt.use_superproject is not False: - print(f'{m.path_prefix}: not using superproject because there is no ' - 'working tree.') - - if not use_super: - continue - m.superproject.SetQuiet(opt.quiet) - print_messages = git_superproject.PrintMessages(opt.use_superproject, m) - m.superproject.SetPrintMessages(print_messages) - update_result = m.superproject.UpdateProjectsRevisionId( - per_manifest[m.path_prefix], git_event_log=self.git_event_log) - manifest_path = update_result.manifest_path - superproject_logging_data['updatedrevisionid'] = bool(manifest_path) - if manifest_path: - m.SetManifestOverride(manifest_path) - need_unload = True - else: - if print_messages: - print(f'{m.path_prefix}: warning: Update of revisionId from ' - 'superproject has failed, repo sync will not use superproject ' - 'to fetch the source. ', - 'Please resync with the --no-use-superproject option to avoid ' - 'this repo warning.', - file=sys.stderr) - if update_result.fatal and opt.use_superproject is not None: - sys.exit(1) - if need_unload: - m.outer_client.manifest.Unload() - - def _FetchProjectList(self, opt, projects): - """Main function of the fetch worker. - - The projects we're given share the same underlying git object store, so we - have to fetch them in serial. - - Delegates most of the work to _FetchHelper. - - Args: - opt: Program options returned from optparse. See _Options(). - projects: Projects to fetch. - """ - return [self._FetchOne(opt, x) for x in projects] + # A value of 0 means we want parallel jobs, but we'll determine the default + # value later on. + PARALLEL_JOBS = 0 + + def _Options(self, p, show_smart=True): + p.add_option( + "--jobs-network", + default=None, + type=int, + metavar="JOBS", + help="number of network jobs to run in parallel (defaults to " + "--jobs or 1)", + ) + p.add_option( + "--jobs-checkout", + default=None, + type=int, + metavar="JOBS", + help="number of local checkout jobs to run in parallel (defaults " + f"to --jobs or {DEFAULT_LOCAL_JOBS})", + ) + + p.add_option( + "-f", + "--force-broken", + dest="force_broken", + action="store_true", + help="obsolete option (to be deleted in the future)", + ) + p.add_option( + "--fail-fast", + dest="fail_fast", + action="store_true", + help="stop syncing after first error is hit", + ) + p.add_option( + "--force-sync", + dest="force_sync", + action="store_true", + help="overwrite an existing git directory if it needs to " + "point to a different object directory. WARNING: this " + "may cause loss of data", + ) + p.add_option( + "--force-remove-dirty", + dest="force_remove_dirty", + action="store_true", + help="force remove projects with uncommitted modifications if " + "projects no longer exist in the manifest. " + "WARNING: this may cause loss of data", + ) + p.add_option( + "-l", + "--local-only", + dest="local_only", + action="store_true", + help="only update working tree, don't fetch", + ) + p.add_option( + "--no-manifest-update", + "--nmu", + dest="mp_update", + action="store_false", + default="true", + help="use the existing manifest checkout as-is. " + "(do not update to the latest revision)", + ) + p.add_option( + "-n", + "--network-only", + dest="network_only", + action="store_true", + help="fetch only, don't update working tree", + ) + p.add_option( + "-d", + "--detach", + dest="detach_head", + action="store_true", + help="detach projects back to manifest revision", + ) + p.add_option( + "-c", + "--current-branch", + dest="current_branch_only", + action="store_true", + help="fetch only current branch from server", + ) + p.add_option( + "--no-current-branch", + dest="current_branch_only", + action="store_false", + help="fetch all branches from server", + ) + p.add_option( + "-m", + "--manifest-name", + dest="manifest_name", + help="temporary manifest to use for this sync", + metavar="NAME.xml", + ) + p.add_option( + "--clone-bundle", + action="store_true", + help="enable use of /clone.bundle on HTTP/HTTPS", + ) + p.add_option( + "--no-clone-bundle", + dest="clone_bundle", + action="store_false", + help="disable use of /clone.bundle on HTTP/HTTPS", + ) + p.add_option( + "-u", + "--manifest-server-username", + action="store", + dest="manifest_server_username", + help="username to authenticate with the manifest server", + ) + p.add_option( + "-p", + "--manifest-server-password", + action="store", + dest="manifest_server_password", + help="password to authenticate with the manifest server", + ) + p.add_option( + "--fetch-submodules", + dest="fetch_submodules", + action="store_true", + help="fetch submodules from server", + ) + p.add_option( + "--use-superproject", + action="store_true", + help="use the manifest superproject to sync projects; implies -c", + ) + p.add_option( + "--no-use-superproject", + action="store_false", + dest="use_superproject", + help="disable use of manifest superprojects", + ) + p.add_option("--tags", action="store_true", help="fetch tags") + p.add_option( + "--no-tags", + dest="tags", + action="store_false", + help="don't fetch tags (default)", + ) + p.add_option( + "--optimized-fetch", + dest="optimized_fetch", + action="store_true", + help="only fetch projects fixed to sha1 if revision does not exist " + "locally", + ) + p.add_option( + "--retry-fetches", + default=0, + action="store", + type="int", + help="number of times to retry fetches on transient errors", + ) + p.add_option( + "--prune", + action="store_true", + help="delete refs that no longer exist on the remote (default)", + ) + p.add_option( + "--no-prune", + dest="prune", + action="store_false", + help="do not delete refs that no longer exist on the remote", + ) + p.add_option( + "--auto-gc", + action="store_true", + default=None, + help="run garbage collection on all synced projects", + ) + p.add_option( + "--no-auto-gc", + dest="auto_gc", + action="store_false", + help="do not run garbage collection on any projects (default)", + ) + if show_smart: + p.add_option( + "-s", + "--smart-sync", + dest="smart_sync", + action="store_true", + help="smart sync using manifest from the latest known good " + "build", + ) + p.add_option( + "-t", + "--smart-tag", + dest="smart_tag", + action="store", + help="smart sync using manifest from a known tag", + ) + + g = p.add_option_group("repo Version options") + g.add_option( + "--no-repo-verify", + dest="repo_verify", + default=True, + action="store_false", + help="do not verify repo source code", + ) + g.add_option( + "--repo-upgraded", + dest="repo_upgraded", + action="store_true", + help=SUPPRESS_HELP, + ) - def _FetchOne(self, opt, project): - """Fetch git objects for a single project. + def _GetBranch(self, manifest_project): + """Returns the branch name for getting the approved smartsync manifest. + + Args: + manifest_project: The manifestProject to query. + """ + b = manifest_project.GetBranch(manifest_project.CurrentBranch) + branch = b.merge + if branch.startswith(R_HEADS): + branch = branch[len(R_HEADS) :] + return branch + + def _GetCurrentBranchOnly(self, opt, manifest): + """Returns whether current-branch or use-superproject options are + enabled. + + Args: + opt: Program options returned from optparse. See _Options(). + manifest: The manifest to use. + + Returns: + True if a superproject is requested, otherwise the value of the + current_branch option (True, False or None). + """ + return ( + git_superproject.UseSuperproject(opt.use_superproject, manifest) + or opt.current_branch_only + ) - Args: - opt: Program options returned from optparse. See _Options(). - project: Project object for the project to fetch. + def _UpdateProjectsRevisionId( + self, opt, args, superproject_logging_data, manifest + ): + """Update revisionId of projects with the commit from the superproject. + + This function updates each project's revisionId with the commit hash + from the superproject. It writes the updated manifest into a file and + reloads the manifest from it. When appropriate, sub manifests are also + processed. + + Args: + opt: Program options returned from optparse. See _Options(). + args: Arguments to pass to GetProjects. See the GetProjects + docstring for details. + superproject_logging_data: A dictionary of superproject data to log. + manifest: The manifest to use. + """ + have_superproject = manifest.superproject or any( + m.superproject for m in manifest.all_children + ) + if not have_superproject: + return + + if opt.local_only and manifest.superproject: + manifest_path = manifest.superproject.manifest_path + if manifest_path: + self._ReloadManifest(manifest_path, manifest) + return + + all_projects = self.GetProjects( + args, + missing_ok=True, + submodules_ok=opt.fetch_submodules, + manifest=manifest, + all_manifests=not opt.this_manifest_only, + ) - Returns: - Whether the fetch was successful. - """ - start = time.time() - success = False - remote_fetched = False - buf = io.StringIO() - try: - sync_result = project.Sync_NetworkHalf( - quiet=opt.quiet, - verbose=opt.verbose, - output_redir=buf, - current_branch_only=self._GetCurrentBranchOnly(opt, project.manifest), - force_sync=opt.force_sync, - clone_bundle=opt.clone_bundle, - tags=opt.tags, archive=project.manifest.IsArchive, - optimized_fetch=opt.optimized_fetch, - retry_fetches=opt.retry_fetches, - prune=opt.prune, - ssh_proxy=self.ssh_proxy, - clone_filter=project.manifest.CloneFilter, - partial_clone_exclude=project.manifest.PartialCloneExclude) - success = sync_result.success - remote_fetched = sync_result.remote_fetched - - output = buf.getvalue() - if (opt.verbose or not success) and output: - print('\n' + output.rstrip()) - - if not success: - print('error: Cannot fetch %s from %s' - % (project.name, project.remote.url), - file=sys.stderr) - except KeyboardInterrupt: - print(f'Keyboard interrupt while processing {project.name}') - except GitError as e: - print('error.GitError: Cannot fetch %s' % str(e), file=sys.stderr) - except Exception as e: - print('error: Cannot fetch %s (%s: %s)' - % (project.name, type(e).__name__, str(e)), file=sys.stderr) - raise - - finish = time.time() - return _FetchOneResult(success, project, start, finish, remote_fetched) - - @classmethod - def _FetchInitChild(cls, ssh_proxy): - cls.ssh_proxy = ssh_proxy - - def _Fetch(self, projects, opt, err_event, ssh_proxy): - ret = True - - jobs = opt.jobs_network - fetched = set() - remote_fetched = set() - pm = Progress('Fetching', len(projects), delay=False, quiet=opt.quiet) - - objdir_project_map = dict() - for project in projects: - objdir_project_map.setdefault(project.objdir, []).append(project) - projects_list = list(objdir_project_map.values()) - - def _ProcessResults(results_sets): - ret = True - for results in results_sets: - for result in results: - success = result.success - project = result.project - start = result.start - finish = result.finish - self._fetch_times.Set(project, finish - start) - self.event_log.AddSync(project, event_log.TASK_SYNC_NETWORK, - start, finish, success) - if result.remote_fetched: - remote_fetched.add(project) - # Check for any errors before running any more tasks. - # ...we'll let existing jobs finish, though. - if not success: - ret = False - else: - fetched.add(project.gitdir) - pm.update(msg=f'Last synced: {project.name}') - if not ret and opt.fail_fast: - break - return ret - - # We pass the ssh proxy settings via the class. This allows multiprocessing - # to pickle it up when spawning children. We can't pass it as an argument - # to _FetchProjectList below as multiprocessing is unable to pickle those. - Sync.ssh_proxy = None - - # NB: Multiprocessing is heavy, so don't spin it up for one job. - if len(projects_list) == 1 or jobs == 1: - self._FetchInitChild(ssh_proxy) - if not _ProcessResults(self._FetchProjectList(opt, x) for x in projects_list): - ret = False - else: - # Favor throughput over responsiveness when quiet. It seems that imap() - # will yield results in batches relative to chunksize, so even as the - # children finish a sync, we won't see the result until one child finishes - # ~chunksize jobs. When using a large --jobs with large chunksize, this - # can be jarring as there will be a large initial delay where repo looks - # like it isn't doing anything and sits at 0%, but then suddenly completes - # a lot of jobs all at once. Since this code is more network bound, we - # can accept a bit more CPU overhead with a smaller chunksize so that the - # user sees more immediate & continuous feedback. - if opt.quiet: - chunksize = WORKER_BATCH_SIZE - else: - pm.update(inc=0, msg='warming up') - chunksize = 4 - with multiprocessing.Pool(jobs, initializer=self._FetchInitChild, - initargs=(ssh_proxy,)) as pool: - results = pool.imap_unordered( - functools.partial(self._FetchProjectList, opt), - projects_list, - chunksize=chunksize) - if not _ProcessResults(results): - ret = False - pool.close() - - # Cleanup the reference now that we're done with it, and we're going to - # release any resources it points to. If we don't, later multiprocessing - # usage (e.g. checkouts) will try to pickle and then crash. - del Sync.ssh_proxy - - pm.end() - self._fetch_times.Save() - - if not self.outer_client.manifest.IsArchive: - self._GCProjects(projects, opt, err_event) - - return _FetchResult(ret, fetched) - - def _FetchMain(self, opt, args, all_projects, err_event, - ssh_proxy, manifest): - """The main network fetch loop. - - Args: - opt: Program options returned from optparse. See _Options(). - args: Command line args used to filter out projects. - all_projects: List of all projects that should be fetched. - err_event: Whether an error was hit while processing. - ssh_proxy: SSH manager for clients & masters. - manifest: The manifest to use. - - Returns: - List of all projects that should be checked out. - """ - rp = manifest.repoProject - - to_fetch = [] - now = time.time() - if _ONE_DAY_S <= (now - rp.LastFetch): - to_fetch.append(rp) - to_fetch.extend(all_projects) - to_fetch.sort(key=self._fetch_times.Get, reverse=True) - - result = self._Fetch(to_fetch, opt, err_event, ssh_proxy) - success = result.success - fetched = result.projects - if not success: - err_event.set() - - _PostRepoFetch(rp, opt.repo_verify) - if opt.network_only: - # bail out now; the rest touches the working tree - if err_event.is_set(): - print('\nerror: Exited sync due to fetch errors.\n', file=sys.stderr) - sys.exit(1) - return _FetchMainResult([]) - - # Iteratively fetch missing and/or nested unregistered submodules - previously_missing_set = set() - while True: - self._ReloadManifest(None, manifest) - all_projects = self.GetProjects(args, - missing_ok=True, - submodules_ok=opt.fetch_submodules, - manifest=manifest, - all_manifests=not opt.this_manifest_only) - missing = [] - for project in all_projects: - if project.gitdir not in fetched: - missing.append(project) - if not missing: - break - # Stop us from non-stopped fetching actually-missing repos: If set of - # missing repos has not been changed from last fetch, we break. - missing_set = set(p.name for p in missing) - if previously_missing_set == missing_set: - break - previously_missing_set = missing_set - result = self._Fetch(missing, opt, err_event, ssh_proxy) - success = result.success - new_fetched = result.projects - if not success: - err_event.set() - fetched.update(new_fetched) - - return _FetchMainResult(all_projects) - - def _CheckoutOne(self, detach_head, force_sync, project): - """Checkout work tree for one project - - Args: - detach_head: Whether to leave a detached HEAD. - force_sync: Force checking out of the repo. - project: Project object for the project to checkout. - - Returns: - Whether the fetch was successful. - """ - start = time.time() - syncbuf = SyncBuffer(project.manifest.manifestProject.config, - detach_head=detach_head) - success = False - try: - project.Sync_LocalHalf(syncbuf, force_sync=force_sync) - success = syncbuf.Finish() - except GitError as e: - print('error.GitError: Cannot checkout %s: %s' % - (project.name, str(e)), file=sys.stderr) - except Exception as e: - print('error: Cannot checkout %s: %s: %s' % - (project.name, type(e).__name__, str(e)), - file=sys.stderr) - raise - - if not success: - print('error: Cannot checkout %s' % (project.name), file=sys.stderr) - finish = time.time() - return _CheckoutOneResult(success, project, start, finish) - - def _Checkout(self, all_projects, opt, err_results): - """Checkout projects listed in all_projects - - Args: - all_projects: List of all projects that should be checked out. - opt: Program options returned from optparse. See _Options(). - err_results: A list of strings, paths to git repos where checkout failed. - """ - # Only checkout projects with worktrees. - all_projects = [x for x in all_projects if x.worktree] + per_manifest = collections.defaultdict(list) + if opt.this_manifest_only: + per_manifest[manifest.path_prefix] = all_projects + else: + for p in all_projects: + per_manifest[p.manifest.path_prefix].append(p) + + superproject_logging_data = {} + need_unload = False + for m in self.ManifestList(opt): + if m.path_prefix not in per_manifest: + continue + use_super = git_superproject.UseSuperproject( + opt.use_superproject, m + ) + if superproject_logging_data: + superproject_logging_data["multimanifest"] = True + superproject_logging_data.update( + superproject=use_super, + haslocalmanifests=bool(m.HasLocalManifests), + hassuperprojecttag=bool(m.superproject), + ) + if use_super and (m.IsMirror or m.IsArchive): + # Don't use superproject, because we have no working tree. + use_super = False + superproject_logging_data["superproject"] = False + superproject_logging_data["noworktree"] = True + if opt.use_superproject is not False: + print( + f"{m.path_prefix}: not using superproject because " + "there is no working tree." + ) + + if not use_super: + continue + m.superproject.SetQuiet(opt.quiet) + print_messages = git_superproject.PrintMessages( + opt.use_superproject, m + ) + m.superproject.SetPrintMessages(print_messages) + update_result = m.superproject.UpdateProjectsRevisionId( + per_manifest[m.path_prefix], git_event_log=self.git_event_log + ) + manifest_path = update_result.manifest_path + superproject_logging_data["updatedrevisionid"] = bool(manifest_path) + if manifest_path: + m.SetManifestOverride(manifest_path) + need_unload = True + else: + if print_messages: + print( + f"{m.path_prefix}: warning: Update of revisionId from " + "superproject has failed, repo sync will not use " + "superproject to fetch the source. ", + "Please resync with the --no-use-superproject option " + "to avoid this repo warning.", + file=sys.stderr, + ) + if update_result.fatal and opt.use_superproject is not None: + sys.exit(1) + if need_unload: + m.outer_client.manifest.Unload() + + def _FetchProjectList(self, opt, projects): + """Main function of the fetch worker. + + The projects we're given share the same underlying git object store, so + we have to fetch them in serial. + + Delegates most of the work to _FetchHelper. + + Args: + opt: Program options returned from optparse. See _Options(). + projects: Projects to fetch. + """ + return [self._FetchOne(opt, x) for x in projects] + + def _FetchOne(self, opt, project): + """Fetch git objects for a single project. + + Args: + opt: Program options returned from optparse. See _Options(). + project: Project object for the project to fetch. + + Returns: + Whether the fetch was successful. + """ + start = time.time() + success = False + remote_fetched = False + buf = io.StringIO() + try: + sync_result = project.Sync_NetworkHalf( + quiet=opt.quiet, + verbose=opt.verbose, + output_redir=buf, + current_branch_only=self._GetCurrentBranchOnly( + opt, project.manifest + ), + force_sync=opt.force_sync, + clone_bundle=opt.clone_bundle, + tags=opt.tags, + archive=project.manifest.IsArchive, + optimized_fetch=opt.optimized_fetch, + retry_fetches=opt.retry_fetches, + prune=opt.prune, + ssh_proxy=self.ssh_proxy, + clone_filter=project.manifest.CloneFilter, + partial_clone_exclude=project.manifest.PartialCloneExclude, + ) + success = sync_result.success + remote_fetched = sync_result.remote_fetched + + output = buf.getvalue() + if (opt.verbose or not success) and output: + print("\n" + output.rstrip()) + + if not success: + print( + "error: Cannot fetch %s from %s" + % (project.name, project.remote.url), + file=sys.stderr, + ) + except KeyboardInterrupt: + print(f"Keyboard interrupt while processing {project.name}") + except GitError as e: + print("error.GitError: Cannot fetch %s" % str(e), file=sys.stderr) + except Exception as e: + print( + "error: Cannot fetch %s (%s: %s)" + % (project.name, type(e).__name__, str(e)), + file=sys.stderr, + ) + raise + + finish = time.time() + return _FetchOneResult(success, project, start, finish, remote_fetched) + + @classmethod + def _FetchInitChild(cls, ssh_proxy): + cls.ssh_proxy = ssh_proxy + + def _Fetch(self, projects, opt, err_event, ssh_proxy): + ret = True + + jobs = opt.jobs_network + fetched = set() + remote_fetched = set() + pm = Progress("Fetching", len(projects), delay=False, quiet=opt.quiet) + + objdir_project_map = dict() + for project in projects: + objdir_project_map.setdefault(project.objdir, []).append(project) + projects_list = list(objdir_project_map.values()) + + def _ProcessResults(results_sets): + ret = True + for results in results_sets: + for result in results: + success = result.success + project = result.project + start = result.start + finish = result.finish + self._fetch_times.Set(project, finish - start) + self.event_log.AddSync( + project, + event_log.TASK_SYNC_NETWORK, + start, + finish, + success, + ) + if result.remote_fetched: + remote_fetched.add(project) + # Check for any errors before running any more tasks. + # ...we'll let existing jobs finish, though. + if not success: + ret = False + else: + fetched.add(project.gitdir) + pm.update(msg=f"Last synced: {project.name}") + if not ret and opt.fail_fast: + break + return ret - def _ProcessResults(pool, pm, results): - ret = True - for result in results: + # We pass the ssh proxy settings via the class. This allows + # multiprocessing to pickle it up when spawning children. We can't pass + # it as an argument to _FetchProjectList below as multiprocessing is + # unable to pickle those. + Sync.ssh_proxy = None + + # NB: Multiprocessing is heavy, so don't spin it up for one job. + if len(projects_list) == 1 or jobs == 1: + self._FetchInitChild(ssh_proxy) + if not _ProcessResults( + self._FetchProjectList(opt, x) for x in projects_list + ): + ret = False + else: + # Favor throughput over responsiveness when quiet. It seems that + # imap() will yield results in batches relative to chunksize, so + # even as the children finish a sync, we won't see the result until + # one child finishes ~chunksize jobs. When using a large --jobs + # with large chunksize, this can be jarring as there will be a large + # initial delay where repo looks like it isn't doing anything and + # sits at 0%, but then suddenly completes a lot of jobs all at once. + # Since this code is more network bound, we can accept a bit more + # CPU overhead with a smaller chunksize so that the user sees more + # immediate & continuous feedback. + if opt.quiet: + chunksize = WORKER_BATCH_SIZE + else: + pm.update(inc=0, msg="warming up") + chunksize = 4 + with multiprocessing.Pool( + jobs, initializer=self._FetchInitChild, initargs=(ssh_proxy,) + ) as pool: + results = pool.imap_unordered( + functools.partial(self._FetchProjectList, opt), + projects_list, + chunksize=chunksize, + ) + if not _ProcessResults(results): + ret = False + pool.close() + + # Cleanup the reference now that we're done with it, and we're going to + # release any resources it points to. If we don't, later + # multiprocessing usage (e.g. checkouts) will try to pickle and then + # crash. + del Sync.ssh_proxy + + pm.end() + self._fetch_times.Save() + + if not self.outer_client.manifest.IsArchive: + self._GCProjects(projects, opt, err_event) + + return _FetchResult(ret, fetched) + + def _FetchMain( + self, opt, args, all_projects, err_event, ssh_proxy, manifest + ): + """The main network fetch loop. + + Args: + opt: Program options returned from optparse. See _Options(). + args: Command line args used to filter out projects. + all_projects: List of all projects that should be fetched. + err_event: Whether an error was hit while processing. + ssh_proxy: SSH manager for clients & masters. + manifest: The manifest to use. + + Returns: + List of all projects that should be checked out. + """ + rp = manifest.repoProject + + to_fetch = [] + now = time.time() + if _ONE_DAY_S <= (now - rp.LastFetch): + to_fetch.append(rp) + to_fetch.extend(all_projects) + to_fetch.sort(key=self._fetch_times.Get, reverse=True) + + result = self._Fetch(to_fetch, opt, err_event, ssh_proxy) success = result.success - project = result.project - start = result.start - finish = result.finish - self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL, - start, finish, success) - # Check for any errors before running any more tasks. - # ...we'll let existing jobs finish, though. + fetched = result.projects if not success: - ret = False - err_results.append(project.RelPath(local=opt.this_manifest_only)) - if opt.fail_fast: - if pool: - pool.close() + err_event.set() + + _PostRepoFetch(rp, opt.repo_verify) + if opt.network_only: + # Bail out now; the rest touches the working tree. + if err_event.is_set(): + print( + "\nerror: Exited sync due to fetch errors.\n", + file=sys.stderr, + ) + sys.exit(1) + return _FetchMainResult([]) + + # Iteratively fetch missing and/or nested unregistered submodules. + previously_missing_set = set() + while True: + self._ReloadManifest(None, manifest) + all_projects = self.GetProjects( + args, + missing_ok=True, + submodules_ok=opt.fetch_submodules, + manifest=manifest, + all_manifests=not opt.this_manifest_only, + ) + missing = [] + for project in all_projects: + if project.gitdir not in fetched: + missing.append(project) + if not missing: + break + # Stop us from non-stopped fetching actually-missing repos: If set + # of missing repos has not been changed from last fetch, we break. + missing_set = set(p.name for p in missing) + if previously_missing_set == missing_set: + break + previously_missing_set = missing_set + result = self._Fetch(missing, opt, err_event, ssh_proxy) + success = result.success + new_fetched = result.projects + if not success: + err_event.set() + fetched.update(new_fetched) + + return _FetchMainResult(all_projects) + + def _CheckoutOne(self, detach_head, force_sync, project): + """Checkout work tree for one project + + Args: + detach_head: Whether to leave a detached HEAD. + force_sync: Force checking out of the repo. + project: Project object for the project to checkout. + + Returns: + Whether the fetch was successful. + """ + start = time.time() + syncbuf = SyncBuffer( + project.manifest.manifestProject.config, detach_head=detach_head + ) + success = False + try: + project.Sync_LocalHalf(syncbuf, force_sync=force_sync) + success = syncbuf.Finish() + except GitError as e: + print( + "error.GitError: Cannot checkout %s: %s" + % (project.name, str(e)), + file=sys.stderr, + ) + except Exception as e: + print( + "error: Cannot checkout %s: %s: %s" + % (project.name, type(e).__name__, str(e)), + file=sys.stderr, + ) + raise + + if not success: + print("error: Cannot checkout %s" % (project.name), file=sys.stderr) + finish = time.time() + return _CheckoutOneResult(success, project, start, finish) + + def _Checkout(self, all_projects, opt, err_results): + """Checkout projects listed in all_projects + + Args: + all_projects: List of all projects that should be checked out. + opt: Program options returned from optparse. See _Options(). + err_results: A list of strings, paths to git repos where checkout + failed. + """ + # Only checkout projects with worktrees. + all_projects = [x for x in all_projects if x.worktree] + + def _ProcessResults(pool, pm, results): + ret = True + for result in results: + success = result.success + project = result.project + start = result.start + finish = result.finish + self.event_log.AddSync( + project, event_log.TASK_SYNC_LOCAL, start, finish, success + ) + # Check for any errors before running any more tasks. + # ...we'll let existing jobs finish, though. + if not success: + ret = False + err_results.append( + project.RelPath(local=opt.this_manifest_only) + ) + if opt.fail_fast: + if pool: + pool.close() + return ret + pm.update(msg=project.name) return ret - pm.update(msg=project.name) - return ret - - return self.ExecuteInParallel( - opt.jobs_checkout, - functools.partial(self._CheckoutOne, opt.detach_head, opt.force_sync), - all_projects, - callback=_ProcessResults, - output=Progress('Checking out', len(all_projects), quiet=opt.quiet)) and not err_results - - @staticmethod - def _GetPreciousObjectsState(project: Project, opt): - """Get the preciousObjects state for the project. - - Args: - project (Project): the project to examine, and possibly correct. - opt (optparse.Values): options given to sync. - - Returns: - Expected state of extensions.preciousObjects: - False: Should be disabled. (not present) - True: Should be enabled. - """ - if project.use_git_worktrees: - return False - projects = project.manifest.GetProjectsWithName(project.name, - all_manifests=True) - if len(projects) == 1: - return False - relpath = project.RelPath(local=opt.this_manifest_only) - if len(projects) > 1: - # Objects are potentially shared with another project. - # See the logic in Project.Sync_NetworkHalf regarding UseAlternates. - # - When False, shared projects share (via symlink) - # .repo/project-objects/{PROJECT_NAME}.git as the one-and-only objects - # directory. All objects are precious, since there is no project with a - # complete set of refs. - # - When True, shared projects share (via info/alternates) - # .repo/project-objects/{PROJECT_NAME}.git as an alternate object store, - # which is written only on the first clone of the project, and is not - # written subsequently. (When Sync_NetworkHalf sees that it exists, it - # makes sure that the alternates file points there, and uses a - # project-local .git/objects directory for all syncs going forward. - # We do not support switching between the options. The environment - # variable is present for testing and migration only. - return not project.UseAlternates - - return False - - def _SetPreciousObjectsState(self, project: Project, opt): - """Correct the preciousObjects state for the project. - - Args: - project: the project to examine, and possibly correct. - opt: options given to sync. - """ - expected = self._GetPreciousObjectsState(project, opt) - actual = project.config.GetBoolean('extensions.preciousObjects') or False - relpath = project.RelPath(local=opt.this_manifest_only) - - if expected != actual: - # If this is unexpected, log it and repair. - Trace(f'{relpath} expected preciousObjects={expected}, got {actual}') - if expected: - if not opt.quiet: - print('\r%s: Shared project %s found, disabling pruning.' % - (relpath, project.name)) - if git_require((2, 7, 0)): - project.EnableRepositoryExtension('preciousObjects') - else: - # This isn't perfect, but it's the best we can do with old git. - print('\r%s: WARNING: shared projects are unreliable when using ' - 'old versions of git; please upgrade to git-2.7.0+.' - % (relpath,), - file=sys.stderr) - project.config.SetString('gc.pruneExpire', 'never') - else: - if not opt.quiet: - print(f'\r{relpath}: not shared, disabling pruning.') - project.config.SetString('extensions.preciousObjects', None) - project.config.SetString('gc.pruneExpire', None) - def _GCProjects(self, projects, opt, err_event): - """Perform garbage collection. + return ( + self.ExecuteInParallel( + opt.jobs_checkout, + functools.partial( + self._CheckoutOne, opt.detach_head, opt.force_sync + ), + all_projects, + callback=_ProcessResults, + output=Progress( + "Checking out", len(all_projects), quiet=opt.quiet + ), + ) + and not err_results + ) - If We are skipping garbage collection (opt.auto_gc not set), we still want - to potentially mark objects precious, so that `git gc` does not discard - shared objects. - """ - if not opt.auto_gc: - # Just repair preciousObjects state, and return. - for project in projects: - self._SetPreciousObjectsState(project, opt) - return - - pm = Progress('Garbage collecting', len(projects), delay=False, - quiet=opt.quiet) - pm.update(inc=0, msg='prescan') - - tidy_dirs = {} - for project in projects: - self._SetPreciousObjectsState(project, opt) - - project.config.SetString('gc.autoDetach', 'false') - # Only call git gc once per objdir, but call pack-refs for the remainder. - if project.objdir not in tidy_dirs: - tidy_dirs[project.objdir] = ( - True, # Run a full gc. - project.bare_git, + @staticmethod + def _GetPreciousObjectsState(project: Project, opt): + """Get the preciousObjects state for the project. + + Args: + project (Project): the project to examine, and possibly correct. + opt (optparse.Values): options given to sync. + + Returns: + Expected state of extensions.preciousObjects: + False: Should be disabled. (not present) + True: Should be enabled. + """ + if project.use_git_worktrees: + return False + projects = project.manifest.GetProjectsWithName( + project.name, all_manifests=True ) - elif project.gitdir not in tidy_dirs: - tidy_dirs[project.gitdir] = ( - False, # Do not run a full gc; just run pack-refs. - project.bare_git, + if len(projects) == 1: + return False + if len(projects) > 1: + # Objects are potentially shared with another project. + # See the logic in Project.Sync_NetworkHalf regarding UseAlternates. + # - When False, shared projects share (via symlink) + # .repo/project-objects/{PROJECT_NAME}.git as the one-and-only + # objects directory. All objects are precious, since there is no + # project with a complete set of refs. + # - When True, shared projects share (via info/alternates) + # .repo/project-objects/{PROJECT_NAME}.git as an alternate object + # store, which is written only on the first clone of the project, + # and is not written subsequently. (When Sync_NetworkHalf sees + # that it exists, it makes sure that the alternates file points + # there, and uses a project-local .git/objects directory for all + # syncs going forward. + # We do not support switching between the options. The environment + # variable is present for testing and migration only. + return not project.UseAlternates + + return False + + def _SetPreciousObjectsState(self, project: Project, opt): + """Correct the preciousObjects state for the project. + + Args: + project: the project to examine, and possibly correct. + opt: options given to sync. + """ + expected = self._GetPreciousObjectsState(project, opt) + actual = ( + project.config.GetBoolean("extensions.preciousObjects") or False ) - - jobs = opt.jobs - - if jobs < 2: - for (run_gc, bare_git) in tidy_dirs.values(): - pm.update(msg=bare_git._project.name) - - if run_gc: - bare_git.gc('--auto') + relpath = project.RelPath(local=opt.this_manifest_only) + + if expected != actual: + # If this is unexpected, log it and repair. + Trace( + f"{relpath} expected preciousObjects={expected}, got {actual}" + ) + if expected: + if not opt.quiet: + print( + "\r%s: Shared project %s found, disabling pruning." + % (relpath, project.name) + ) + if git_require((2, 7, 0)): + project.EnableRepositoryExtension("preciousObjects") + else: + # This isn't perfect, but it's the best we can do with old + # git. + print( + "\r%s: WARNING: shared projects are unreliable when " + "using old versions of git; please upgrade to " + "git-2.7.0+." % (relpath,), + file=sys.stderr, + ) + project.config.SetString("gc.pruneExpire", "never") + else: + if not opt.quiet: + print(f"\r{relpath}: not shared, disabling pruning.") + project.config.SetString("extensions.preciousObjects", None) + project.config.SetString("gc.pruneExpire", None) + + def _GCProjects(self, projects, opt, err_event): + """Perform garbage collection. + + If We are skipping garbage collection (opt.auto_gc not set), we still + want to potentially mark objects precious, so that `git gc` does not + discard shared objects. + """ + if not opt.auto_gc: + # Just repair preciousObjects state, and return. + for project in projects: + self._SetPreciousObjectsState(project, opt) + return + + pm = Progress( + "Garbage collecting", len(projects), delay=False, quiet=opt.quiet + ) + pm.update(inc=0, msg="prescan") + + tidy_dirs = {} + for project in projects: + self._SetPreciousObjectsState(project, opt) + + project.config.SetString("gc.autoDetach", "false") + # Only call git gc once per objdir, but call pack-refs for the + # remainder. + if project.objdir not in tidy_dirs: + tidy_dirs[project.objdir] = ( + True, # Run a full gc. + project.bare_git, + ) + elif project.gitdir not in tidy_dirs: + tidy_dirs[project.gitdir] = ( + False, # Do not run a full gc; just run pack-refs. + project.bare_git, + ) + + jobs = opt.jobs + + if jobs < 2: + for run_gc, bare_git in tidy_dirs.values(): + pm.update(msg=bare_git._project.name) + + if run_gc: + bare_git.gc("--auto") + else: + bare_git.pack_refs() + pm.end() + return + + cpu_count = os.cpu_count() + config = {"pack.threads": cpu_count // jobs if cpu_count > jobs else 1} + + threads = set() + sem = _threading.Semaphore(jobs) + + def tidy_up(run_gc, bare_git): + pm.start(bare_git._project.name) + try: + try: + if run_gc: + bare_git.gc("--auto", config=config) + else: + bare_git.pack_refs(config=config) + except GitError: + err_event.set() + except Exception: + err_event.set() + raise + finally: + pm.finish(bare_git._project.name) + sem.release() + + for run_gc, bare_git in tidy_dirs.values(): + if err_event.is_set() and opt.fail_fast: + break + sem.acquire() + t = _threading.Thread( + target=tidy_up, + args=( + run_gc, + bare_git, + ), + ) + t.daemon = True + threads.add(t) + t.start() + + for t in threads: + t.join() + pm.end() + + def _ReloadManifest(self, manifest_name, manifest): + """Reload the manfiest from the file specified by the |manifest_name|. + + It unloads the manifest if |manifest_name| is None. + + Args: + manifest_name: Manifest file to be reloaded. + manifest: The manifest to use. + """ + if manifest_name: + # Override calls Unload already. + manifest.Override(manifest_name) else: - bare_git.pack_refs() - pm.end() - return - - cpu_count = os.cpu_count() - config = {'pack.threads': cpu_count // jobs if cpu_count > jobs else 1} - - threads = set() - sem = _threading.Semaphore(jobs) + manifest.Unload() + + def UpdateProjectList(self, opt, manifest): + """Update the cached projects list for |manifest| + + In a multi-manifest checkout, each manifest has its own project.list. + + Args: + opt: Program options returned from optparse. See _Options(). + manifest: The manifest to use. + + Returns: + 0: success + 1: failure + """ + new_project_paths = [] + for project in self.GetProjects( + None, missing_ok=True, manifest=manifest, all_manifests=False + ): + if project.relpath: + new_project_paths.append(project.relpath) + file_name = "project.list" + file_path = os.path.join(manifest.subdir, file_name) + old_project_paths = [] + + if os.path.exists(file_path): + with open(file_path, "r") as fd: + old_project_paths = fd.read().split("\n") + # In reversed order, so subfolders are deleted before parent folder. + for path in sorted(old_project_paths, reverse=True): + if not path: + continue + if path not in new_project_paths: + # If the path has already been deleted, we don't need to do + # it. + gitdir = os.path.join(manifest.topdir, path, ".git") + if os.path.exists(gitdir): + project = Project( + manifest=manifest, + name=path, + remote=RemoteSpec("origin"), + gitdir=gitdir, + objdir=gitdir, + use_git_worktrees=os.path.isfile(gitdir), + worktree=os.path.join(manifest.topdir, path), + relpath=path, + revisionExpr="HEAD", + revisionId=None, + groups=None, + ) + if not project.DeleteWorktree( + quiet=opt.quiet, force=opt.force_remove_dirty + ): + return 1 + + new_project_paths.sort() + with open(file_path, "w") as fd: + fd.write("\n".join(new_project_paths)) + fd.write("\n") + return 0 + + def UpdateCopyLinkfileList(self, manifest): + """Save all dests of copyfile and linkfile, and update them if needed. + + Returns: + Whether update was successful. + """ + new_paths = {} + new_linkfile_paths = [] + new_copyfile_paths = [] + for project in self.GetProjects( + None, missing_ok=True, manifest=manifest, all_manifests=False + ): + new_linkfile_paths.extend(x.dest for x in project.linkfiles) + new_copyfile_paths.extend(x.dest for x in project.copyfiles) + + new_paths = { + "linkfile": new_linkfile_paths, + "copyfile": new_copyfile_paths, + } + + copylinkfile_name = "copy-link-files.json" + copylinkfile_path = os.path.join(manifest.subdir, copylinkfile_name) + old_copylinkfile_paths = {} + + if os.path.exists(copylinkfile_path): + with open(copylinkfile_path, "rb") as fp: + try: + old_copylinkfile_paths = json.load(fp) + except Exception: + print( + "error: %s is not a json formatted file." + % copylinkfile_path, + file=sys.stderr, + ) + platform_utils.remove(copylinkfile_path) + return False + + need_remove_files = [] + need_remove_files.extend( + set(old_copylinkfile_paths.get("linkfile", [])) + - set(new_linkfile_paths) + ) + need_remove_files.extend( + set(old_copylinkfile_paths.get("copyfile", [])) + - set(new_copyfile_paths) + ) + + for need_remove_file in need_remove_files: + # Try to remove the updated copyfile or linkfile. + # So, if the file is not exist, nothing need to do. + platform_utils.remove(need_remove_file, missing_ok=True) + + # Create copy-link-files.json, save dest path of "copyfile" and + # "linkfile". + with open(copylinkfile_path, "w", encoding="utf-8") as fp: + json.dump(new_paths, fp) + return True + + def _SmartSyncSetup(self, opt, smart_sync_manifest_path, manifest): + if not manifest.manifest_server: + print( + "error: cannot smart sync: no manifest server defined in " + "manifest", + file=sys.stderr, + ) + sys.exit(1) + + manifest_server = manifest.manifest_server + if not opt.quiet: + print("Using manifest server %s" % manifest_server) + + if "@" not in manifest_server: + username = None + password = None + if opt.manifest_server_username and opt.manifest_server_password: + username = opt.manifest_server_username + password = opt.manifest_server_password + else: + try: + info = netrc.netrc() + except IOError: + # .netrc file does not exist or could not be opened. + pass + else: + try: + parse_result = urllib.parse.urlparse(manifest_server) + if parse_result.hostname: + auth = info.authenticators(parse_result.hostname) + if auth: + username, _account, password = auth + else: + print( + "No credentials found for %s in .netrc" + % parse_result.hostname, + file=sys.stderr, + ) + except netrc.NetrcParseError as e: + print( + "Error parsing .netrc file: %s" % e, file=sys.stderr + ) + + if username and password: + manifest_server = manifest_server.replace( + "://", "://%s:%s@" % (username, password), 1 + ) + + transport = PersistentTransport(manifest_server) + if manifest_server.startswith("persistent-"): + manifest_server = manifest_server[len("persistent-") :] - def tidy_up(run_gc, bare_git): - pm.start(bare_git._project.name) - try: try: - if run_gc: - bare_git.gc('--auto', config=config) - else: - bare_git.pack_refs(config=config) - except GitError: - err_event.set() - except Exception: - err_event.set() - raise - finally: - pm.finish(bare_git._project.name) - sem.release() - - for (run_gc, bare_git) in tidy_dirs.values(): - if err_event.is_set() and opt.fail_fast: - break - sem.acquire() - t = _threading.Thread(target=tidy_up, args=(run_gc, bare_git,)) - t.daemon = True - threads.add(t) - t.start() - - for t in threads: - t.join() - pm.end() - - def _ReloadManifest(self, manifest_name, manifest): - """Reload the manfiest from the file specified by the |manifest_name|. - - It unloads the manifest if |manifest_name| is None. - - Args: - manifest_name: Manifest file to be reloaded. - manifest: The manifest to use. - """ - if manifest_name: - # Override calls Unload already - manifest.Override(manifest_name) - else: - manifest.Unload() - - def UpdateProjectList(self, opt, manifest): - """Update the cached projects list for |manifest| - - In a multi-manifest checkout, each manifest has its own project.list. + server = xmlrpc.client.Server(manifest_server, transport=transport) + if opt.smart_sync: + branch = self._GetBranch(manifest.manifestProject) + + if "SYNC_TARGET" in os.environ: + target = os.environ["SYNC_TARGET"] + [success, manifest_str] = server.GetApprovedManifest( + branch, target + ) + elif ( + "TARGET_PRODUCT" in os.environ + and "TARGET_BUILD_VARIANT" in os.environ + ): + target = "%s-%s" % ( + os.environ["TARGET_PRODUCT"], + os.environ["TARGET_BUILD_VARIANT"], + ) + [success, manifest_str] = server.GetApprovedManifest( + branch, target + ) + else: + [success, manifest_str] = server.GetApprovedManifest(branch) + else: + assert opt.smart_tag + [success, manifest_str] = server.GetManifest(opt.smart_tag) + + if success: + manifest_name = os.path.basename(smart_sync_manifest_path) + try: + with open(smart_sync_manifest_path, "w") as f: + f.write(manifest_str) + except IOError as e: + print( + "error: cannot write manifest to %s:\n%s" + % (smart_sync_manifest_path, e), + file=sys.stderr, + ) + sys.exit(1) + self._ReloadManifest(manifest_name, manifest) + else: + print( + "error: manifest server RPC call failed: %s" % manifest_str, + file=sys.stderr, + ) + sys.exit(1) + except (socket.error, IOError, xmlrpc.client.Fault) as e: + print( + "error: cannot connect to manifest server %s:\n%s" + % (manifest.manifest_server, e), + file=sys.stderr, + ) + sys.exit(1) + except xmlrpc.client.ProtocolError as e: + print( + "error: cannot connect to manifest server %s:\n%d %s" + % (manifest.manifest_server, e.errcode, e.errmsg), + file=sys.stderr, + ) + sys.exit(1) + + return manifest_name + + def _UpdateAllManifestProjects(self, opt, mp, manifest_name): + """Fetch & update the local manifest project. + + After syncing the manifest project, if the manifest has any sub + manifests, those are recursively processed. + + Args: + opt: Program options returned from optparse. See _Options(). + mp: the manifestProject to query. + manifest_name: Manifest file to be reloaded. + """ + if not mp.standalone_manifest_url: + self._UpdateManifestProject(opt, mp, manifest_name) + + if mp.manifest.submanifests: + for submanifest in mp.manifest.submanifests.values(): + child = submanifest.repo_client.manifest + child.manifestProject.SyncWithPossibleInit( + submanifest, + current_branch_only=self._GetCurrentBranchOnly(opt, child), + verbose=opt.verbose, + tags=opt.tags, + git_event_log=self.git_event_log, + ) + self._UpdateAllManifestProjects( + opt, child.manifestProject, None + ) + + def _UpdateManifestProject(self, opt, mp, manifest_name): + """Fetch & update the local manifest project. + + Args: + opt: Program options returned from optparse. See _Options(). + mp: the manifestProject to query. + manifest_name: Manifest file to be reloaded. + """ + if not opt.local_only: + start = time.time() + success = mp.Sync_NetworkHalf( + quiet=opt.quiet, + verbose=opt.verbose, + current_branch_only=self._GetCurrentBranchOnly( + opt, mp.manifest + ), + force_sync=opt.force_sync, + tags=opt.tags, + optimized_fetch=opt.optimized_fetch, + retry_fetches=opt.retry_fetches, + submodules=mp.manifest.HasSubmodules, + clone_filter=mp.manifest.CloneFilter, + partial_clone_exclude=mp.manifest.PartialCloneExclude, + ) + finish = time.time() + self.event_log.AddSync( + mp, event_log.TASK_SYNC_NETWORK, start, finish, success + ) + + if mp.HasChanges: + syncbuf = SyncBuffer(mp.config) + start = time.time() + mp.Sync_LocalHalf(syncbuf, submodules=mp.manifest.HasSubmodules) + clean = syncbuf.Finish() + self.event_log.AddSync( + mp, event_log.TASK_SYNC_LOCAL, start, time.time(), clean + ) + if not clean: + sys.exit(1) + self._ReloadManifest(manifest_name, mp.manifest) + + def ValidateOptions(self, opt, args): + if opt.force_broken: + print( + "warning: -f/--force-broken is now the default behavior, and " + "the options are deprecated", + file=sys.stderr, + ) + if opt.network_only and opt.detach_head: + self.OptionParser.error("cannot combine -n and -d") + if opt.network_only and opt.local_only: + self.OptionParser.error("cannot combine -n and -l") + if opt.manifest_name and opt.smart_sync: + self.OptionParser.error("cannot combine -m and -s") + if opt.manifest_name and opt.smart_tag: + self.OptionParser.error("cannot combine -m and -t") + if opt.manifest_server_username or opt.manifest_server_password: + if not (opt.smart_sync or opt.smart_tag): + self.OptionParser.error( + "-u and -p may only be combined with -s or -t" + ) + if None in [ + opt.manifest_server_username, + opt.manifest_server_password, + ]: + self.OptionParser.error("both -u and -p must be given") + + if opt.prune is None: + opt.prune = True + + if opt.auto_gc is None and _AUTO_GC: + print( + f"Will run `git gc --auto` because {_REPO_AUTO_GC} is set.", + f"{_REPO_AUTO_GC} is deprecated and will be removed in a ", + "future release. Use `--auto-gc` instead.", + file=sys.stderr, + ) + opt.auto_gc = True + + def _ValidateOptionsWithManifest(self, opt, mp): + """Like ValidateOptions, but after we've updated the manifest. + + Needed to handle sync-xxx option defaults in the manifest. + + Args: + opt: The options to process. + mp: The manifest project to pull defaults from. + """ + if not opt.jobs: + # If the user hasn't made a choice, use the manifest value. + opt.jobs = mp.manifest.default.sync_j + if opt.jobs: + # If --jobs has a non-default value, propagate it as the default for + # --jobs-xxx flags too. + if not opt.jobs_network: + opt.jobs_network = opt.jobs + if not opt.jobs_checkout: + opt.jobs_checkout = opt.jobs + else: + # Neither user nor manifest have made a choice, so setup defaults. + if not opt.jobs_network: + opt.jobs_network = 1 + if not opt.jobs_checkout: + opt.jobs_checkout = DEFAULT_LOCAL_JOBS + opt.jobs = os.cpu_count() + + # Try to stay under user rlimit settings. + # + # Since each worker requires at 3 file descriptors to run `git fetch`, + # use that to scale down the number of jobs. Unfortunately there isn't + # an easy way to determine this reliably as systems change, but it was + # last measured by hand in 2011. + soft_limit, _ = _rlimit_nofile() + jobs_soft_limit = max(1, (soft_limit - 5) // 3) + opt.jobs = min(opt.jobs, jobs_soft_limit) + opt.jobs_network = min(opt.jobs_network, jobs_soft_limit) + opt.jobs_checkout = min(opt.jobs_checkout, jobs_soft_limit) + + def Execute(self, opt, args): + manifest = self.outer_manifest + if not opt.outer_manifest: + manifest = self.manifest + + if opt.manifest_name: + manifest.Override(opt.manifest_name) + + manifest_name = opt.manifest_name + smart_sync_manifest_path = os.path.join( + manifest.manifestProject.worktree, "smart_sync_override.xml" + ) - Args: - opt: Program options returned from optparse. See _Options(). - manifest: The manifest to use. + if opt.clone_bundle is None: + opt.clone_bundle = manifest.CloneBundle - Returns: - 0: success - 1: failure - """ - new_project_paths = [] - for project in self.GetProjects(None, missing_ok=True, manifest=manifest, - all_manifests=False): - if project.relpath: - new_project_paths.append(project.relpath) - file_name = 'project.list' - file_path = os.path.join(manifest.subdir, file_name) - old_project_paths = [] - - if os.path.exists(file_path): - with open(file_path, 'r') as fd: - old_project_paths = fd.read().split('\n') - # In reversed order, so subfolders are deleted before parent folder. - for path in sorted(old_project_paths, reverse=True): - if not path: - continue - if path not in new_project_paths: - # If the path has already been deleted, we don't need to do it - gitdir = os.path.join(manifest.topdir, path, '.git') - if os.path.exists(gitdir): - project = Project( - manifest=manifest, - name=path, - remote=RemoteSpec('origin'), - gitdir=gitdir, - objdir=gitdir, - use_git_worktrees=os.path.isfile(gitdir), - worktree=os.path.join(manifest.topdir, path), - relpath=path, - revisionExpr='HEAD', - revisionId=None, - groups=None) - if not project.DeleteWorktree( - quiet=opt.quiet, - force=opt.force_remove_dirty): - return 1 - - new_project_paths.sort() - with open(file_path, 'w') as fd: - fd.write('\n'.join(new_project_paths)) - fd.write('\n') - return 0 - - def UpdateCopyLinkfileList(self, manifest): - """Save all dests of copyfile and linkfile, and update them if needed. - - Returns: - Whether update was successful. - """ - new_paths = {} - new_linkfile_paths = [] - new_copyfile_paths = [] - for project in self.GetProjects(None, missing_ok=True, - manifest=manifest, all_manifests=False): - new_linkfile_paths.extend(x.dest for x in project.linkfiles) - new_copyfile_paths.extend(x.dest for x in project.copyfiles) - - new_paths = { - 'linkfile': new_linkfile_paths, - 'copyfile': new_copyfile_paths, - } - - copylinkfile_name = 'copy-link-files.json' - copylinkfile_path = os.path.join(manifest.subdir, copylinkfile_name) - old_copylinkfile_paths = {} - - if os.path.exists(copylinkfile_path): - with open(copylinkfile_path, 'rb') as fp: - try: - old_copylinkfile_paths = json.load(fp) - except Exception: - print('error: %s is not a json formatted file.' % - copylinkfile_path, file=sys.stderr) - platform_utils.remove(copylinkfile_path) - return False - - need_remove_files = [] - need_remove_files.extend( - set(old_copylinkfile_paths.get('linkfile', [])) - - set(new_linkfile_paths)) - need_remove_files.extend( - set(old_copylinkfile_paths.get('copyfile', [])) - - set(new_copyfile_paths)) - - for need_remove_file in need_remove_files: - # Try to remove the updated copyfile or linkfile. - # So, if the file is not exist, nothing need to do. - platform_utils.remove(need_remove_file, missing_ok=True) - - # Create copy-link-files.json, save dest path of "copyfile" and "linkfile". - with open(copylinkfile_path, 'w', encoding='utf-8') as fp: - json.dump(new_paths, fp) - return True - - def _SmartSyncSetup(self, opt, smart_sync_manifest_path, manifest): - if not manifest.manifest_server: - print('error: cannot smart sync: no manifest server defined in ' - 'manifest', file=sys.stderr) - sys.exit(1) - - manifest_server = manifest.manifest_server - if not opt.quiet: - print('Using manifest server %s' % manifest_server) - - if '@' not in manifest_server: - username = None - password = None - if opt.manifest_server_username and opt.manifest_server_password: - username = opt.manifest_server_username - password = opt.manifest_server_password - else: - try: - info = netrc.netrc() - except IOError: - # .netrc file does not exist or could not be opened - pass + if opt.smart_sync or opt.smart_tag: + manifest_name = self._SmartSyncSetup( + opt, smart_sync_manifest_path, manifest + ) else: - try: - parse_result = urllib.parse.urlparse(manifest_server) - if parse_result.hostname: - auth = info.authenticators(parse_result.hostname) - if auth: - username, _account, password = auth - else: - print('No credentials found for %s in .netrc' - % parse_result.hostname, file=sys.stderr) - except netrc.NetrcParseError as e: - print('Error parsing .netrc file: %s' % e, file=sys.stderr) - - if (username and password): - manifest_server = manifest_server.replace('://', '://%s:%s@' % - (username, password), - 1) - - transport = PersistentTransport(manifest_server) - if manifest_server.startswith('persistent-'): - manifest_server = manifest_server[len('persistent-'):] - - try: - server = xmlrpc.client.Server(manifest_server, transport=transport) - if opt.smart_sync: - branch = self._GetBranch(manifest.manifestProject) - - if 'SYNC_TARGET' in os.environ: - target = os.environ['SYNC_TARGET'] - [success, manifest_str] = server.GetApprovedManifest(branch, target) - elif ('TARGET_PRODUCT' in os.environ and - 'TARGET_BUILD_VARIANT' in os.environ): - target = '%s-%s' % (os.environ['TARGET_PRODUCT'], - os.environ['TARGET_BUILD_VARIANT']) - [success, manifest_str] = server.GetApprovedManifest(branch, target) + if os.path.isfile(smart_sync_manifest_path): + try: + platform_utils.remove(smart_sync_manifest_path) + except OSError as e: + print( + "error: failed to remove existing smart sync override " + "manifest: %s" % e, + file=sys.stderr, + ) + + err_event = multiprocessing.Event() + + rp = manifest.repoProject + rp.PreSync() + cb = rp.CurrentBranch + if cb: + base = rp.GetBranch(cb).merge + if not base or not base.startswith("refs/heads/"): + print( + "warning: repo is not tracking a remote branch, so it will " + "not receive updates; run `repo init --repo-rev=stable` to " + "fix.", + file=sys.stderr, + ) + + for m in self.ManifestList(opt): + if not m.manifestProject.standalone_manifest_url: + m.manifestProject.PreSync() + + if opt.repo_upgraded: + _PostRepoUpgrade(manifest, quiet=opt.quiet) + + mp = manifest.manifestProject + if opt.mp_update: + self._UpdateAllManifestProjects(opt, mp, manifest_name) else: - [success, manifest_str] = server.GetApprovedManifest(branch) - else: - assert(opt.smart_tag) - [success, manifest_str] = server.GetManifest(opt.smart_tag) + print("Skipping update of local manifest project.") - if success: - manifest_name = os.path.basename(smart_sync_manifest_path) - try: - with open(smart_sync_manifest_path, 'w') as f: - f.write(manifest_str) - except IOError as e: - print('error: cannot write manifest to %s:\n%s' - % (smart_sync_manifest_path, e), - file=sys.stderr) - sys.exit(1) - self._ReloadManifest(manifest_name, manifest) - else: - print('error: manifest server RPC call failed: %s' % - manifest_str, file=sys.stderr) - sys.exit(1) - except (socket.error, IOError, xmlrpc.client.Fault) as e: - print('error: cannot connect to manifest server %s:\n%s' - % (manifest.manifest_server, e), file=sys.stderr) - sys.exit(1) - except xmlrpc.client.ProtocolError as e: - print('error: cannot connect to manifest server %s:\n%d %s' - % (manifest.manifest_server, e.errcode, e.errmsg), - file=sys.stderr) - sys.exit(1) - - return manifest_name - - def _UpdateAllManifestProjects(self, opt, mp, manifest_name): - """Fetch & update the local manifest project. - - After syncing the manifest project, if the manifest has any sub manifests, - those are recursively processed. - - Args: - opt: Program options returned from optparse. See _Options(). - mp: the manifestProject to query. - manifest_name: Manifest file to be reloaded. - """ - if not mp.standalone_manifest_url: - self._UpdateManifestProject(opt, mp, manifest_name) - - if mp.manifest.submanifests: - for submanifest in mp.manifest.submanifests.values(): - child = submanifest.repo_client.manifest - child.manifestProject.SyncWithPossibleInit( - submanifest, - current_branch_only=self._GetCurrentBranchOnly(opt, child), - verbose=opt.verbose, - tags=opt.tags, - git_event_log=self.git_event_log, + # Now that the manifests are up-to-date, setup options whose defaults + # might be in the manifest. + self._ValidateOptionsWithManifest(opt, mp) + + superproject_logging_data = {} + self._UpdateProjectsRevisionId( + opt, args, superproject_logging_data, manifest ) - self._UpdateAllManifestProjects(opt, child.manifestProject, None) - def _UpdateManifestProject(self, opt, mp, manifest_name): - """Fetch & update the local manifest project. + if self.gitc_manifest: + gitc_manifest_projects = self.GetProjects(args, missing_ok=True) + gitc_projects = [] + opened_projects = [] + for project in gitc_manifest_projects: + if ( + project.relpath in self.gitc_manifest.paths + and self.gitc_manifest.paths[project.relpath].old_revision + ): + opened_projects.append(project.relpath) + else: + gitc_projects.append(project.relpath) + + if not args: + gitc_projects = None + + if gitc_projects != [] and not opt.local_only: + print( + "Updating GITC client: %s" + % self.gitc_manifest.gitc_client_name + ) + manifest = GitcManifest( + self.repodir, self.gitc_manifest.gitc_client_name + ) + if manifest_name: + manifest.Override(manifest_name) + else: + manifest.Override(manifest.manifestFile) + gitc_utils.generate_gitc_manifest( + self.gitc_manifest, manifest, gitc_projects + ) + print("GITC client successfully synced.") + + # The opened projects need to be synced as normal, therefore we + # generate a new args list to represent the opened projects. + # TODO: make this more reliable -- if there's a project name/path + # overlap, this may choose the wrong project. + args = [ + os.path.relpath(manifest.paths[path].worktree, os.getcwd()) + for path in opened_projects + ] + if not args: + return + + all_projects = self.GetProjects( + args, + missing_ok=True, + submodules_ok=opt.fetch_submodules, + manifest=manifest, + all_manifests=not opt.this_manifest_only, + ) - Args: - opt: Program options returned from optparse. See _Options(). - mp: the manifestProject to query. - manifest_name: Manifest file to be reloaded. - """ - if not opt.local_only: - start = time.time() - success = mp.Sync_NetworkHalf(quiet=opt.quiet, verbose=opt.verbose, - current_branch_only=self._GetCurrentBranchOnly(opt, mp.manifest), - force_sync=opt.force_sync, - tags=opt.tags, - optimized_fetch=opt.optimized_fetch, - retry_fetches=opt.retry_fetches, - submodules=mp.manifest.HasSubmodules, - clone_filter=mp.manifest.CloneFilter, - partial_clone_exclude=mp.manifest.PartialCloneExclude) - finish = time.time() - self.event_log.AddSync(mp, event_log.TASK_SYNC_NETWORK, - start, finish, success) - - if mp.HasChanges: - syncbuf = SyncBuffer(mp.config) - start = time.time() - mp.Sync_LocalHalf(syncbuf, submodules=mp.manifest.HasSubmodules) - clean = syncbuf.Finish() - self.event_log.AddSync(mp, event_log.TASK_SYNC_LOCAL, - start, time.time(), clean) - if not clean: - sys.exit(1) - self._ReloadManifest(manifest_name, mp.manifest) - - def ValidateOptions(self, opt, args): - if opt.force_broken: - print('warning: -f/--force-broken is now the default behavior, and the ' - 'options are deprecated', file=sys.stderr) - if opt.network_only and opt.detach_head: - self.OptionParser.error('cannot combine -n and -d') - if opt.network_only and opt.local_only: - self.OptionParser.error('cannot combine -n and -l') - if opt.manifest_name and opt.smart_sync: - self.OptionParser.error('cannot combine -m and -s') - if opt.manifest_name and opt.smart_tag: - self.OptionParser.error('cannot combine -m and -t') - if opt.manifest_server_username or opt.manifest_server_password: - if not (opt.smart_sync or opt.smart_tag): - self.OptionParser.error('-u and -p may only be combined with -s or -t') - if None in [opt.manifest_server_username, opt.manifest_server_password]: - self.OptionParser.error('both -u and -p must be given') - - if opt.prune is None: - opt.prune = True - - if opt.auto_gc is None and _AUTO_GC: - print(f"Will run `git gc --auto` because {_REPO_AUTO_GC} is set.", - f'{_REPO_AUTO_GC} is deprecated and will be removed in a future', - 'release. Use `--auto-gc` instead.', file=sys.stderr) - opt.auto_gc = True - - def _ValidateOptionsWithManifest(self, opt, mp): - """Like ValidateOptions, but after we've updated the manifest. - - Needed to handle sync-xxx option defaults in the manifest. - - Args: - opt: The options to process. - mp: The manifest project to pull defaults from. - """ - if not opt.jobs: - # If the user hasn't made a choice, use the manifest value. - opt.jobs = mp.manifest.default.sync_j - if opt.jobs: - # If --jobs has a non-default value, propagate it as the default for - # --jobs-xxx flags too. - if not opt.jobs_network: - opt.jobs_network = opt.jobs - if not opt.jobs_checkout: - opt.jobs_checkout = opt.jobs - else: - # Neither user nor manifest have made a choice, so setup defaults. - if not opt.jobs_network: - opt.jobs_network = 1 - if not opt.jobs_checkout: - opt.jobs_checkout = DEFAULT_LOCAL_JOBS - opt.jobs = os.cpu_count() - - # Try to stay under user rlimit settings. - # - # Since each worker requires at 3 file descriptors to run `git fetch`, use - # that to scale down the number of jobs. Unfortunately there isn't an easy - # way to determine this reliably as systems change, but it was last measured - # by hand in 2011. - soft_limit, _ = _rlimit_nofile() - jobs_soft_limit = max(1, (soft_limit - 5) // 3) - opt.jobs = min(opt.jobs, jobs_soft_limit) - opt.jobs_network = min(opt.jobs_network, jobs_soft_limit) - opt.jobs_checkout = min(opt.jobs_checkout, jobs_soft_limit) - - def Execute(self, opt, args): - manifest = self.outer_manifest - if not opt.outer_manifest: - manifest = self.manifest - - if opt.manifest_name: - manifest.Override(opt.manifest_name) - - manifest_name = opt.manifest_name - smart_sync_manifest_path = os.path.join( - manifest.manifestProject.worktree, 'smart_sync_override.xml') - - if opt.clone_bundle is None: - opt.clone_bundle = manifest.CloneBundle - - if opt.smart_sync or opt.smart_tag: - manifest_name = self._SmartSyncSetup(opt, smart_sync_manifest_path, manifest) - else: - if os.path.isfile(smart_sync_manifest_path): - try: - platform_utils.remove(smart_sync_manifest_path) - except OSError as e: - print('error: failed to remove existing smart sync override manifest: %s' % - e, file=sys.stderr) - - err_event = multiprocessing.Event() - - rp = manifest.repoProject - rp.PreSync() - cb = rp.CurrentBranch - if cb: - base = rp.GetBranch(cb).merge - if not base or not base.startswith('refs/heads/'): - print('warning: repo is not tracking a remote branch, so it will not ' - 'receive updates; run `repo init --repo-rev=stable` to fix.', - file=sys.stderr) - - for m in self.ManifestList(opt): - if not m.manifestProject.standalone_manifest_url: - m.manifestProject.PreSync() - - if opt.repo_upgraded: - _PostRepoUpgrade(manifest, quiet=opt.quiet) - - mp = manifest.manifestProject - if opt.mp_update: - self._UpdateAllManifestProjects(opt, mp, manifest_name) - else: - print('Skipping update of local manifest project.') - - # Now that the manifests are up-to-date, setup options whose defaults might - # be in the manifest. - self._ValidateOptionsWithManifest(opt, mp) - - superproject_logging_data = {} - self._UpdateProjectsRevisionId(opt, args, superproject_logging_data, - manifest) - - if self.gitc_manifest: - gitc_manifest_projects = self.GetProjects(args, missing_ok=True) - gitc_projects = [] - opened_projects = [] - for project in gitc_manifest_projects: - if project.relpath in self.gitc_manifest.paths and \ - self.gitc_manifest.paths[project.relpath].old_revision: - opened_projects.append(project.relpath) - else: - gitc_projects.append(project.relpath) + err_network_sync = False + err_update_projects = False + err_update_linkfiles = False + + self._fetch_times = _FetchTimes(manifest) + if not opt.local_only: + with multiprocessing.Manager() as manager: + with ssh.ProxyManager(manager) as ssh_proxy: + # Initialize the socket dir once in the parent. + ssh_proxy.sock() + result = self._FetchMain( + opt, args, all_projects, err_event, ssh_proxy, manifest + ) + all_projects = result.all_projects + + if opt.network_only: + return + + # If we saw an error, exit with code 1 so that other scripts can + # check. + if err_event.is_set(): + err_network_sync = True + if opt.fail_fast: + print( + "\nerror: Exited sync due to fetch errors.\n" + "Local checkouts *not* updated. Resolve network issues " + "& retry.\n" + "`repo sync -l` will update some local checkouts.", + file=sys.stderr, + ) + sys.exit(1) + + for m in self.ManifestList(opt): + if m.IsMirror or m.IsArchive: + # Bail out now, we have no working tree. + continue + + if self.UpdateProjectList(opt, m): + err_event.set() + err_update_projects = True + if opt.fail_fast: + print( + "\nerror: Local checkouts *not* updated.", + file=sys.stderr, + ) + sys.exit(1) + + err_update_linkfiles = not self.UpdateCopyLinkfileList(m) + if err_update_linkfiles: + err_event.set() + if opt.fail_fast: + print( + "\nerror: Local update copyfile or linkfile failed.", + file=sys.stderr, + ) + sys.exit(1) + + err_results = [] + # NB: We don't exit here because this is the last step. + err_checkout = not self._Checkout(all_projects, opt, err_results) + if err_checkout: + err_event.set() + + printed_notices = set() + # If there's a notice that's supposed to print at the end of the sync, + # print it now... But avoid printing duplicate messages, and preserve + # order. + for m in sorted(self.ManifestList(opt), key=lambda x: x.path_prefix): + if m.notice and m.notice not in printed_notices: + print(m.notice) + printed_notices.add(m.notice) + + # If we saw an error, exit with code 1 so that other scripts can check. + if err_event.is_set(): + print("\nerror: Unable to fully sync the tree.", file=sys.stderr) + if err_network_sync: + print( + "error: Downloading network changes failed.", + file=sys.stderr, + ) + if err_update_projects: + print( + "error: Updating local project lists failed.", + file=sys.stderr, + ) + if err_update_linkfiles: + print( + "error: Updating copyfiles or linkfiles failed.", + file=sys.stderr, + ) + if err_checkout: + print( + "error: Checking out local projects failed.", + file=sys.stderr, + ) + if err_results: + print( + "Failing repos:\n%s" % "\n".join(err_results), + file=sys.stderr, + ) + print( + 'Try re-running with "-j1 --fail-fast" to exit at the first ' + "error.", + file=sys.stderr, + ) + sys.exit(1) + + # Log the previous sync analysis state from the config. + self.git_event_log.LogDataConfigEvents( + mp.config.GetSyncAnalysisStateData(), "previous_sync_state" + ) - if not args: - gitc_projects = None + # Update and log with the new sync analysis state. + mp.config.UpdateSyncAnalysisState(opt, superproject_logging_data) + self.git_event_log.LogDataConfigEvents( + mp.config.GetSyncAnalysisStateData(), "current_sync_state" + ) - if gitc_projects != [] and not opt.local_only: - print('Updating GITC client: %s' % self.gitc_manifest.gitc_client_name) - manifest = GitcManifest(self.repodir, self.gitc_manifest.gitc_client_name) - if manifest_name: - manifest.Override(manifest_name) - else: - manifest.Override(manifest.manifestFile) - gitc_utils.generate_gitc_manifest(self.gitc_manifest, - manifest, - gitc_projects) - print('GITC client successfully synced.') - - # The opened projects need to be synced as normal, therefore we - # generate a new args list to represent the opened projects. - # TODO: make this more reliable -- if there's a project name/path overlap, - # this may choose the wrong project. - args = [os.path.relpath(manifest.paths[path].worktree, os.getcwd()) - for path in opened_projects] - if not args: - return - - all_projects = self.GetProjects(args, - missing_ok=True, - submodules_ok=opt.fetch_submodules, - manifest=manifest, - all_manifests=not opt.this_manifest_only) - - err_network_sync = False - err_update_projects = False - err_update_linkfiles = False - - self._fetch_times = _FetchTimes(manifest) - if not opt.local_only: - with multiprocessing.Manager() as manager: - with ssh.ProxyManager(manager) as ssh_proxy: - # Initialize the socket dir once in the parent. - ssh_proxy.sock() - result = self._FetchMain(opt, args, all_projects, err_event, - ssh_proxy, manifest) - all_projects = result.all_projects - - if opt.network_only: - return - - # If we saw an error, exit with code 1 so that other scripts can check. - if err_event.is_set(): - err_network_sync = True - if opt.fail_fast: - print('\nerror: Exited sync due to fetch errors.\n' - 'Local checkouts *not* updated. Resolve network issues & ' - 'retry.\n' - '`repo sync -l` will update some local checkouts.', - file=sys.stderr) - sys.exit(1) - - for m in self.ManifestList(opt): - if m.IsMirror or m.IsArchive: - # bail out now, we have no working tree - continue - - if self.UpdateProjectList(opt, m): - err_event.set() - err_update_projects = True - if opt.fail_fast: - print('\nerror: Local checkouts *not* updated.', file=sys.stderr) - sys.exit(1) - - err_update_linkfiles = not self.UpdateCopyLinkfileList(m) - if err_update_linkfiles: - err_event.set() - if opt.fail_fast: - print('\nerror: Local update copyfile or linkfile failed.', file=sys.stderr) - sys.exit(1) - - err_results = [] - # NB: We don't exit here because this is the last step. - err_checkout = not self._Checkout(all_projects, opt, err_results) - if err_checkout: - err_event.set() - - printed_notices = set() - # If there's a notice that's supposed to print at the end of the sync, - # print it now... But avoid printing duplicate messages, and preserve - # order. - for m in sorted(self.ManifestList(opt), key=lambda x: x.path_prefix): - if m.notice and m.notice not in printed_notices: - print(m.notice) - printed_notices.add(m.notice) - - # If we saw an error, exit with code 1 so that other scripts can check. - if err_event.is_set(): - print('\nerror: Unable to fully sync the tree.', file=sys.stderr) - if err_network_sync: - print('error: Downloading network changes failed.', file=sys.stderr) - if err_update_projects: - print('error: Updating local project lists failed.', file=sys.stderr) - if err_update_linkfiles: - print('error: Updating copyfiles or linkfiles failed.', file=sys.stderr) - if err_checkout: - print('error: Checking out local projects failed.', file=sys.stderr) - if err_results: - print('Failing repos:\n%s' % '\n'.join(err_results), file=sys.stderr) - print('Try re-running with "-j1 --fail-fast" to exit at the first error.', - file=sys.stderr) - sys.exit(1) - - # Log the previous sync analysis state from the config. - self.git_event_log.LogDataConfigEvents(mp.config.GetSyncAnalysisStateData(), - 'previous_sync_state') - - # Update and log with the new sync analysis state. - mp.config.UpdateSyncAnalysisState(opt, superproject_logging_data) - self.git_event_log.LogDataConfigEvents(mp.config.GetSyncAnalysisStateData(), - 'current_sync_state') - - if not opt.quiet: - print('repo sync has finished successfully.') + if not opt.quiet: + print("repo sync has finished successfully.") def _PostRepoUpgrade(manifest, quiet=False): - # Link the docs for the internal .repo/ layout for people - link = os.path.join(manifest.repodir, 'internal-fs-layout.md') - if not platform_utils.islink(link): - target = os.path.join('repo', 'docs', 'internal-fs-layout.md') - try: - platform_utils.symlink(target, link) - except Exception: - pass - - wrapper = Wrapper() - if wrapper.NeedSetupGnuPG(): - wrapper.SetupGnuPG(quiet) - for project in manifest.projects: - if project.Exists: - project.PostRepoUpgrade() + # Link the docs for the internal .repo/ layout for people. + link = os.path.join(manifest.repodir, "internal-fs-layout.md") + if not platform_utils.islink(link): + target = os.path.join("repo", "docs", "internal-fs-layout.md") + try: + platform_utils.symlink(target, link) + except Exception: + pass + + wrapper = Wrapper() + if wrapper.NeedSetupGnuPG(): + wrapper.SetupGnuPG(quiet) + for project in manifest.projects: + if project.Exists: + project.PostRepoUpgrade() def _PostRepoFetch(rp, repo_verify=True, verbose=False): - if rp.HasChanges: - print('info: A new version of repo is available', file=sys.stderr) - wrapper = Wrapper() - try: - rev = rp.bare_git.describe(rp.GetRevisionId()) - except GitError: - rev = None - _, new_rev = wrapper.check_repo_rev(rp.gitdir, rev, repo_verify=repo_verify) - # See if we're held back due to missing signed tag. - current_revid = rp.bare_git.rev_parse('HEAD') - new_revid = rp.bare_git.rev_parse('--verify', new_rev) - if current_revid != new_revid: - # We want to switch to the new rev, but also not trash any uncommitted - # changes. This helps with local testing/hacking. - # If a local change has been made, we will throw that away. - # We also have to make sure this will switch to an older commit if that's - # the latest tag in order to support release rollback. - try: - rp.work_git.reset('--keep', new_rev) - except GitError as e: - sys.exit(str(e)) - print('info: Restarting repo with latest version', file=sys.stderr) - raise RepoChangedException(['--repo-upgraded']) + if rp.HasChanges: + print("info: A new version of repo is available", file=sys.stderr) + wrapper = Wrapper() + try: + rev = rp.bare_git.describe(rp.GetRevisionId()) + except GitError: + rev = None + _, new_rev = wrapper.check_repo_rev( + rp.gitdir, rev, repo_verify=repo_verify + ) + # See if we're held back due to missing signed tag. + current_revid = rp.bare_git.rev_parse("HEAD") + new_revid = rp.bare_git.rev_parse("--verify", new_rev) + if current_revid != new_revid: + # We want to switch to the new rev, but also not trash any + # uncommitted changes. This helps with local testing/hacking. + # If a local change has been made, we will throw that away. + # We also have to make sure this will switch to an older commit if + # that's the latest tag in order to support release rollback. + try: + rp.work_git.reset("--keep", new_rev) + except GitError as e: + sys.exit(str(e)) + print("info: Restarting repo with latest version", file=sys.stderr) + raise RepoChangedException(["--repo-upgraded"]) + else: + print( + "warning: Skipped upgrade to unverified version", + file=sys.stderr, + ) else: - print('warning: Skipped upgrade to unverified version', file=sys.stderr) - else: - if verbose: - print('repo version %s is current' % rp.work_git.describe(HEAD), - file=sys.stderr) + if verbose: + print( + "repo version %s is current" % rp.work_git.describe(HEAD), + file=sys.stderr, + ) class _FetchTimes(object): - _ALPHA = 0.5 - - def __init__(self, manifest): - self._path = os.path.join(manifest.repodir, '.repo_fetchtimes.json') - self._times = None - self._seen = set() - - def Get(self, project): - self._Load() - return self._times.get(project.name, _ONE_DAY_S) - - def Set(self, project, t): - self._Load() - name = project.name - old = self._times.get(name, t) - self._seen.add(name) - a = self._ALPHA - self._times[name] = (a * t) + ((1 - a) * old) - - def _Load(self): - if self._times is None: - try: - with open(self._path) as f: - self._times = json.load(f) - except (IOError, ValueError): - platform_utils.remove(self._path, missing_ok=True) - self._times = {} - - def Save(self): - if self._times is None: - return - - to_delete = [] - for name in self._times: - if name not in self._seen: - to_delete.append(name) - for name in to_delete: - del self._times[name] - - try: - with open(self._path, 'w') as f: - json.dump(self._times, f, indent=2) - except (IOError, TypeError): - platform_utils.remove(self._path, missing_ok=True) + _ALPHA = 0.5 + + def __init__(self, manifest): + self._path = os.path.join(manifest.repodir, ".repo_fetchtimes.json") + self._times = None + self._seen = set() + + def Get(self, project): + self._Load() + return self._times.get(project.name, _ONE_DAY_S) + + def Set(self, project, t): + self._Load() + name = project.name + old = self._times.get(name, t) + self._seen.add(name) + a = self._ALPHA + self._times[name] = (a * t) + ((1 - a) * old) + + def _Load(self): + if self._times is None: + try: + with open(self._path) as f: + self._times = json.load(f) + except (IOError, ValueError): + platform_utils.remove(self._path, missing_ok=True) + self._times = {} + + def Save(self): + if self._times is None: + return + + to_delete = [] + for name in self._times: + if name not in self._seen: + to_delete.append(name) + for name in to_delete: + del self._times[name] + + try: + with open(self._path, "w") as f: + json.dump(self._times, f, indent=2) + except (IOError, TypeError): + platform_utils.remove(self._path, missing_ok=True) + # This is a replacement for xmlrpc.client.Transport using urllib2 # and supporting persistent-http[s]. It cannot change hosts from @@ -1525,98 +1848,105 @@ class _FetchTimes(object): class PersistentTransport(xmlrpc.client.Transport): - def __init__(self, orig_host): - self.orig_host = orig_host - - def request(self, host, handler, request_body, verbose=False): - with GetUrlCookieFile(self.orig_host, not verbose) as (cookiefile, proxy): - # Python doesn't understand cookies with the #HttpOnly_ prefix - # Since we're only using them for HTTP, copy the file temporarily, - # stripping those prefixes away. - if cookiefile: - tmpcookiefile = tempfile.NamedTemporaryFile(mode='w') - tmpcookiefile.write("# HTTP Cookie File") - try: - with open(cookiefile) as f: - for line in f: - if line.startswith("#HttpOnly_"): - line = line[len("#HttpOnly_"):] - tmpcookiefile.write(line) - tmpcookiefile.flush() - - cookiejar = cookielib.MozillaCookieJar(tmpcookiefile.name) - try: - cookiejar.load() - except cookielib.LoadError: - cookiejar = cookielib.CookieJar() - finally: - tmpcookiefile.close() - else: - cookiejar = cookielib.CookieJar() - - proxyhandler = urllib.request.ProxyHandler - if proxy: - proxyhandler = urllib.request.ProxyHandler({ - "http": proxy, - "https": proxy}) - - opener = urllib.request.build_opener( - urllib.request.HTTPCookieProcessor(cookiejar), - proxyhandler) - - url = urllib.parse.urljoin(self.orig_host, handler) - parse_results = urllib.parse.urlparse(url) - - scheme = parse_results.scheme - if scheme == 'persistent-http': - scheme = 'http' - if scheme == 'persistent-https': - # If we're proxying through persistent-https, use http. The - # proxy itself will do the https. - if proxy: - scheme = 'http' - else: - scheme = 'https' - - # Parse out any authentication information using the base class - host, extra_headers, _ = self.get_host_info(parse_results.netloc) - - url = urllib.parse.urlunparse(( - scheme, - host, - parse_results.path, - parse_results.params, - parse_results.query, - parse_results.fragment)) - - request = urllib.request.Request(url, request_body) - if extra_headers is not None: - for (name, header) in extra_headers: - request.add_header(name, header) - request.add_header('Content-Type', 'text/xml') - try: - response = opener.open(request) - except urllib.error.HTTPError as e: - if e.code == 501: - # We may have been redirected through a login process - # but our POST turned into a GET. Retry. - response = opener.open(request) - else: - raise - - p, u = xmlrpc.client.getparser() - # Response should be fairly small, so read it all at once. - # This way we can show it to the user in case of error (e.g. HTML). - data = response.read() - try: - p.feed(data) - except xml.parsers.expat.ExpatError as e: - raise IOError( - f'Parsing the manifest failed: {e}\n' - f'Please report this to your manifest server admin.\n' - f'Here is the full response:\n{data.decode("utf-8")}') - p.close() - return u.close() - - def close(self): - pass + def __init__(self, orig_host): + self.orig_host = orig_host + + def request(self, host, handler, request_body, verbose=False): + with GetUrlCookieFile(self.orig_host, not verbose) as ( + cookiefile, + proxy, + ): + # Python doesn't understand cookies with the #HttpOnly_ prefix + # Since we're only using them for HTTP, copy the file temporarily, + # stripping those prefixes away. + if cookiefile: + tmpcookiefile = tempfile.NamedTemporaryFile(mode="w") + tmpcookiefile.write("# HTTP Cookie File") + try: + with open(cookiefile) as f: + for line in f: + if line.startswith("#HttpOnly_"): + line = line[len("#HttpOnly_") :] + tmpcookiefile.write(line) + tmpcookiefile.flush() + + cookiejar = cookielib.MozillaCookieJar(tmpcookiefile.name) + try: + cookiejar.load() + except cookielib.LoadError: + cookiejar = cookielib.CookieJar() + finally: + tmpcookiefile.close() + else: + cookiejar = cookielib.CookieJar() + + proxyhandler = urllib.request.ProxyHandler + if proxy: + proxyhandler = urllib.request.ProxyHandler( + {"http": proxy, "https": proxy} + ) + + opener = urllib.request.build_opener( + urllib.request.HTTPCookieProcessor(cookiejar), proxyhandler + ) + + url = urllib.parse.urljoin(self.orig_host, handler) + parse_results = urllib.parse.urlparse(url) + + scheme = parse_results.scheme + if scheme == "persistent-http": + scheme = "http" + if scheme == "persistent-https": + # If we're proxying through persistent-https, use http. The + # proxy itself will do the https. + if proxy: + scheme = "http" + else: + scheme = "https" + + # Parse out any authentication information using the base class. + host, extra_headers, _ = self.get_host_info(parse_results.netloc) + + url = urllib.parse.urlunparse( + ( + scheme, + host, + parse_results.path, + parse_results.params, + parse_results.query, + parse_results.fragment, + ) + ) + + request = urllib.request.Request(url, request_body) + if extra_headers is not None: + for name, header in extra_headers: + request.add_header(name, header) + request.add_header("Content-Type", "text/xml") + try: + response = opener.open(request) + except urllib.error.HTTPError as e: + if e.code == 501: + # We may have been redirected through a login process + # but our POST turned into a GET. Retry. + response = opener.open(request) + else: + raise + + p, u = xmlrpc.client.getparser() + # Response should be fairly small, so read it all at once. + # This way we can show it to the user in case of error (e.g. HTML). + data = response.read() + try: + p.feed(data) + except xml.parsers.expat.ExpatError as e: + raise IOError( + f"Parsing the manifest failed: {e}\n" + f"Please report this to your manifest server admin.\n" + f'Here is the full response:\n{data.decode("utf-8")}' + ) + p.close() + return u.close() + + def close(self): + pass diff --git a/subcmds/upload.py b/subcmds/upload.py index 9c279230..63216afb 100644 --- a/subcmds/upload.py +++ b/subcmds/upload.py @@ -32,69 +32,77 @@ _DEFAULT_UNUSUAL_COMMIT_THRESHOLD = 5 def _VerifyPendingCommits(branches: List[ReviewableBranch]) -> bool: - """Perform basic safety checks on the given set of branches. - - Ensures that each branch does not have a "large" number of commits - and, if so, prompts the user to confirm they want to proceed with - the upload. - - Returns true if all branches pass the safety check or the user - confirmed. Returns false if the upload should be aborted. - """ - - # Determine if any branch has a suspicious number of commits. - many_commits = False - for branch in branches: - # Get the user's unusual threshold for the branch. - # - # Each branch may be configured to have a different threshold. - remote = branch.project.GetBranch(branch.name).remote - key = f'review.{remote.review}.uploadwarningthreshold' - threshold = branch.project.config.GetInt(key) - if threshold is None: - threshold = _DEFAULT_UNUSUAL_COMMIT_THRESHOLD - - # If the branch has more commits than the threshold, show a warning. - if len(branch.commits) > threshold: - many_commits = True - break - - # If any branch has many commits, prompt the user. - if many_commits: - if len(branches) > 1: - print('ATTENTION: One or more branches has an unusually high number ' - 'of commits.') - else: - print('ATTENTION: You are uploading an unusually high number of commits.') - print('YOU PROBABLY DO NOT MEAN TO DO THIS. (Did you rebase across ' - 'branches?)') - answer = input( - "If you are sure you intend to do this, type 'yes': ").strip() - return answer == 'yes' - - return True + """Perform basic safety checks on the given set of branches. + + Ensures that each branch does not have a "large" number of commits + and, if so, prompts the user to confirm they want to proceed with + the upload. + + Returns true if all branches pass the safety check or the user + confirmed. Returns false if the upload should be aborted. + """ + + # Determine if any branch has a suspicious number of commits. + many_commits = False + for branch in branches: + # Get the user's unusual threshold for the branch. + # + # Each branch may be configured to have a different threshold. + remote = branch.project.GetBranch(branch.name).remote + key = f"review.{remote.review}.uploadwarningthreshold" + threshold = branch.project.config.GetInt(key) + if threshold is None: + threshold = _DEFAULT_UNUSUAL_COMMIT_THRESHOLD + + # If the branch has more commits than the threshold, show a warning. + if len(branch.commits) > threshold: + many_commits = True + break + + # If any branch has many commits, prompt the user. + if many_commits: + if len(branches) > 1: + print( + "ATTENTION: One or more branches has an unusually high number " + "of commits." + ) + else: + print( + "ATTENTION: You are uploading an unusually high number of " + "commits." + ) + print( + "YOU PROBABLY DO NOT MEAN TO DO THIS. (Did you rebase across " + "branches?)" + ) + answer = input( + "If you are sure you intend to do this, type 'yes': " + ).strip() + return answer == "yes" + + return True def _die(fmt, *args): - msg = fmt % args - print('error: %s' % msg, file=sys.stderr) - sys.exit(1) + msg = fmt % args + print("error: %s" % msg, file=sys.stderr) + sys.exit(1) def _SplitEmails(values): - result = [] - for value in values: - result.extend([s.strip() for s in value.split(',')]) - return result + result = [] + for value in values: + result.extend([s.strip() for s in value.split(",")]) + return result class Upload(InteractiveCommand): - COMMON = True - helpSummary = "Upload changes for code review" - helpUsage = """ + COMMON = True + helpSummary = "Upload changes for code review" + helpUsage = """ %prog [--re --cc] []... """ - helpDescription = """ + helpDescription = """ The '%prog' command is used to send changes to the Gerrit Code Review system. It searches for topic branches in local projects that have not yet been published for review. If multiple topic @@ -195,443 +203,611 @@ threshold to a different value. Gerrit Code Review: https://www.gerritcodereview.com/ """ - PARALLEL_JOBS = DEFAULT_LOCAL_JOBS - - def _Options(self, p): - p.add_option('-t', - dest='auto_topic', action='store_true', - help='send local branch name to Gerrit Code Review') - p.add_option('--hashtag', '--ht', - dest='hashtags', action='append', default=[], - help='add hashtags (comma delimited) to the review') - p.add_option('--hashtag-branch', '--htb', - action='store_true', - help='add local branch name as a hashtag') - p.add_option('-l', '--label', - dest='labels', action='append', default=[], - help='add a label when uploading') - p.add_option('--re', '--reviewers', - type='string', action='append', dest='reviewers', - help='request reviews from these people') - p.add_option('--cc', - type='string', action='append', dest='cc', - help='also send email to these email addresses') - p.add_option('--br', '--branch', - type='string', action='store', dest='branch', - help='(local) branch to upload') - p.add_option('-c', '--current-branch', - dest='current_branch', action='store_true', - help='upload current git branch') - p.add_option('--no-current-branch', - dest='current_branch', action='store_false', - help='upload all git branches') - # Turn this into a warning & remove this someday. - p.add_option('--cbr', - dest='current_branch', action='store_true', - help=optparse.SUPPRESS_HELP) - p.add_option('--ne', '--no-emails', - action='store_false', dest='notify', default=True, - help='do not send e-mails on upload') - p.add_option('-p', '--private', - action='store_true', dest='private', default=False, - help='upload as a private change (deprecated; use --wip)') - p.add_option('-w', '--wip', - action='store_true', dest='wip', default=False, - help='upload as a work-in-progress change') - p.add_option('-r', '--ready', - action='store_true', default=False, - help='mark change as ready (clears work-in-progress setting)') - p.add_option('-o', '--push-option', - type='string', action='append', dest='push_options', - default=[], - help='additional push options to transmit') - p.add_option('-D', '--destination', '--dest', - type='string', action='store', dest='dest_branch', - metavar='BRANCH', - help='submit for review on this target branch') - p.add_option('-n', '--dry-run', - dest='dryrun', default=False, action='store_true', - help='do everything except actually upload the CL') - p.add_option('-y', '--yes', - default=False, action='store_true', - help='answer yes to all safe prompts') - p.add_option('--ignore-untracked-files', - action='store_true', default=False, - help='ignore untracked files in the working copy') - p.add_option('--no-ignore-untracked-files', - dest='ignore_untracked_files', action='store_false', - help='always ask about untracked files in the working copy') - p.add_option('--no-cert-checks', - dest='validate_certs', action='store_false', default=True, - help='disable verifying ssl certs (unsafe)') - RepoHook.AddOptionGroup(p, 'pre-upload') - - def _SingleBranch(self, opt, branch, people): - project = branch.project - name = branch.name - remote = project.GetBranch(name).remote - - key = 'review.%s.autoupload' % remote.review - answer = project.config.GetBoolean(key) - - if answer is False: - _die("upload blocked by %s = false" % key) - - if answer is None: - date = branch.date - commit_list = branch.commits - - destination = opt.dest_branch or project.dest_branch or project.revisionExpr - print('Upload project %s/ to remote branch %s%s:' % - (project.RelPath(local=opt.this_manifest_only), destination, - ' (private)' if opt.private else '')) - print(' branch %s (%2d commit%s, %s):' % ( - name, - len(commit_list), - len(commit_list) != 1 and 's' or '', - date)) - for commit in commit_list: - print(' %s' % commit) - - print('to %s (y/N)? ' % remote.review, end='', flush=True) - if opt.yes: - print('<--yes>') - answer = True - else: - answer = sys.stdin.readline().strip().lower() - answer = answer in ('y', 'yes', '1', 'true', 't') - if not answer: - _die("upload aborted by user") - - # Perform some basic safety checks prior to uploading. - if not opt.yes and not _VerifyPendingCommits([branch]): - _die("upload aborted by user") - - self._UploadAndReport(opt, [branch], people) - - def _MultipleBranches(self, opt, pending, people): - projects = {} - branches = {} - - script = [] - script.append('# Uncomment the branches to upload:') - for project, avail in pending: - project_path = project.RelPath(local=opt.this_manifest_only) - script.append('#') - script.append(f'# project {project_path}/:') - - b = {} - for branch in avail: - if branch is None: - continue + PARALLEL_JOBS = DEFAULT_LOCAL_JOBS + + def _Options(self, p): + p.add_option( + "-t", + dest="auto_topic", + action="store_true", + help="send local branch name to Gerrit Code Review", + ) + p.add_option( + "--hashtag", + "--ht", + dest="hashtags", + action="append", + default=[], + help="add hashtags (comma delimited) to the review", + ) + p.add_option( + "--hashtag-branch", + "--htb", + action="store_true", + help="add local branch name as a hashtag", + ) + p.add_option( + "-l", + "--label", + dest="labels", + action="append", + default=[], + help="add a label when uploading", + ) + p.add_option( + "--re", + "--reviewers", + type="string", + action="append", + dest="reviewers", + help="request reviews from these people", + ) + p.add_option( + "--cc", + type="string", + action="append", + dest="cc", + help="also send email to these email addresses", + ) + p.add_option( + "--br", + "--branch", + type="string", + action="store", + dest="branch", + help="(local) branch to upload", + ) + p.add_option( + "-c", + "--current-branch", + dest="current_branch", + action="store_true", + help="upload current git branch", + ) + p.add_option( + "--no-current-branch", + dest="current_branch", + action="store_false", + help="upload all git branches", + ) + # Turn this into a warning & remove this someday. + p.add_option( + "--cbr", + dest="current_branch", + action="store_true", + help=optparse.SUPPRESS_HELP, + ) + p.add_option( + "--ne", + "--no-emails", + action="store_false", + dest="notify", + default=True, + help="do not send e-mails on upload", + ) + p.add_option( + "-p", + "--private", + action="store_true", + dest="private", + default=False, + help="upload as a private change (deprecated; use --wip)", + ) + p.add_option( + "-w", + "--wip", + action="store_true", + dest="wip", + default=False, + help="upload as a work-in-progress change", + ) + p.add_option( + "-r", + "--ready", + action="store_true", + default=False, + help="mark change as ready (clears work-in-progress setting)", + ) + p.add_option( + "-o", + "--push-option", + type="string", + action="append", + dest="push_options", + default=[], + help="additional push options to transmit", + ) + p.add_option( + "-D", + "--destination", + "--dest", + type="string", + action="store", + dest="dest_branch", + metavar="BRANCH", + help="submit for review on this target branch", + ) + p.add_option( + "-n", + "--dry-run", + dest="dryrun", + default=False, + action="store_true", + help="do everything except actually upload the CL", + ) + p.add_option( + "-y", + "--yes", + default=False, + action="store_true", + help="answer yes to all safe prompts", + ) + p.add_option( + "--ignore-untracked-files", + action="store_true", + default=False, + help="ignore untracked files in the working copy", + ) + p.add_option( + "--no-ignore-untracked-files", + dest="ignore_untracked_files", + action="store_false", + help="always ask about untracked files in the working copy", + ) + p.add_option( + "--no-cert-checks", + dest="validate_certs", + action="store_false", + default=True, + help="disable verifying ssl certs (unsafe)", + ) + RepoHook.AddOptionGroup(p, "pre-upload") + + def _SingleBranch(self, opt, branch, people): + project = branch.project name = branch.name - date = branch.date - commit_list = branch.commits - - if b: - script.append('#') - destination = opt.dest_branch or project.dest_branch or project.revisionExpr - script.append('# branch %s (%2d commit%s, %s) to remote branch %s:' % ( - name, - len(commit_list), - len(commit_list) != 1 and 's' or '', - date, - destination)) - for commit in commit_list: - script.append('# %s' % commit) - b[name] = branch - - projects[project_path] = project - branches[project_path] = b - script.append('') - - script = Editor.EditString("\n".join(script)).split("\n") - - project_re = re.compile(r'^#?\s*project\s*([^\s]+)/:$') - branch_re = re.compile(r'^\s*branch\s*([^\s(]+)\s*\(.*') - - project = None - todo = [] - - for line in script: - m = project_re.match(line) - if m: - name = m.group(1) - project = projects.get(name) - if not project: - _die('project %s not available for upload', name) - continue - - m = branch_re.match(line) - if m: - name = m.group(1) - if not project: - _die('project for branch %s not in script', name) - project_path = project.RelPath(local=opt.this_manifest_only) - branch = branches[project_path].get(name) - if not branch: - _die('branch %s not in %s', name, project_path) - todo.append(branch) - if not todo: - _die("nothing uncommented for upload") - - # Perform some basic safety checks prior to uploading. - if not opt.yes and not _VerifyPendingCommits(todo): - _die("upload aborted by user") - - self._UploadAndReport(opt, todo, people) - - def _AppendAutoList(self, branch, people): - """ - Appends the list of reviewers in the git project's config. - Appends the list of users in the CC list in the git project's config if a - non-empty reviewer list was found. - """ - name = branch.name - project = branch.project - - key = 'review.%s.autoreviewer' % project.GetBranch(name).remote.review - raw_list = project.config.GetString(key) - if raw_list is not None: - people[0].extend([entry.strip() for entry in raw_list.split(',')]) - - key = 'review.%s.autocopy' % project.GetBranch(name).remote.review - raw_list = project.config.GetString(key) - if raw_list is not None and len(people[0]) > 0: - people[1].extend([entry.strip() for entry in raw_list.split(',')]) - - def _FindGerritChange(self, branch): - last_pub = branch.project.WasPublished(branch.name) - if last_pub is None: - return "" - - refs = branch.GetPublishedRefs() - try: - # refs/changes/XYZ/N --> XYZ - return refs.get(last_pub).split('/')[-2] - except (AttributeError, IndexError): - return "" - - def _UploadAndReport(self, opt, todo, original_people): - have_errors = False - for branch in todo: - try: - people = copy.deepcopy(original_people) - self._AppendAutoList(branch, people) - - # Check if there are local changes that may have been forgotten - changes = branch.project.UncommitedFiles() - if opt.ignore_untracked_files: - untracked = set(branch.project.UntrackedFiles()) - changes = [x for x in changes if x not in untracked] - - if changes: - key = 'review.%s.autoupload' % branch.project.remote.review - answer = branch.project.config.GetBoolean(key) - - # if they want to auto upload, let's not ask because it could be automated - if answer is None: - print() - print('Uncommitted changes in %s (did you forget to amend?):' - % branch.project.name) - print('\n'.join(changes)) - print('Continue uploading? (y/N) ', end='', flush=True) + remote = project.GetBranch(name).remote + + key = "review.%s.autoupload" % remote.review + answer = project.config.GetBoolean(key) + + if answer is False: + _die("upload blocked by %s = false" % key) + + if answer is None: + date = branch.date + commit_list = branch.commits + + destination = ( + opt.dest_branch or project.dest_branch or project.revisionExpr + ) + print( + "Upload project %s/ to remote branch %s%s:" + % ( + project.RelPath(local=opt.this_manifest_only), + destination, + " (private)" if opt.private else "", + ) + ) + print( + " branch %s (%2d commit%s, %s):" + % ( + name, + len(commit_list), + len(commit_list) != 1 and "s" or "", + date, + ) + ) + for commit in commit_list: + print(" %s" % commit) + + print("to %s (y/N)? " % remote.review, end="", flush=True) if opt.yes: - print('<--yes>') - a = 'yes' + print("<--yes>") + answer = True + else: + answer = sys.stdin.readline().strip().lower() + answer = answer in ("y", "yes", "1", "true", "t") + if not answer: + _die("upload aborted by user") + + # Perform some basic safety checks prior to uploading. + if not opt.yes and not _VerifyPendingCommits([branch]): + _die("upload aborted by user") + + self._UploadAndReport(opt, [branch], people) + + def _MultipleBranches(self, opt, pending, people): + projects = {} + branches = {} + + script = [] + script.append("# Uncomment the branches to upload:") + for project, avail in pending: + project_path = project.RelPath(local=opt.this_manifest_only) + script.append("#") + script.append(f"# project {project_path}/:") + + b = {} + for branch in avail: + if branch is None: + continue + name = branch.name + date = branch.date + commit_list = branch.commits + + if b: + script.append("#") + destination = ( + opt.dest_branch + or project.dest_branch + or project.revisionExpr + ) + script.append( + "# branch %s (%2d commit%s, %s) to remote branch %s:" + % ( + name, + len(commit_list), + len(commit_list) != 1 and "s" or "", + date, + destination, + ) + ) + for commit in commit_list: + script.append("# %s" % commit) + b[name] = branch + + projects[project_path] = project + branches[project_path] = b + script.append("") + + script = Editor.EditString("\n".join(script)).split("\n") + + project_re = re.compile(r"^#?\s*project\s*([^\s]+)/:$") + branch_re = re.compile(r"^\s*branch\s*([^\s(]+)\s*\(.*") + + project = None + todo = [] + + for line in script: + m = project_re.match(line) + if m: + name = m.group(1) + project = projects.get(name) + if not project: + _die("project %s not available for upload", name) + continue + + m = branch_re.match(line) + if m: + name = m.group(1) + if not project: + _die("project for branch %s not in script", name) + project_path = project.RelPath(local=opt.this_manifest_only) + branch = branches[project_path].get(name) + if not branch: + _die("branch %s not in %s", name, project_path) + todo.append(branch) + if not todo: + _die("nothing uncommented for upload") + + # Perform some basic safety checks prior to uploading. + if not opt.yes and not _VerifyPendingCommits(todo): + _die("upload aborted by user") + + self._UploadAndReport(opt, todo, people) + + def _AppendAutoList(self, branch, people): + """ + Appends the list of reviewers in the git project's config. + Appends the list of users in the CC list in the git project's config if + a non-empty reviewer list was found. + """ + name = branch.name + project = branch.project + + key = "review.%s.autoreviewer" % project.GetBranch(name).remote.review + raw_list = project.config.GetString(key) + if raw_list is not None: + people[0].extend([entry.strip() for entry in raw_list.split(",")]) + + key = "review.%s.autocopy" % project.GetBranch(name).remote.review + raw_list = project.config.GetString(key) + if raw_list is not None and len(people[0]) > 0: + people[1].extend([entry.strip() for entry in raw_list.split(",")]) + + def _FindGerritChange(self, branch): + last_pub = branch.project.WasPublished(branch.name) + if last_pub is None: + return "" + + refs = branch.GetPublishedRefs() + try: + # refs/changes/XYZ/N --> XYZ + return refs.get(last_pub).split("/")[-2] + except (AttributeError, IndexError): + return "" + + def _UploadAndReport(self, opt, todo, original_people): + have_errors = False + for branch in todo: + try: + people = copy.deepcopy(original_people) + self._AppendAutoList(branch, people) + + # Check if there are local changes that may have been forgotten. + changes = branch.project.UncommitedFiles() + if opt.ignore_untracked_files: + untracked = set(branch.project.UntrackedFiles()) + changes = [x for x in changes if x not in untracked] + + if changes: + key = "review.%s.autoupload" % branch.project.remote.review + answer = branch.project.config.GetBoolean(key) + + # If they want to auto upload, let's not ask because it + # could be automated. + if answer is None: + print() + print( + "Uncommitted changes in %s (did you forget to " + "amend?):" % branch.project.name + ) + print("\n".join(changes)) + print("Continue uploading? (y/N) ", end="", flush=True) + if opt.yes: + print("<--yes>") + a = "yes" + else: + a = sys.stdin.readline().strip().lower() + if a not in ("y", "yes", "t", "true", "on"): + print("skipping upload", file=sys.stderr) + branch.uploaded = False + branch.error = "User aborted" + continue + + # Check if topic branches should be sent to the server during + # upload. + if opt.auto_topic is not True: + key = "review.%s.uploadtopic" % branch.project.remote.review + opt.auto_topic = branch.project.config.GetBoolean(key) + + def _ExpandCommaList(value): + """Split |value| up into comma delimited entries.""" + if not value: + return + for ret in value.split(","): + ret = ret.strip() + if ret: + yield ret + + # Check if hashtags should be included. + key = "review.%s.uploadhashtags" % branch.project.remote.review + hashtags = set( + _ExpandCommaList(branch.project.config.GetString(key)) + ) + for tag in opt.hashtags: + hashtags.update(_ExpandCommaList(tag)) + if opt.hashtag_branch: + hashtags.add(branch.name) + + # Check if labels should be included. + key = "review.%s.uploadlabels" % branch.project.remote.review + labels = set( + _ExpandCommaList(branch.project.config.GetString(key)) + ) + for label in opt.labels: + labels.update(_ExpandCommaList(label)) + + # Handle e-mail notifications. + if opt.notify is False: + notify = "NONE" + else: + key = ( + "review.%s.uploadnotify" % branch.project.remote.review + ) + notify = branch.project.config.GetString(key) + + destination = opt.dest_branch or branch.project.dest_branch + + if branch.project.dest_branch and not opt.dest_branch: + merge_branch = self._GetMergeBranch( + branch.project, local_branch=branch.name + ) + + full_dest = destination + if not full_dest.startswith(R_HEADS): + full_dest = R_HEADS + full_dest + + # If the merge branch of the local branch is different from + # the project's revision AND destination, this might not be + # intentional. + if ( + merge_branch + and merge_branch != branch.project.revisionExpr + and merge_branch != full_dest + ): + print( + f"For local branch {branch.name}: merge branch " + f"{merge_branch} does not match destination branch " + f"{destination}" + ) + print("skipping upload.") + print( + f"Please use `--destination {destination}` if this " + "is intentional" + ) + branch.uploaded = False + continue + + branch.UploadForReview( + people, + dryrun=opt.dryrun, + auto_topic=opt.auto_topic, + hashtags=hashtags, + labels=labels, + private=opt.private, + notify=notify, + wip=opt.wip, + ready=opt.ready, + dest_branch=destination, + validate_certs=opt.validate_certs, + push_options=opt.push_options, + ) + + branch.uploaded = True + except UploadError as e: + branch.error = e + branch.uploaded = False + have_errors = True + + print(file=sys.stderr) + print("-" * 70, file=sys.stderr) + + if have_errors: + for branch in todo: + if not branch.uploaded: + if len(str(branch.error)) <= 30: + fmt = " (%s)" + else: + fmt = "\n (%s)" + print( + ("[FAILED] %-15s %-15s" + fmt) + % ( + branch.project.RelPath(local=opt.this_manifest_only) + + "/", + branch.name, + str(branch.error), + ), + file=sys.stderr, + ) + print() + + for branch in todo: + if branch.uploaded: + print( + "[OK ] %-15s %s" + % ( + branch.project.RelPath(local=opt.this_manifest_only) + + "/", + branch.name, + ), + file=sys.stderr, + ) + + if have_errors: + sys.exit(1) + + def _GetMergeBranch(self, project, local_branch=None): + if local_branch is None: + p = GitCommand( + project, + ["rev-parse", "--abbrev-ref", "HEAD"], + capture_stdout=True, + capture_stderr=True, + ) + p.Wait() + local_branch = p.stdout.strip() + p = GitCommand( + project, + ["config", "--get", "branch.%s.merge" % local_branch], + capture_stdout=True, + capture_stderr=True, + ) + p.Wait() + merge_branch = p.stdout.strip() + return merge_branch + + @staticmethod + def _GatherOne(opt, project): + """Figure out the upload status for |project|.""" + if opt.current_branch: + cbr = project.CurrentBranch + up_branch = project.GetUploadableBranch(cbr) + avail = [up_branch] if up_branch else None + else: + avail = project.GetUploadableBranches(opt.branch) + return (project, avail) + + def Execute(self, opt, args): + projects = self.GetProjects( + args, all_manifests=not opt.this_manifest_only + ) + + def _ProcessResults(_pool, _out, results): + pending = [] + for result in results: + project, avail = result + if avail is None: + print( + 'repo: error: %s: Unable to upload branch "%s". ' + "You might be able to fix the branch by running:\n" + " git branch --set-upstream-to m/%s" + % ( + project.RelPath(local=opt.this_manifest_only), + project.CurrentBranch, + project.manifest.branch, + ), + file=sys.stderr, + ) + elif avail: + pending.append(result) + return pending + + pending = self.ExecuteInParallel( + opt.jobs, + functools.partial(self._GatherOne, opt), + projects, + callback=_ProcessResults, + ) + + if not pending: + if opt.branch is None: + print( + "repo: error: no branches ready for upload", file=sys.stderr + ) else: - a = sys.stdin.readline().strip().lower() - if a not in ('y', 'yes', 't', 'true', 'on'): - print("skipping upload", file=sys.stderr) - branch.uploaded = False - branch.error = 'User aborted' - continue - - # Check if topic branches should be sent to the server during upload - if opt.auto_topic is not True: - key = 'review.%s.uploadtopic' % branch.project.remote.review - opt.auto_topic = branch.project.config.GetBoolean(key) - - def _ExpandCommaList(value): - """Split |value| up into comma delimited entries.""" - if not value: - return - for ret in value.split(','): - ret = ret.strip() - if ret: - yield ret - - # Check if hashtags should be included. - key = 'review.%s.uploadhashtags' % branch.project.remote.review - hashtags = set(_ExpandCommaList(branch.project.config.GetString(key))) - for tag in opt.hashtags: - hashtags.update(_ExpandCommaList(tag)) - if opt.hashtag_branch: - hashtags.add(branch.name) - - # Check if labels should be included. - key = 'review.%s.uploadlabels' % branch.project.remote.review - labels = set(_ExpandCommaList(branch.project.config.GetString(key))) - for label in opt.labels: - labels.update(_ExpandCommaList(label)) - - # Handle e-mail notifications. - if opt.notify is False: - notify = 'NONE' + print( + 'repo: error: no branches named "%s" ready for upload' + % (opt.branch,), + file=sys.stderr, + ) + return 1 + + manifests = { + project.manifest.topdir: project.manifest + for (project, available) in pending + } + ret = 0 + for manifest in manifests.values(): + pending_proj_names = [ + project.name + for (project, available) in pending + if project.manifest.topdir == manifest.topdir + ] + pending_worktrees = [ + project.worktree + for (project, available) in pending + if project.manifest.topdir == manifest.topdir + ] + hook = RepoHook.FromSubcmd( + hook_type="pre-upload", + manifest=manifest, + opt=opt, + abort_if_user_denies=True, + ) + if not hook.Run( + project_list=pending_proj_names, worktree_list=pending_worktrees + ): + ret = 1 + if ret: + return ret + + reviewers = _SplitEmails(opt.reviewers) if opt.reviewers else [] + cc = _SplitEmails(opt.cc) if opt.cc else [] + people = (reviewers, cc) + + if len(pending) == 1 and len(pending[0][1]) == 1: + self._SingleBranch(opt, pending[0][1][0], people) else: - key = 'review.%s.uploadnotify' % branch.project.remote.review - notify = branch.project.config.GetString(key) - - destination = opt.dest_branch or branch.project.dest_branch - - if branch.project.dest_branch and not opt.dest_branch: - - merge_branch = self._GetMergeBranch( - branch.project, local_branch=branch.name) - - full_dest = destination - if not full_dest.startswith(R_HEADS): - full_dest = R_HEADS + full_dest - - # If the merge branch of the local branch is different from the - # project's revision AND destination, this might not be intentional. - if (merge_branch and merge_branch != branch.project.revisionExpr - and merge_branch != full_dest): - print(f'For local branch {branch.name}: merge branch ' - f'{merge_branch} does not match destination branch ' - f'{destination}') - print('skipping upload.') - print(f'Please use `--destination {destination}` if this is intentional') - branch.uploaded = False - continue - - branch.UploadForReview(people, - dryrun=opt.dryrun, - auto_topic=opt.auto_topic, - hashtags=hashtags, - labels=labels, - private=opt.private, - notify=notify, - wip=opt.wip, - ready=opt.ready, - dest_branch=destination, - validate_certs=opt.validate_certs, - push_options=opt.push_options) - - branch.uploaded = True - except UploadError as e: - branch.error = e - branch.uploaded = False - have_errors = True - - print(file=sys.stderr) - print('----------------------------------------------------------------------', file=sys.stderr) - - if have_errors: - for branch in todo: - if not branch.uploaded: - if len(str(branch.error)) <= 30: - fmt = ' (%s)' - else: - fmt = '\n (%s)' - print(('[FAILED] %-15s %-15s' + fmt) % ( - branch.project.RelPath(local=opt.this_manifest_only) + '/', - branch.name, - str(branch.error)), - file=sys.stderr) - print() - - for branch in todo: - if branch.uploaded: - print('[OK ] %-15s %s' % ( - branch.project.RelPath(local=opt.this_manifest_only) + '/', - branch.name), - file=sys.stderr) - - if have_errors: - sys.exit(1) - - def _GetMergeBranch(self, project, local_branch=None): - if local_branch is None: - p = GitCommand(project, - ['rev-parse', '--abbrev-ref', 'HEAD'], - capture_stdout=True, - capture_stderr=True) - p.Wait() - local_branch = p.stdout.strip() - p = GitCommand(project, - ['config', '--get', 'branch.%s.merge' % local_branch], - capture_stdout=True, - capture_stderr=True) - p.Wait() - merge_branch = p.stdout.strip() - return merge_branch - - @staticmethod - def _GatherOne(opt, project): - """Figure out the upload status for |project|.""" - if opt.current_branch: - cbr = project.CurrentBranch - up_branch = project.GetUploadableBranch(cbr) - avail = [up_branch] if up_branch else None - else: - avail = project.GetUploadableBranches(opt.branch) - return (project, avail) - - def Execute(self, opt, args): - projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only) - - def _ProcessResults(_pool, _out, results): - pending = [] - for result in results: - project, avail = result - if avail is None: - print('repo: error: %s: Unable to upload branch "%s". ' - 'You might be able to fix the branch by running:\n' - ' git branch --set-upstream-to m/%s' % - (project.RelPath(local=opt.this_manifest_only), project.CurrentBranch, - project.manifest.branch), - file=sys.stderr) - elif avail: - pending.append(result) - return pending - - pending = self.ExecuteInParallel( - opt.jobs, - functools.partial(self._GatherOne, opt), - projects, - callback=_ProcessResults) - - if not pending: - if opt.branch is None: - print('repo: error: no branches ready for upload', file=sys.stderr) - else: - print('repo: error: no branches named "%s" ready for upload' % - (opt.branch,), file=sys.stderr) - return 1 - - manifests = {project.manifest.topdir: project.manifest - for (project, available) in pending} - ret = 0 - for manifest in manifests.values(): - pending_proj_names = [project.name for (project, available) in pending - if project.manifest.topdir == manifest.topdir] - pending_worktrees = [project.worktree for (project, available) in pending - if project.manifest.topdir == manifest.topdir] - hook = RepoHook.FromSubcmd( - hook_type='pre-upload', manifest=manifest, - opt=opt, abort_if_user_denies=True) - if not hook.Run(project_list=pending_proj_names, - worktree_list=pending_worktrees): - ret = 1 - if ret: - return ret - - reviewers = _SplitEmails(opt.reviewers) if opt.reviewers else [] - cc = _SplitEmails(opt.cc) if opt.cc else [] - people = (reviewers, cc) - - if len(pending) == 1 and len(pending[0][1]) == 1: - self._SingleBranch(opt, pending[0][1][0], people) - else: - self._MultipleBranches(opt, pending, people) + self._MultipleBranches(opt, pending, people) diff --git a/subcmds/version.py b/subcmds/version.py index c68cb0af..c539db63 100644 --- a/subcmds/version.py +++ b/subcmds/version.py @@ -22,45 +22,52 @@ from wrapper import Wrapper class Version(Command, MirrorSafeCommand): - wrapper_version = None - wrapper_path = None + wrapper_version = None + wrapper_path = None - COMMON = False - helpSummary = "Display the version of repo" - helpUsage = """ + COMMON = False + helpSummary = "Display the version of repo" + helpUsage = """ %prog """ - def Execute(self, opt, args): - rp = self.manifest.repoProject - rem = rp.GetRemote() - branch = rp.GetBranch('default') + def Execute(self, opt, args): + rp = self.manifest.repoProject + rem = rp.GetRemote() + branch = rp.GetBranch("default") - # These might not be the same. Report them both. - src_ver = RepoSourceVersion() - rp_ver = rp.bare_git.describe(HEAD) - print('repo version %s' % rp_ver) - print(' (from %s)' % rem.url) - print(' (tracking %s)' % branch.merge) - print(' (%s)' % rp.bare_git.log('-1', '--format=%cD', HEAD)) + # These might not be the same. Report them both. + src_ver = RepoSourceVersion() + rp_ver = rp.bare_git.describe(HEAD) + print("repo version %s" % rp_ver) + print(" (from %s)" % rem.url) + print(" (tracking %s)" % branch.merge) + print(" (%s)" % rp.bare_git.log("-1", "--format=%cD", HEAD)) - if self.wrapper_path is not None: - print('repo launcher version %s' % self.wrapper_version) - print(' (from %s)' % self.wrapper_path) + if self.wrapper_path is not None: + print("repo launcher version %s" % self.wrapper_version) + print(" (from %s)" % self.wrapper_path) - if src_ver != rp_ver: - print(' (currently at %s)' % src_ver) + if src_ver != rp_ver: + print(" (currently at %s)" % src_ver) - print('repo User-Agent %s' % user_agent.repo) - print('git %s' % git.version_tuple().full) - print('git User-Agent %s' % user_agent.git) - print('Python %s' % sys.version) - uname = platform.uname() - if sys.version_info.major < 3: - # Python 3 returns a named tuple, but Python 2 is simpler. - print(uname) - else: - print('OS %s %s (%s)' % (uname.system, uname.release, uname.version)) - print('CPU %s (%s)' % - (uname.machine, uname.processor if uname.processor else 'unknown')) - print('Bug reports:', Wrapper().BUG_URL) + print("repo User-Agent %s" % user_agent.repo) + print("git %s" % git.version_tuple().full) + print("git User-Agent %s" % user_agent.git) + print("Python %s" % sys.version) + uname = platform.uname() + if sys.version_info.major < 3: + # Python 3 returns a named tuple, but Python 2 is simpler. + print(uname) + else: + print( + "OS %s %s (%s)" % (uname.system, uname.release, uname.version) + ) + print( + "CPU %s (%s)" + % ( + uname.machine, + uname.processor if uname.processor else "unknown", + ) + ) + print("Bug reports:", Wrapper().BUG_URL) -- cgit v1.2.3-54-g00ecf