From ea2e330e43c182dc16b0111ebc69ee5a71ee4ce1 Mon Sep 17 00:00:00 2001 From: Gavin Mak Date: Sat, 11 Mar 2023 06:46:20 +0000 Subject: Format codebase with black and check formatting in CQ Apply rules set by https://gerrit-review.googlesource.com/c/git-repo/+/362954/ across the codebase and fix any lingering errors caught by flake8. Also check black formatting in run_tests (and CQ). Bug: b/267675342 Change-Id: I972d77649dac351150dcfeb1cd1ad0ea2efc1956 Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/363474 Reviewed-by: Mike Frysinger Tested-by: Gavin Mak Commit-Queue: Gavin Mak --- subcmds/forall.py | 516 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 285 insertions(+), 231 deletions(-) (limited to 'subcmds/forall.py') diff --git a/subcmds/forall.py b/subcmds/forall.py index f9f34e33..0a897357 100644 --- a/subcmds/forall.py +++ b/subcmds/forall.py @@ -23,31 +23,36 @@ import sys import subprocess from color import Coloring -from command import DEFAULT_LOCAL_JOBS, Command, MirrorSafeCommand, WORKER_BATCH_SIZE +from command import ( + DEFAULT_LOCAL_JOBS, + Command, + MirrorSafeCommand, + WORKER_BATCH_SIZE, +) from error import ManifestInvalidRevisionError _CAN_COLOR = [ - 'branch', - 'diff', - 'grep', - 'log', + "branch", + "diff", + "grep", + "log", ] class ForallColoring(Coloring): - def __init__(self, config): - Coloring.__init__(self, config, 'forall') - self.project = self.printer('project', attr='bold') + def __init__(self, config): + Coloring.__init__(self, config, "forall") + self.project = self.printer("project", attr="bold") class Forall(Command, MirrorSafeCommand): - COMMON = False - helpSummary = "Run a shell command in each project" - helpUsage = """ + COMMON = False + helpSummary = "Run a shell command in each project" + helpUsage = """ %prog [...] -c [...] %prog -r str1 [str2] ... -c [...] """ - helpDescription = """ + helpDescription = """ Executes the same shell command in each project. The -r option allows running the command only on projects matching @@ -125,236 +130,285 @@ terminal and are not redirected. If -e is used, when a command exits unsuccessfully, '%prog' will abort without iterating through the remaining projects. """ - PARALLEL_JOBS = DEFAULT_LOCAL_JOBS - - @staticmethod - def _cmd_option(option, _opt_str, _value, parser): - setattr(parser.values, option.dest, list(parser.rargs)) - while parser.rargs: - del parser.rargs[0] - - def _Options(self, p): - p.add_option('-r', '--regex', - dest='regex', action='store_true', - help='execute the command only on projects matching regex or wildcard expression') - p.add_option('-i', '--inverse-regex', - dest='inverse_regex', action='store_true', - help='execute the command only on projects not matching regex or ' - 'wildcard expression') - p.add_option('-g', '--groups', - dest='groups', - help='execute the command only on projects matching the specified groups') - p.add_option('-c', '--command', - help='command (and arguments) to execute', - dest='command', - action='callback', - callback=self._cmd_option) - p.add_option('-e', '--abort-on-errors', - dest='abort_on_errors', action='store_true', - help='abort if a command exits unsuccessfully') - p.add_option('--ignore-missing', action='store_true', - help='silently skip & do not exit non-zero due missing ' - 'checkouts') - - g = p.get_option_group('--quiet') - g.add_option('-p', - dest='project_header', action='store_true', - help='show project headers before output') - p.add_option('--interactive', - action='store_true', - help='force interactive usage') - - def WantPager(self, opt): - return opt.project_header and opt.jobs == 1 - - def ValidateOptions(self, opt, args): - if not opt.command: - self.Usage() - - def Execute(self, opt, args): - cmd = [opt.command[0]] - all_trees = not opt.this_manifest_only - - shell = True - if re.compile(r'^[a-z0-9A-Z_/\.-]+$').match(cmd[0]): - shell = False - - if shell: - cmd.append(cmd[0]) - cmd.extend(opt.command[1:]) - - # Historically, forall operated interactively, and in serial. If the user - # has selected 1 job, then default to interacive mode. - if opt.jobs == 1: - opt.interactive = True - - if opt.project_header \ - and not shell \ - and cmd[0] == 'git': - # If this is a direct git command that can enable colorized - # output and the user prefers coloring, add --color into the - # command line because we are going to wrap the command into - # a pipe and git won't know coloring should activate. - # - for cn in cmd[1:]: - if not cn.startswith('-'): - break - else: - cn = None - if cn and cn in _CAN_COLOR: - class ColorCmd(Coloring): - def __init__(self, config, cmd): - Coloring.__init__(self, config, cmd) - if ColorCmd(self.manifest.manifestProject.config, cn).is_on: - cmd.insert(cmd.index(cn) + 1, '--color') - - mirror = self.manifest.IsMirror - rc = 0 - - smart_sync_manifest_name = "smart_sync_override.xml" - smart_sync_manifest_path = os.path.join( - self.manifest.manifestProject.worktree, smart_sync_manifest_name) - - if os.path.isfile(smart_sync_manifest_path): - self.manifest.Override(smart_sync_manifest_path) - - if opt.regex: - projects = self.FindProjects(args, all_manifests=all_trees) - elif opt.inverse_regex: - projects = self.FindProjects(args, inverse=True, all_manifests=all_trees) - else: - projects = self.GetProjects(args, groups=opt.groups, all_manifests=all_trees) - - os.environ['REPO_COUNT'] = str(len(projects)) - - try: - config = self.manifest.manifestProject.config - with multiprocessing.Pool(opt.jobs, InitWorker) as pool: - results_it = pool.imap( - functools.partial(DoWorkWrapper, mirror, opt, cmd, shell, config), - enumerate(projects), - chunksize=WORKER_BATCH_SIZE) - first = True - for (r, output) in results_it: - if output: - if first: - first = False - elif opt.project_header: - print() - # To simplify the DoWorkWrapper, take care of automatic newlines. - end = '\n' - if output[-1] == '\n': - end = '' - print(output, end=end) - rc = rc or r - if r != 0 and opt.abort_on_errors: - raise Exception('Aborting due to previous error') - except (KeyboardInterrupt, WorkerKeyboardInterrupt): - # Catch KeyboardInterrupt raised inside and outside of workers - rc = rc or errno.EINTR - except Exception as e: - # Catch any other exceptions raised - print('forall: unhandled error, terminating the pool: %s: %s' % - (type(e).__name__, e), - file=sys.stderr) - rc = rc or getattr(e, 'errno', 1) - if rc != 0: - sys.exit(rc) + PARALLEL_JOBS = DEFAULT_LOCAL_JOBS + + @staticmethod + def _cmd_option(option, _opt_str, _value, parser): + setattr(parser.values, option.dest, list(parser.rargs)) + while parser.rargs: + del parser.rargs[0] + + def _Options(self, p): + p.add_option( + "-r", + "--regex", + dest="regex", + action="store_true", + help="execute the command only on projects matching regex or " + "wildcard expression", + ) + p.add_option( + "-i", + "--inverse-regex", + dest="inverse_regex", + action="store_true", + help="execute the command only on projects not matching regex or " + "wildcard expression", + ) + p.add_option( + "-g", + "--groups", + dest="groups", + help="execute the command only on projects matching the specified " + "groups", + ) + p.add_option( + "-c", + "--command", + help="command (and arguments) to execute", + dest="command", + action="callback", + callback=self._cmd_option, + ) + p.add_option( + "-e", + "--abort-on-errors", + dest="abort_on_errors", + action="store_true", + help="abort if a command exits unsuccessfully", + ) + p.add_option( + "--ignore-missing", + action="store_true", + help="silently skip & do not exit non-zero due missing " + "checkouts", + ) + + g = p.get_option_group("--quiet") + g.add_option( + "-p", + dest="project_header", + action="store_true", + help="show project headers before output", + ) + p.add_option( + "--interactive", action="store_true", help="force interactive usage" + ) + + def WantPager(self, opt): + return opt.project_header and opt.jobs == 1 + + def ValidateOptions(self, opt, args): + if not opt.command: + self.Usage() + + def Execute(self, opt, args): + cmd = [opt.command[0]] + all_trees = not opt.this_manifest_only + + shell = True + if re.compile(r"^[a-z0-9A-Z_/\.-]+$").match(cmd[0]): + shell = False + + if shell: + cmd.append(cmd[0]) + cmd.extend(opt.command[1:]) + + # Historically, forall operated interactively, and in serial. If the + # user has selected 1 job, then default to interacive mode. + if opt.jobs == 1: + opt.interactive = True + + if opt.project_header and not shell and cmd[0] == "git": + # If this is a direct git command that can enable colorized + # output and the user prefers coloring, add --color into the + # command line because we are going to wrap the command into + # a pipe and git won't know coloring should activate. + # + for cn in cmd[1:]: + if not cn.startswith("-"): + break + else: + cn = None + if cn and cn in _CAN_COLOR: + + class ColorCmd(Coloring): + def __init__(self, config, cmd): + Coloring.__init__(self, config, cmd) + + if ColorCmd(self.manifest.manifestProject.config, cn).is_on: + cmd.insert(cmd.index(cn) + 1, "--color") + + mirror = self.manifest.IsMirror + rc = 0 + + smart_sync_manifest_name = "smart_sync_override.xml" + smart_sync_manifest_path = os.path.join( + self.manifest.manifestProject.worktree, smart_sync_manifest_name + ) + + if os.path.isfile(smart_sync_manifest_path): + self.manifest.Override(smart_sync_manifest_path) + + if opt.regex: + projects = self.FindProjects(args, all_manifests=all_trees) + elif opt.inverse_regex: + projects = self.FindProjects( + args, inverse=True, all_manifests=all_trees + ) + else: + projects = self.GetProjects( + args, groups=opt.groups, all_manifests=all_trees + ) + + os.environ["REPO_COUNT"] = str(len(projects)) + + try: + config = self.manifest.manifestProject.config + with multiprocessing.Pool(opt.jobs, InitWorker) as pool: + results_it = pool.imap( + functools.partial( + DoWorkWrapper, mirror, opt, cmd, shell, config + ), + enumerate(projects), + chunksize=WORKER_BATCH_SIZE, + ) + first = True + for r, output in results_it: + if output: + if first: + first = False + elif opt.project_header: + print() + # To simplify the DoWorkWrapper, take care of automatic + # newlines. + end = "\n" + if output[-1] == "\n": + end = "" + print(output, end=end) + rc = rc or r + if r != 0 and opt.abort_on_errors: + raise Exception("Aborting due to previous error") + except (KeyboardInterrupt, WorkerKeyboardInterrupt): + # Catch KeyboardInterrupt raised inside and outside of workers + rc = rc or errno.EINTR + except Exception as e: + # Catch any other exceptions raised + print( + "forall: unhandled error, terminating the pool: %s: %s" + % (type(e).__name__, e), + file=sys.stderr, + ) + rc = rc or getattr(e, "errno", 1) + if rc != 0: + sys.exit(rc) class WorkerKeyboardInterrupt(Exception): - """ Keyboard interrupt exception for worker processes. """ + """Keyboard interrupt exception for worker processes.""" def InitWorker(): - signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGINT, signal.SIG_IGN) def DoWorkWrapper(mirror, opt, cmd, shell, config, args): - """ A wrapper around the DoWork() method. + """A wrapper around the DoWork() method. - Catch the KeyboardInterrupt exceptions here and re-raise them as a different, - ``Exception``-based exception to stop it flooding the console with stacktraces - and making the parent hang indefinitely. + Catch the KeyboardInterrupt exceptions here and re-raise them as a + different, ``Exception``-based exception to stop it flooding the console + with stacktraces and making the parent hang indefinitely. - """ - cnt, project = args - try: - return DoWork(project, mirror, opt, cmd, shell, cnt, config) - except KeyboardInterrupt: - print('%s: Worker interrupted' % project.name) - raise WorkerKeyboardInterrupt() + """ + cnt, project = args + try: + return DoWork(project, mirror, opt, cmd, shell, cnt, config) + except KeyboardInterrupt: + print("%s: Worker interrupted" % project.name) + raise WorkerKeyboardInterrupt() def DoWork(project, mirror, opt, cmd, shell, cnt, config): - env = os.environ.copy() - - def setenv(name, val): - if val is None: - val = '' - env[name] = val - - setenv('REPO_PROJECT', project.name) - setenv('REPO_OUTERPATH', project.manifest.path_prefix) - setenv('REPO_INNERPATH', project.relpath) - setenv('REPO_PATH', project.RelPath(local=opt.this_manifest_only)) - setenv('REPO_REMOTE', project.remote.name) - try: - # If we aren't in a fully synced state and we don't have the ref the manifest - # wants, then this will fail. Ignore it for the purposes of this code. - lrev = '' if mirror else project.GetRevisionId() - except ManifestInvalidRevisionError: - lrev = '' - setenv('REPO_LREV', lrev) - setenv('REPO_RREV', project.revisionExpr) - setenv('REPO_UPSTREAM', project.upstream) - setenv('REPO_DEST_BRANCH', project.dest_branch) - setenv('REPO_I', str(cnt + 1)) - for annotation in project.annotations: - setenv("REPO__%s" % (annotation.name), annotation.value) - - if mirror: - setenv('GIT_DIR', project.gitdir) - cwd = project.gitdir - else: - cwd = project.worktree - - if not os.path.exists(cwd): - # Allow the user to silently ignore missing checkouts so they can run on - # partial checkouts (good for infra recovery tools). - if opt.ignore_missing: - return (0, '') - - output = '' - if ((opt.project_header and opt.verbose) - or not opt.project_header): - output = 'skipping %s/' % project.RelPath(local=opt.this_manifest_only) - return (1, output) - - if opt.verbose: - stderr = subprocess.STDOUT - else: - stderr = subprocess.DEVNULL - - stdin = None if opt.interactive else subprocess.DEVNULL - - result = subprocess.run( - cmd, cwd=cwd, shell=shell, env=env, check=False, - encoding='utf-8', errors='replace', - stdin=stdin, stdout=subprocess.PIPE, stderr=stderr) - - output = result.stdout - if opt.project_header: - if output: - buf = io.StringIO() - out = ForallColoring(config) - out.redirect(buf) - if mirror: - project_header_path = project.name - else: - project_header_path = project.RelPath(local=opt.this_manifest_only) - out.project('project %s/' % project_header_path) - out.nl() - buf.write(output) - output = buf.getvalue() - return (result.returncode, output) + env = os.environ.copy() + + def setenv(name, val): + if val is None: + val = "" + env[name] = val + + setenv("REPO_PROJECT", project.name) + setenv("REPO_OUTERPATH", project.manifest.path_prefix) + setenv("REPO_INNERPATH", project.relpath) + setenv("REPO_PATH", project.RelPath(local=opt.this_manifest_only)) + setenv("REPO_REMOTE", project.remote.name) + try: + # If we aren't in a fully synced state and we don't have the ref the + # manifest wants, then this will fail. Ignore it for the purposes of + # this code. + lrev = "" if mirror else project.GetRevisionId() + except ManifestInvalidRevisionError: + lrev = "" + setenv("REPO_LREV", lrev) + setenv("REPO_RREV", project.revisionExpr) + setenv("REPO_UPSTREAM", project.upstream) + setenv("REPO_DEST_BRANCH", project.dest_branch) + setenv("REPO_I", str(cnt + 1)) + for annotation in project.annotations: + setenv("REPO__%s" % (annotation.name), annotation.value) + + if mirror: + setenv("GIT_DIR", project.gitdir) + cwd = project.gitdir + else: + cwd = project.worktree + + if not os.path.exists(cwd): + # Allow the user to silently ignore missing checkouts so they can run on + # partial checkouts (good for infra recovery tools). + if opt.ignore_missing: + return (0, "") + + output = "" + if (opt.project_header and opt.verbose) or not opt.project_header: + output = "skipping %s/" % project.RelPath( + local=opt.this_manifest_only + ) + return (1, output) + + if opt.verbose: + stderr = subprocess.STDOUT + else: + stderr = subprocess.DEVNULL + + stdin = None if opt.interactive else subprocess.DEVNULL + + result = subprocess.run( + cmd, + cwd=cwd, + shell=shell, + env=env, + check=False, + encoding="utf-8", + errors="replace", + stdin=stdin, + stdout=subprocess.PIPE, + stderr=stderr, + ) + + output = result.stdout + if opt.project_header: + if output: + buf = io.StringIO() + out = ForallColoring(config) + out.redirect(buf) + if mirror: + project_header_path = project.name + else: + project_header_path = project.RelPath( + local=opt.this_manifest_only + ) + out.project("project %s/" % project_header_path) + out.nl() + buf.write(output) + output = buf.getvalue() + return (result.returncode, output) -- cgit v1.2.3-54-g00ecf