From ea2e330e43c182dc16b0111ebc69ee5a71ee4ce1 Mon Sep 17 00:00:00 2001 From: Gavin Mak Date: Sat, 11 Mar 2023 06:46:20 +0000 Subject: Format codebase with black and check formatting in CQ Apply rules set by https://gerrit-review.googlesource.com/c/git-repo/+/362954/ across the codebase and fix any lingering errors caught by flake8. Also check black formatting in run_tests (and CQ). Bug: b/267675342 Change-Id: I972d77649dac351150dcfeb1cd1ad0ea2efc1956 Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/363474 Reviewed-by: Mike Frysinger Tested-by: Gavin Mak Commit-Queue: Gavin Mak --- gitc_utils.py | 231 ++++++++++++++++++++++++++++++---------------------------- 1 file changed, 121 insertions(+), 110 deletions(-) (limited to 'gitc_utils.py') diff --git a/gitc_utils.py b/gitc_utils.py index dfcfd2a4..7b72048f 100644 --- a/gitc_utils.py +++ b/gitc_utils.py @@ -28,128 +28,139 @@ NUM_BATCH_RETRIEVE_REVISIONID = 32 def get_gitc_manifest_dir(): - return wrapper.Wrapper().get_gitc_manifest_dir() + return wrapper.Wrapper().get_gitc_manifest_dir() def parse_clientdir(gitc_fs_path): - return wrapper.Wrapper().gitc_parse_clientdir(gitc_fs_path) + return wrapper.Wrapper().gitc_parse_clientdir(gitc_fs_path) def _get_project_revision(args): - """Worker for _set_project_revisions to lookup one project remote.""" - (i, url, expr) = args - gitcmd = git_command.GitCommand( - None, ['ls-remote', url, expr], capture_stdout=True, cwd='/tmp') - rc = gitcmd.Wait() - return (i, rc, gitcmd.stdout.split('\t', 1)[0]) + """Worker for _set_project_revisions to lookup one project remote.""" + (i, url, expr) = args + gitcmd = git_command.GitCommand( + None, ["ls-remote", url, expr], capture_stdout=True, cwd="/tmp" + ) + rc = gitcmd.Wait() + return (i, rc, gitcmd.stdout.split("\t", 1)[0]) def _set_project_revisions(projects): - """Sets the revisionExpr for a list of projects. - - Because of the limit of open file descriptors allowed, length of projects - should not be overly large. Recommend calling this function multiple times - with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects. - - Args: - projects: List of project objects to set the revionExpr for. - """ - # Retrieve the commit id for each project based off of it's current - # revisionExpr and it is not already a commit id. - with multiprocessing.Pool(NUM_BATCH_RETRIEVE_REVISIONID) as pool: - results_iter = pool.imap_unordered( - _get_project_revision, - ((i, project.remote.url, project.revisionExpr) - for i, project in enumerate(projects) - if not git_config.IsId(project.revisionExpr)), - chunksize=8) - for (i, rc, revisionExpr) in results_iter: - project = projects[i] - if rc: - print('FATAL: Failed to retrieve revisionExpr for %s' % project.name) - pool.terminate() - sys.exit(1) - if not revisionExpr: - pool.terminate() - raise ManifestParseError('Invalid SHA-1 revision project %s (%s)' % - (project.remote.url, project.revisionExpr)) - project.revisionExpr = revisionExpr + """Sets the revisionExpr for a list of projects. + + Because of the limit of open file descriptors allowed, length of projects + should not be overly large. Recommend calling this function multiple times + with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects. + + Args: + projects: List of project objects to set the revionExpr for. + """ + # Retrieve the commit id for each project based off of its current + # revisionExpr and it is not already a commit id. + with multiprocessing.Pool(NUM_BATCH_RETRIEVE_REVISIONID) as pool: + results_iter = pool.imap_unordered( + _get_project_revision, + ( + (i, project.remote.url, project.revisionExpr) + for i, project in enumerate(projects) + if not git_config.IsId(project.revisionExpr) + ), + chunksize=8, + ) + for i, rc, revisionExpr in results_iter: + project = projects[i] + if rc: + print( + "FATAL: Failed to retrieve revisionExpr for %s" + % project.name + ) + pool.terminate() + sys.exit(1) + if not revisionExpr: + pool.terminate() + raise ManifestParseError( + "Invalid SHA-1 revision project %s (%s)" + % (project.remote.url, project.revisionExpr) + ) + project.revisionExpr = revisionExpr def generate_gitc_manifest(gitc_manifest, manifest, paths=None): - """Generate a manifest for shafsd to use for this GITC client. - - Args: - gitc_manifest: Current gitc manifest, or None if there isn't one yet. - manifest: A GitcManifest object loaded with the current repo manifest. - paths: List of project paths we want to update. - """ - - print('Generating GITC Manifest by fetching revision SHAs for each ' - 'project.') - if paths is None: - paths = list(manifest.paths.keys()) - - groups = [x for x in re.split(r'[,\s]+', manifest.GetGroupsStr()) if x] - - # Convert the paths to projects, and filter them to the matched groups. - projects = [manifest.paths[p] for p in paths] - projects = [p for p in projects if p.MatchesGroups(groups)] - - if gitc_manifest is not None: - for path, proj in manifest.paths.items(): - if not proj.MatchesGroups(groups): - continue - - if not proj.upstream and not git_config.IsId(proj.revisionExpr): - proj.upstream = proj.revisionExpr - - if path not in gitc_manifest.paths: - # Any new projects need their first revision, even if we weren't asked - # for them. - projects.append(proj) - elif path not in paths: - # And copy revisions from the previous manifest if we're not updating - # them now. - gitc_proj = gitc_manifest.paths[path] - if gitc_proj.old_revision: - proj.revisionExpr = None - proj.old_revision = gitc_proj.old_revision - else: - proj.revisionExpr = gitc_proj.revisionExpr - - _set_project_revisions(projects) - - if gitc_manifest is not None: - for path, proj in gitc_manifest.paths.items(): - if proj.old_revision and path in paths: - # If we updated a project that has been started, keep the old-revision - # updated. - repo_proj = manifest.paths[path] - repo_proj.old_revision = repo_proj.revisionExpr - repo_proj.revisionExpr = None - - # Convert URLs from relative to absolute. - for _name, remote in manifest.remotes.items(): - remote.fetchUrl = remote.resolvedFetchUrl - - # Save the manifest. - save_manifest(manifest) + """Generate a manifest for shafsd to use for this GITC client. + + Args: + gitc_manifest: Current gitc manifest, or None if there isn't one yet. + manifest: A GitcManifest object loaded with the current repo manifest. + paths: List of project paths we want to update. + """ + + print( + "Generating GITC Manifest by fetching revision SHAs for each " + "project." + ) + if paths is None: + paths = list(manifest.paths.keys()) + + groups = [x for x in re.split(r"[,\s]+", manifest.GetGroupsStr()) if x] + + # Convert the paths to projects, and filter them to the matched groups. + projects = [manifest.paths[p] for p in paths] + projects = [p for p in projects if p.MatchesGroups(groups)] + + if gitc_manifest is not None: + for path, proj in manifest.paths.items(): + if not proj.MatchesGroups(groups): + continue + + if not proj.upstream and not git_config.IsId(proj.revisionExpr): + proj.upstream = proj.revisionExpr + + if path not in gitc_manifest.paths: + # Any new projects need their first revision, even if we weren't + # asked for them. + projects.append(proj) + elif path not in paths: + # And copy revisions from the previous manifest if we're not + # updating them now. + gitc_proj = gitc_manifest.paths[path] + if gitc_proj.old_revision: + proj.revisionExpr = None + proj.old_revision = gitc_proj.old_revision + else: + proj.revisionExpr = gitc_proj.revisionExpr + + _set_project_revisions(projects) + + if gitc_manifest is not None: + for path, proj in gitc_manifest.paths.items(): + if proj.old_revision and path in paths: + # If we updated a project that has been started, keep the + # old-revision updated. + repo_proj = manifest.paths[path] + repo_proj.old_revision = repo_proj.revisionExpr + repo_proj.revisionExpr = None + + # Convert URLs from relative to absolute. + for _name, remote in manifest.remotes.items(): + remote.fetchUrl = remote.resolvedFetchUrl + + # Save the manifest. + save_manifest(manifest) def save_manifest(manifest, client_dir=None): - """Save the manifest file in the client_dir. - - Args: - manifest: Manifest object to save. - client_dir: Client directory to save the manifest in. - """ - if not client_dir: - manifest_file = manifest.manifestFile - else: - manifest_file = os.path.join(client_dir, '.manifest') - with open(manifest_file, 'w') as f: - manifest.Save(f, groups=manifest.GetGroupsStr()) - # TODO(sbasi/jorg): Come up with a solution to remove the sleep below. - # Give the GITC filesystem time to register the manifest changes. - time.sleep(3) + """Save the manifest file in the client_dir. + + Args: + manifest: Manifest object to save. + client_dir: Client directory to save the manifest in. + """ + if not client_dir: + manifest_file = manifest.manifestFile + else: + manifest_file = os.path.join(client_dir, ".manifest") + with open(manifest_file, "w") as f: + manifest.Save(f, groups=manifest.GetGroupsStr()) + # TODO(sbasi/jorg): Come up with a solution to remove the sleep below. + # Give the GITC filesystem time to register the manifest changes. + time.sleep(3) -- cgit v1.2.3-54-g00ecf