diff options
author | Gavin Mak <gavinmak@google.com> | 2023-03-11 06:46:20 +0000 |
---|---|---|
committer | LUCI <gerrit-scoped@luci-project-accounts.iam.gserviceaccount.com> | 2023-03-22 17:46:28 +0000 |
commit | ea2e330e43c182dc16b0111ebc69ee5a71ee4ce1 (patch) | |
tree | dc33ba0e56825b3e007d0589891756724725a465 /gitc_utils.py | |
parent | 1604cf255f8c1786a23388db6d5277ac7949a24a (diff) | |
download | git-repo-ea2e330e43c182dc16b0111ebc69ee5a71ee4ce1.tar.gz |
Format codebase with black and check formatting in CQ
Apply rules set by https://gerrit-review.googlesource.com/c/git-repo/+/362954/ across the codebase and fix any lingering errors caught
by flake8. Also check black formatting in run_tests (and CQ).
Bug: b/267675342
Change-Id: I972d77649dac351150dcfeb1cd1ad0ea2efc1956
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/363474
Reviewed-by: Mike Frysinger <vapier@google.com>
Tested-by: Gavin Mak <gavinmak@google.com>
Commit-Queue: Gavin Mak <gavinmak@google.com>
Diffstat (limited to 'gitc_utils.py')
-rw-r--r-- | gitc_utils.py | 231 |
1 files changed, 121 insertions, 110 deletions
diff --git a/gitc_utils.py b/gitc_utils.py index dfcfd2a4..7b72048f 100644 --- a/gitc_utils.py +++ b/gitc_utils.py | |||
@@ -28,128 +28,139 @@ NUM_BATCH_RETRIEVE_REVISIONID = 32 | |||
28 | 28 | ||
29 | 29 | ||
30 | def get_gitc_manifest_dir(): | 30 | def get_gitc_manifest_dir(): |
31 | return wrapper.Wrapper().get_gitc_manifest_dir() | 31 | return wrapper.Wrapper().get_gitc_manifest_dir() |
32 | 32 | ||
33 | 33 | ||
34 | def parse_clientdir(gitc_fs_path): | 34 | def parse_clientdir(gitc_fs_path): |
35 | return wrapper.Wrapper().gitc_parse_clientdir(gitc_fs_path) | 35 | return wrapper.Wrapper().gitc_parse_clientdir(gitc_fs_path) |
36 | 36 | ||
37 | 37 | ||
38 | def _get_project_revision(args): | 38 | def _get_project_revision(args): |
39 | """Worker for _set_project_revisions to lookup one project remote.""" | 39 | """Worker for _set_project_revisions to lookup one project remote.""" |
40 | (i, url, expr) = args | 40 | (i, url, expr) = args |
41 | gitcmd = git_command.GitCommand( | 41 | gitcmd = git_command.GitCommand( |
42 | None, ['ls-remote', url, expr], capture_stdout=True, cwd='/tmp') | 42 | None, ["ls-remote", url, expr], capture_stdout=True, cwd="/tmp" |
43 | rc = gitcmd.Wait() | 43 | ) |
44 | return (i, rc, gitcmd.stdout.split('\t', 1)[0]) | 44 | rc = gitcmd.Wait() |
45 | return (i, rc, gitcmd.stdout.split("\t", 1)[0]) | ||
45 | 46 | ||
46 | 47 | ||
47 | def _set_project_revisions(projects): | 48 | def _set_project_revisions(projects): |
48 | """Sets the revisionExpr for a list of projects. | 49 | """Sets the revisionExpr for a list of projects. |
49 | 50 | ||
50 | Because of the limit of open file descriptors allowed, length of projects | 51 | Because of the limit of open file descriptors allowed, length of projects |
51 | should not be overly large. Recommend calling this function multiple times | 52 | should not be overly large. Recommend calling this function multiple times |
52 | with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects. | 53 | with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects. |
53 | 54 | ||
54 | Args: | 55 | Args: |
55 | projects: List of project objects to set the revionExpr for. | 56 | projects: List of project objects to set the revionExpr for. |
56 | """ | 57 | """ |
57 | # Retrieve the commit id for each project based off of it's current | 58 | # Retrieve the commit id for each project based off of its current |
58 | # revisionExpr and it is not already a commit id. | 59 | # revisionExpr and it is not already a commit id. |
59 | with multiprocessing.Pool(NUM_BATCH_RETRIEVE_REVISIONID) as pool: | 60 | with multiprocessing.Pool(NUM_BATCH_RETRIEVE_REVISIONID) as pool: |
60 | results_iter = pool.imap_unordered( | 61 | results_iter = pool.imap_unordered( |
61 | _get_project_revision, | 62 | _get_project_revision, |
62 | ((i, project.remote.url, project.revisionExpr) | 63 | ( |
63 | for i, project in enumerate(projects) | 64 | (i, project.remote.url, project.revisionExpr) |
64 | if not git_config.IsId(project.revisionExpr)), | 65 | for i, project in enumerate(projects) |
65 | chunksize=8) | 66 | if not git_config.IsId(project.revisionExpr) |
66 | for (i, rc, revisionExpr) in results_iter: | 67 | ), |
67 | project = projects[i] | 68 | chunksize=8, |
68 | if rc: | 69 | ) |
69 | print('FATAL: Failed to retrieve revisionExpr for %s' % project.name) | 70 | for i, rc, revisionExpr in results_iter: |
70 | pool.terminate() | 71 | project = projects[i] |
71 | sys.exit(1) | 72 | if rc: |
72 | if not revisionExpr: | 73 | print( |
73 | pool.terminate() | 74 | "FATAL: Failed to retrieve revisionExpr for %s" |
74 | raise ManifestParseError('Invalid SHA-1 revision project %s (%s)' % | 75 | % project.name |
75 | (project.remote.url, project.revisionExpr)) | 76 | ) |
76 | project.revisionExpr = revisionExpr | 77 | pool.terminate() |
78 | sys.exit(1) | ||
79 | if not revisionExpr: | ||
80 | pool.terminate() | ||
81 | raise ManifestParseError( | ||
82 | "Invalid SHA-1 revision project %s (%s)" | ||
83 | % (project.remote.url, project.revisionExpr) | ||
84 | ) | ||
85 | project.revisionExpr = revisionExpr | ||
77 | 86 | ||
78 | 87 | ||
79 | def generate_gitc_manifest(gitc_manifest, manifest, paths=None): | 88 | def generate_gitc_manifest(gitc_manifest, manifest, paths=None): |
80 | """Generate a manifest for shafsd to use for this GITC client. | 89 | """Generate a manifest for shafsd to use for this GITC client. |
81 | 90 | ||
82 | Args: | 91 | Args: |
83 | gitc_manifest: Current gitc manifest, or None if there isn't one yet. | 92 | gitc_manifest: Current gitc manifest, or None if there isn't one yet. |
84 | manifest: A GitcManifest object loaded with the current repo manifest. | 93 | manifest: A GitcManifest object loaded with the current repo manifest. |
85 | paths: List of project paths we want to update. | 94 | paths: List of project paths we want to update. |
86 | """ | 95 | """ |
87 | 96 | ||
88 | print('Generating GITC Manifest by fetching revision SHAs for each ' | 97 | print( |
89 | 'project.') | 98 | "Generating GITC Manifest by fetching revision SHAs for each " |
90 | if paths is None: | 99 | "project." |
91 | paths = list(manifest.paths.keys()) | 100 | ) |
92 | 101 | if paths is None: | |
93 | groups = [x for x in re.split(r'[,\s]+', manifest.GetGroupsStr()) if x] | 102 | paths = list(manifest.paths.keys()) |
94 | 103 | ||
95 | # Convert the paths to projects, and filter them to the matched groups. | 104 | groups = [x for x in re.split(r"[,\s]+", manifest.GetGroupsStr()) if x] |
96 | projects = [manifest.paths[p] for p in paths] | 105 | |
97 | projects = [p for p in projects if p.MatchesGroups(groups)] | 106 | # Convert the paths to projects, and filter them to the matched groups. |
98 | 107 | projects = [manifest.paths[p] for p in paths] | |
99 | if gitc_manifest is not None: | 108 | projects = [p for p in projects if p.MatchesGroups(groups)] |
100 | for path, proj in manifest.paths.items(): | 109 | |
101 | if not proj.MatchesGroups(groups): | 110 | if gitc_manifest is not None: |
102 | continue | 111 | for path, proj in manifest.paths.items(): |
103 | 112 | if not proj.MatchesGroups(groups): | |
104 | if not proj.upstream and not git_config.IsId(proj.revisionExpr): | 113 | continue |
105 | proj.upstream = proj.revisionExpr | 114 | |
106 | 115 | if not proj.upstream and not git_config.IsId(proj.revisionExpr): | |
107 | if path not in gitc_manifest.paths: | 116 | proj.upstream = proj.revisionExpr |
108 | # Any new projects need their first revision, even if we weren't asked | 117 | |
109 | # for them. | 118 | if path not in gitc_manifest.paths: |
110 | projects.append(proj) | 119 | # Any new projects need their first revision, even if we weren't |
111 | elif path not in paths: | 120 | # asked for them. |
112 | # And copy revisions from the previous manifest if we're not updating | 121 | projects.append(proj) |
113 | # them now. | 122 | elif path not in paths: |
114 | gitc_proj = gitc_manifest.paths[path] | 123 | # And copy revisions from the previous manifest if we're not |
115 | if gitc_proj.old_revision: | 124 | # updating them now. |
116 | proj.revisionExpr = None | 125 | gitc_proj = gitc_manifest.paths[path] |
117 | proj.old_revision = gitc_proj.old_revision | 126 | if gitc_proj.old_revision: |
118 | else: | 127 | proj.revisionExpr = None |
119 | proj.revisionExpr = gitc_proj.revisionExpr | 128 | proj.old_revision = gitc_proj.old_revision |
120 | 129 | else: | |
121 | _set_project_revisions(projects) | 130 | proj.revisionExpr = gitc_proj.revisionExpr |
122 | 131 | ||
123 | if gitc_manifest is not None: | 132 | _set_project_revisions(projects) |
124 | for path, proj in gitc_manifest.paths.items(): | 133 | |
125 | if proj.old_revision and path in paths: | 134 | if gitc_manifest is not None: |
126 | # If we updated a project that has been started, keep the old-revision | 135 | for path, proj in gitc_manifest.paths.items(): |
127 | # updated. | 136 | if proj.old_revision and path in paths: |
128 | repo_proj = manifest.paths[path] | 137 | # If we updated a project that has been started, keep the |
129 | repo_proj.old_revision = repo_proj.revisionExpr | 138 | # old-revision updated. |
130 | repo_proj.revisionExpr = None | 139 | repo_proj = manifest.paths[path] |
131 | 140 | repo_proj.old_revision = repo_proj.revisionExpr | |
132 | # Convert URLs from relative to absolute. | 141 | repo_proj.revisionExpr = None |
133 | for _name, remote in manifest.remotes.items(): | 142 | |
134 | remote.fetchUrl = remote.resolvedFetchUrl | 143 | # Convert URLs from relative to absolute. |
135 | 144 | for _name, remote in manifest.remotes.items(): | |
136 | # Save the manifest. | 145 | remote.fetchUrl = remote.resolvedFetchUrl |
137 | save_manifest(manifest) | 146 | |
147 | # Save the manifest. | ||
148 | save_manifest(manifest) | ||
138 | 149 | ||
139 | 150 | ||
140 | def save_manifest(manifest, client_dir=None): | 151 | def save_manifest(manifest, client_dir=None): |
141 | """Save the manifest file in the client_dir. | 152 | """Save the manifest file in the client_dir. |
142 | 153 | ||
143 | Args: | 154 | Args: |
144 | manifest: Manifest object to save. | 155 | manifest: Manifest object to save. |
145 | client_dir: Client directory to save the manifest in. | 156 | client_dir: Client directory to save the manifest in. |
146 | """ | 157 | """ |
147 | if not client_dir: | 158 | if not client_dir: |
148 | manifest_file = manifest.manifestFile | 159 | manifest_file = manifest.manifestFile |
149 | else: | 160 | else: |
150 | manifest_file = os.path.join(client_dir, '.manifest') | 161 | manifest_file = os.path.join(client_dir, ".manifest") |
151 | with open(manifest_file, 'w') as f: | 162 | with open(manifest_file, "w") as f: |
152 | manifest.Save(f, groups=manifest.GetGroupsStr()) | 163 | manifest.Save(f, groups=manifest.GetGroupsStr()) |
153 | # TODO(sbasi/jorg): Come up with a solution to remove the sleep below. | 164 | # TODO(sbasi/jorg): Come up with a solution to remove the sleep below. |
154 | # Give the GITC filesystem time to register the manifest changes. | 165 | # Give the GITC filesystem time to register the manifest changes. |
155 | time.sleep(3) | 166 | time.sleep(3) |