diff options
author | LaMont Jones <lamontjones@google.com> | 2021-11-18 22:40:18 +0000 |
---|---|---|
committer | LaMont Jones <lamontjones@google.com> | 2022-02-17 21:57:55 +0000 |
commit | cc879a97c3e2614d19b15b4661c3cab4d33139c9 (patch) | |
tree | 69d225e9f0e9d79fec8f423d9c40c275f0bf3b8c /manifest_xml.py | |
parent | 87cce68b28c34fa86895baa8d7f48307382e6c75 (diff) | |
download | git-repo-cc879a97c3e2614d19b15b4661c3cab4d33139c9.tar.gz |
Add multi-manifest support with <submanifest> elementv2.22
To be addressed in another change:
- a partial `repo sync` (with a list of projects/paths to sync)
requires `--this-tree-only`.
Change-Id: I6c7400bf001540e9d7694fa70934f8f204cb5f57
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/322657
Tested-by: LaMont Jones <lamontjones@google.com>
Reviewed-by: Mike Frysinger <vapier@google.com>
Diffstat (limited to 'manifest_xml.py')
-rw-r--r-- | manifest_xml.py | 454 |
1 files changed, 427 insertions, 27 deletions
diff --git a/manifest_xml.py b/manifest_xml.py index 7c5906da..7a4eb1e8 100644 --- a/manifest_xml.py +++ b/manifest_xml.py | |||
@@ -33,6 +33,9 @@ from wrapper import Wrapper | |||
33 | MANIFEST_FILE_NAME = 'manifest.xml' | 33 | MANIFEST_FILE_NAME = 'manifest.xml' |
34 | LOCAL_MANIFEST_NAME = 'local_manifest.xml' | 34 | LOCAL_MANIFEST_NAME = 'local_manifest.xml' |
35 | LOCAL_MANIFESTS_DIR_NAME = 'local_manifests' | 35 | LOCAL_MANIFESTS_DIR_NAME = 'local_manifests' |
36 | SUBMANIFEST_DIR = 'submanifests' | ||
37 | # Limit submanifests to an arbitrary depth for loop detection. | ||
38 | MAX_SUBMANIFEST_DEPTH = 8 | ||
36 | 39 | ||
37 | # Add all projects from local manifest into a group. | 40 | # Add all projects from local manifest into a group. |
38 | LOCAL_MANIFEST_GROUP_PREFIX = 'local:' | 41 | LOCAL_MANIFEST_GROUP_PREFIX = 'local:' |
@@ -197,10 +200,122 @@ class _XmlRemote(object): | |||
197 | self.annotations.append(Annotation(name, value, keep)) | 200 | self.annotations.append(Annotation(name, value, keep)) |
198 | 201 | ||
199 | 202 | ||
203 | class _XmlSubmanifest: | ||
204 | """Manage the <submanifest> element specified in the manifest. | ||
205 | |||
206 | Attributes: | ||
207 | name: a string, the name for this submanifest. | ||
208 | remote: a string, the remote.name for this submanifest. | ||
209 | project: a string, the name of the manifest project. | ||
210 | revision: a string, the commitish. | ||
211 | manifestName: a string, the submanifest file name. | ||
212 | groups: a list of strings, the groups to add to all projects in the submanifest. | ||
213 | path: a string, the relative path for the submanifest checkout. | ||
214 | annotations: (derived) a list of annotations. | ||
215 | present: (derived) a boolean, whether the submanifest's manifest file is present. | ||
216 | """ | ||
217 | def __init__(self, | ||
218 | name, | ||
219 | remote=None, | ||
220 | project=None, | ||
221 | revision=None, | ||
222 | manifestName=None, | ||
223 | groups=None, | ||
224 | path=None, | ||
225 | parent=None): | ||
226 | self.name = name | ||
227 | self.remote = remote | ||
228 | self.project = project | ||
229 | self.revision = revision | ||
230 | self.manifestName = manifestName | ||
231 | self.groups = groups | ||
232 | self.path = path | ||
233 | self.annotations = [] | ||
234 | outer_client = parent._outer_client or parent | ||
235 | if self.remote and not self.project: | ||
236 | raise ManifestParseError( | ||
237 | f'Submanifest {name}: must specify project when remote is given.') | ||
238 | rc = self.repo_client = RepoClient( | ||
239 | parent.repodir, manifestName, parent_groups=','.join(groups) or '', | ||
240 | submanifest_path=self.relpath, outer_client=outer_client) | ||
241 | |||
242 | self.present = os.path.exists(os.path.join(self.repo_client.subdir, | ||
243 | MANIFEST_FILE_NAME)) | ||
244 | |||
245 | def __eq__(self, other): | ||
246 | if not isinstance(other, _XmlSubmanifest): | ||
247 | return False | ||
248 | return ( | ||
249 | self.name == other.name and | ||
250 | self.remote == other.remote and | ||
251 | self.project == other.project and | ||
252 | self.revision == other.revision and | ||
253 | self.manifestName == other.manifestName and | ||
254 | self.groups == other.groups and | ||
255 | self.path == other.path and | ||
256 | sorted(self.annotations) == sorted(other.annotations)) | ||
257 | |||
258 | def __ne__(self, other): | ||
259 | return not self.__eq__(other) | ||
260 | |||
261 | def ToSubmanifestSpec(self, root): | ||
262 | """Return a SubmanifestSpec object, populating attributes""" | ||
263 | mp = root.manifestProject | ||
264 | remote = root.remotes[self.remote or root.default.remote.name] | ||
265 | # If a project was given, generate the url from the remote and project. | ||
266 | # If not, use this manifestProject's url. | ||
267 | if self.project: | ||
268 | manifestUrl = remote.ToRemoteSpec(self.project).url | ||
269 | else: | ||
270 | manifestUrl = mp.GetRemote(mp.remote.name).url | ||
271 | manifestName = self.manifestName or 'default.xml' | ||
272 | revision = self.revision or self.name | ||
273 | path = self.path or revision.split('/')[-1] | ||
274 | groups = self.groups or [] | ||
275 | |||
276 | return SubmanifestSpec(self.name, manifestUrl, manifestName, revision, path, | ||
277 | groups) | ||
278 | |||
279 | @property | ||
280 | def relpath(self): | ||
281 | """The path of this submanifest relative to the parent manifest.""" | ||
282 | revision = self.revision or self.name | ||
283 | return self.path or revision.split('/')[-1] | ||
284 | |||
285 | def GetGroupsStr(self): | ||
286 | """Returns the `groups` given for this submanifest.""" | ||
287 | if self.groups: | ||
288 | return ','.join(self.groups) | ||
289 | return '' | ||
290 | |||
291 | def AddAnnotation(self, name, value, keep): | ||
292 | """Add annotations to the submanifest.""" | ||
293 | self.annotations.append(Annotation(name, value, keep)) | ||
294 | |||
295 | |||
296 | class SubmanifestSpec: | ||
297 | """The submanifest element, with all fields expanded.""" | ||
298 | |||
299 | def __init__(self, | ||
300 | name, | ||
301 | manifestUrl, | ||
302 | manifestName, | ||
303 | revision, | ||
304 | path, | ||
305 | groups): | ||
306 | self.name = name | ||
307 | self.manifestUrl = manifestUrl | ||
308 | self.manifestName = manifestName | ||
309 | self.revision = revision | ||
310 | self.path = path | ||
311 | self.groups = groups or [] | ||
312 | |||
313 | |||
200 | class XmlManifest(object): | 314 | class XmlManifest(object): |
201 | """manages the repo configuration file""" | 315 | """manages the repo configuration file""" |
202 | 316 | ||
203 | def __init__(self, repodir, manifest_file, local_manifests=None): | 317 | def __init__(self, repodir, manifest_file, local_manifests=None, |
318 | outer_client=None, parent_groups='', submanifest_path=''): | ||
204 | """Initialize. | 319 | """Initialize. |
205 | 320 | ||
206 | Args: | 321 | Args: |
@@ -210,23 +325,37 @@ class XmlManifest(object): | |||
210 | be |repodir|/|MANIFEST_FILE_NAME|. | 325 | be |repodir|/|MANIFEST_FILE_NAME|. |
211 | local_manifests: Full path to the directory of local override manifests. | 326 | local_manifests: Full path to the directory of local override manifests. |
212 | This will usually be |repodir|/|LOCAL_MANIFESTS_DIR_NAME|. | 327 | This will usually be |repodir|/|LOCAL_MANIFESTS_DIR_NAME|. |
328 | outer_client: RepoClient of the outertree. | ||
329 | parent_groups: a string, the groups to apply to this projects. | ||
330 | submanifest_path: The submanifest root relative to the repo root. | ||
213 | """ | 331 | """ |
214 | # TODO(vapier): Move this out of this class. | 332 | # TODO(vapier): Move this out of this class. |
215 | self.globalConfig = GitConfig.ForUser() | 333 | self.globalConfig = GitConfig.ForUser() |
216 | 334 | ||
217 | self.repodir = os.path.abspath(repodir) | 335 | self.repodir = os.path.abspath(repodir) |
218 | self.topdir = os.path.dirname(self.repodir) | 336 | self._CheckLocalPath(submanifest_path) |
337 | self.topdir = os.path.join(os.path.dirname(self.repodir), submanifest_path) | ||
219 | self.manifestFile = manifest_file | 338 | self.manifestFile = manifest_file |
220 | self.local_manifests = local_manifests | 339 | self.local_manifests = local_manifests |
221 | self._load_local_manifests = True | 340 | self._load_local_manifests = True |
341 | self.parent_groups = parent_groups | ||
342 | |||
343 | if outer_client and self.isGitcClient: | ||
344 | raise ManifestParseError('Multi-manifest is incompatible with `gitc-init`') | ||
345 | |||
346 | if submanifest_path and not outer_client: | ||
347 | # If passing a submanifest_path, there must be an outer_client. | ||
348 | raise ManifestParseError(f'Bad call to {self.__class__.__name__}') | ||
349 | |||
350 | # If self._outer_client is None, this is not a checkout that supports | ||
351 | # multi-tree. | ||
352 | self._outer_client = outer_client or self | ||
222 | 353 | ||
223 | self.repoProject = MetaProject(self, 'repo', | 354 | self.repoProject = MetaProject(self, 'repo', |
224 | gitdir=os.path.join(repodir, 'repo/.git'), | 355 | gitdir=os.path.join(repodir, 'repo/.git'), |
225 | worktree=os.path.join(repodir, 'repo')) | 356 | worktree=os.path.join(repodir, 'repo')) |
226 | 357 | ||
227 | mp = MetaProject(self, 'manifests', | 358 | mp = self.SubmanifestProject(self.path_prefix) |
228 | gitdir=os.path.join(repodir, 'manifests.git'), | ||
229 | worktree=os.path.join(repodir, 'manifests')) | ||
230 | self.manifestProject = mp | 359 | self.manifestProject = mp |
231 | 360 | ||
232 | # This is a bit hacky, but we're in a chicken & egg situation: all the | 361 | # This is a bit hacky, but we're in a chicken & egg situation: all the |
@@ -311,6 +440,31 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
311 | ae.setAttribute('value', a.value) | 440 | ae.setAttribute('value', a.value) |
312 | e.appendChild(ae) | 441 | e.appendChild(ae) |
313 | 442 | ||
443 | def _SubmanifestToXml(self, r, doc, root): | ||
444 | """Generate XML <submanifest/> node.""" | ||
445 | e = doc.createElement('submanifest') | ||
446 | root.appendChild(e) | ||
447 | e.setAttribute('name', r.name) | ||
448 | if r.remote is not None: | ||
449 | e.setAttribute('remote', r.remote) | ||
450 | if r.project is not None: | ||
451 | e.setAttribute('project', r.project) | ||
452 | if r.manifestName is not None: | ||
453 | e.setAttribute('manifest-name', r.manifestName) | ||
454 | if r.revision is not None: | ||
455 | e.setAttribute('revision', r.revision) | ||
456 | if r.path is not None: | ||
457 | e.setAttribute('path', r.path) | ||
458 | if r.groups: | ||
459 | e.setAttribute('groups', r.GetGroupsStr()) | ||
460 | |||
461 | for a in r.annotations: | ||
462 | if a.keep == 'true': | ||
463 | ae = doc.createElement('annotation') | ||
464 | ae.setAttribute('name', a.name) | ||
465 | ae.setAttribute('value', a.value) | ||
466 | e.appendChild(ae) | ||
467 | |||
314 | def _ParseList(self, field): | 468 | def _ParseList(self, field): |
315 | """Parse fields that contain flattened lists. | 469 | """Parse fields that contain flattened lists. |
316 | 470 | ||
@@ -329,6 +483,8 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
329 | 483 | ||
330 | doc = xml.dom.minidom.Document() | 484 | doc = xml.dom.minidom.Document() |
331 | root = doc.createElement('manifest') | 485 | root = doc.createElement('manifest') |
486 | if self.is_submanifest: | ||
487 | root.setAttribute('path', self.path_prefix) | ||
332 | doc.appendChild(root) | 488 | doc.appendChild(root) |
333 | 489 | ||
334 | # Save out the notice. There's a little bit of work here to give it the | 490 | # Save out the notice. There's a little bit of work here to give it the |
@@ -383,6 +539,11 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
383 | root.appendChild(e) | 539 | root.appendChild(e) |
384 | root.appendChild(doc.createTextNode('')) | 540 | root.appendChild(doc.createTextNode('')) |
385 | 541 | ||
542 | for r in sorted(self.submanifests): | ||
543 | self._SubmanifestToXml(self.submanifests[r], doc, root) | ||
544 | if self.submanifests: | ||
545 | root.appendChild(doc.createTextNode('')) | ||
546 | |||
386 | def output_projects(parent, parent_node, projects): | 547 | def output_projects(parent, parent_node, projects): |
387 | for project_name in projects: | 548 | for project_name in projects: |
388 | for project in self._projects[project_name]: | 549 | for project in self._projects[project_name]: |
@@ -537,6 +698,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
537 | 'project', | 698 | 'project', |
538 | 'extend-project', | 699 | 'extend-project', |
539 | 'include', | 700 | 'include', |
701 | 'submanifest', | ||
540 | # These are children of 'project' nodes. | 702 | # These are children of 'project' nodes. |
541 | 'annotation', | 703 | 'annotation', |
542 | 'project', | 704 | 'project', |
@@ -575,12 +737,74 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
575 | """Manifests can modify e if they support extra project attributes.""" | 737 | """Manifests can modify e if they support extra project attributes.""" |
576 | 738 | ||
577 | @property | 739 | @property |
740 | def is_multimanifest(self): | ||
741 | """Whether this is a multimanifest checkout""" | ||
742 | return bool(self.outer_client.submanifests) | ||
743 | |||
744 | @property | ||
745 | def is_submanifest(self): | ||
746 | """Whether this manifest is a submanifest""" | ||
747 | return self._outer_client and self._outer_client != self | ||
748 | |||
749 | @property | ||
750 | def outer_client(self): | ||
751 | """The instance of the outermost manifest client""" | ||
752 | self._Load() | ||
753 | return self._outer_client | ||
754 | |||
755 | @property | ||
756 | def all_manifests(self): | ||
757 | """Generator yielding all (sub)manifests.""" | ||
758 | self._Load() | ||
759 | outer = self._outer_client | ||
760 | yield outer | ||
761 | for tree in outer.all_children: | ||
762 | yield tree | ||
763 | |||
764 | @property | ||
765 | def all_children(self): | ||
766 | """Generator yielding all child submanifests.""" | ||
767 | self._Load() | ||
768 | for child in self._submanifests.values(): | ||
769 | if child.repo_client: | ||
770 | yield child.repo_client | ||
771 | for tree in child.repo_client.all_children: | ||
772 | yield tree | ||
773 | |||
774 | @property | ||
775 | def path_prefix(self): | ||
776 | """The path of this submanifest, relative to the outermost manifest.""" | ||
777 | if not self._outer_client or self == self._outer_client: | ||
778 | return '' | ||
779 | return os.path.relpath(self.topdir, self._outer_client.topdir) | ||
780 | |||
781 | @property | ||
782 | def all_paths(self): | ||
783 | """All project paths for all (sub)manifests. See `paths`.""" | ||
784 | ret = {} | ||
785 | for tree in self.all_manifests: | ||
786 | prefix = tree.path_prefix | ||
787 | ret.update({os.path.join(prefix, k): v for k, v in tree.paths.items()}) | ||
788 | return ret | ||
789 | |||
790 | @property | ||
791 | def all_projects(self): | ||
792 | """All projects for all (sub)manifests. See `projects`.""" | ||
793 | return list(itertools.chain.from_iterable(x._paths.values() for x in self.all_manifests)) | ||
794 | |||
795 | @property | ||
578 | def paths(self): | 796 | def paths(self): |
797 | """Return all paths for this manifest. | ||
798 | |||
799 | Return: | ||
800 | A dictionary of {path: Project()}. `path` is relative to this manifest. | ||
801 | """ | ||
579 | self._Load() | 802 | self._Load() |
580 | return self._paths | 803 | return self._paths |
581 | 804 | ||
582 | @property | 805 | @property |
583 | def projects(self): | 806 | def projects(self): |
807 | """Return a list of all Projects in this manifest.""" | ||
584 | self._Load() | 808 | self._Load() |
585 | return list(self._paths.values()) | 809 | return list(self._paths.values()) |
586 | 810 | ||
@@ -595,6 +819,12 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
595 | return self._default | 819 | return self._default |
596 | 820 | ||
597 | @property | 821 | @property |
822 | def submanifests(self): | ||
823 | """All submanifests in this manifest.""" | ||
824 | self._Load() | ||
825 | return self._submanifests | ||
826 | |||
827 | @property | ||
598 | def repo_hooks_project(self): | 828 | def repo_hooks_project(self): |
599 | self._Load() | 829 | self._Load() |
600 | return self._repo_hooks_project | 830 | return self._repo_hooks_project |
@@ -651,8 +881,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
651 | return self._load_local_manifests and self.local_manifests | 881 | return self._load_local_manifests and self.local_manifests |
652 | 882 | ||
653 | def IsFromLocalManifest(self, project): | 883 | def IsFromLocalManifest(self, project): |
654 | """Is the project from a local manifest? | 884 | """Is the project from a local manifest?""" |
655 | """ | ||
656 | return any(x.startswith(LOCAL_MANIFEST_GROUP_PREFIX) | 885 | return any(x.startswith(LOCAL_MANIFEST_GROUP_PREFIX) |
657 | for x in project.groups) | 886 | for x in project.groups) |
658 | 887 | ||
@@ -676,6 +905,50 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
676 | def EnableGitLfs(self): | 905 | def EnableGitLfs(self): |
677 | return self.manifestProject.config.GetBoolean('repo.git-lfs') | 906 | return self.manifestProject.config.GetBoolean('repo.git-lfs') |
678 | 907 | ||
908 | def FindManifestByPath(self, path): | ||
909 | """Returns the manifest containing path.""" | ||
910 | path = os.path.abspath(path) | ||
911 | manifest = self._outer_client or self | ||
912 | old = None | ||
913 | while manifest._submanifests and manifest != old: | ||
914 | old = manifest | ||
915 | for name in manifest._submanifests: | ||
916 | tree = manifest._submanifests[name] | ||
917 | if path.startswith(tree.repo_client.manifest.topdir): | ||
918 | manifest = tree.repo_client | ||
919 | break | ||
920 | return manifest | ||
921 | |||
922 | @property | ||
923 | def subdir(self): | ||
924 | """Returns the path for per-submanifest objects for this manifest.""" | ||
925 | return self.SubmanifestInfoDir(self.path_prefix) | ||
926 | |||
927 | def SubmanifestInfoDir(self, submanifest_path, object_path=''): | ||
928 | """Return the path to submanifest-specific info for a submanifest. | ||
929 | |||
930 | Return the full path of the directory in which to put per-manifest objects. | ||
931 | |||
932 | Args: | ||
933 | submanifest_path: a string, the path of the submanifest, relative to the | ||
934 | outermost topdir. If empty, then repodir is returned. | ||
935 | object_path: a string, relative path to append to the submanifest info | ||
936 | directory path. | ||
937 | """ | ||
938 | if submanifest_path: | ||
939 | return os.path.join(self.repodir, SUBMANIFEST_DIR, submanifest_path, | ||
940 | object_path) | ||
941 | else: | ||
942 | return os.path.join(self.repodir, object_path) | ||
943 | |||
944 | def SubmanifestProject(self, submanifest_path): | ||
945 | """Return a manifestProject for a submanifest.""" | ||
946 | subdir = self.SubmanifestInfoDir(submanifest_path) | ||
947 | mp = MetaProject(self, 'manifests', | ||
948 | gitdir=os.path.join(subdir, 'manifests.git'), | ||
949 | worktree=os.path.join(subdir, 'manifests')) | ||
950 | return mp | ||
951 | |||
679 | def GetDefaultGroupsStr(self): | 952 | def GetDefaultGroupsStr(self): |
680 | """Returns the default group string for the platform.""" | 953 | """Returns the default group string for the platform.""" |
681 | return 'default,platform-' + platform.system().lower() | 954 | return 'default,platform-' + platform.system().lower() |
@@ -693,6 +966,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
693 | self._paths = {} | 966 | self._paths = {} |
694 | self._remotes = {} | 967 | self._remotes = {} |
695 | self._default = None | 968 | self._default = None |
969 | self._submanifests = {} | ||
696 | self._repo_hooks_project = None | 970 | self._repo_hooks_project = None |
697 | self._superproject = {} | 971 | self._superproject = {} |
698 | self._contactinfo = ContactInfo(Wrapper().BUG_URL) | 972 | self._contactinfo = ContactInfo(Wrapper().BUG_URL) |
@@ -700,20 +974,29 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
700 | self.branch = None | 974 | self.branch = None |
701 | self._manifest_server = None | 975 | self._manifest_server = None |
702 | 976 | ||
703 | def _Load(self): | 977 | def _Load(self, initial_client=None, submanifest_depth=0): |
978 | if submanifest_depth > MAX_SUBMANIFEST_DEPTH: | ||
979 | raise ManifestParseError('maximum submanifest depth %d exceeded.' % | ||
980 | MAX_SUBMANIFEST_DEPTH) | ||
704 | if not self._loaded: | 981 | if not self._loaded: |
982 | if self._outer_client and self._outer_client != self: | ||
983 | # This will load all clients. | ||
984 | self._outer_client._Load(initial_client=self) | ||
985 | |||
705 | m = self.manifestProject | 986 | m = self.manifestProject |
706 | b = m.GetBranch(m.CurrentBranch).merge | 987 | b = m.GetBranch(m.CurrentBranch).merge |
707 | if b is not None and b.startswith(R_HEADS): | 988 | if b is not None and b.startswith(R_HEADS): |
708 | b = b[len(R_HEADS):] | 989 | b = b[len(R_HEADS):] |
709 | self.branch = b | 990 | self.branch = b |
710 | 991 | ||
992 | parent_groups = self.parent_groups | ||
993 | |||
711 | # The manifestFile was specified by the user which is why we allow include | 994 | # The manifestFile was specified by the user which is why we allow include |
712 | # paths to point anywhere. | 995 | # paths to point anywhere. |
713 | nodes = [] | 996 | nodes = [] |
714 | nodes.append(self._ParseManifestXml( | 997 | nodes.append(self._ParseManifestXml( |
715 | self.manifestFile, self.manifestProject.worktree, | 998 | self.manifestFile, self.manifestProject.worktree, |
716 | restrict_includes=False)) | 999 | parent_groups=parent_groups, restrict_includes=False)) |
717 | 1000 | ||
718 | if self._load_local_manifests and self.local_manifests: | 1001 | if self._load_local_manifests and self.local_manifests: |
719 | try: | 1002 | try: |
@@ -722,9 +1005,10 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
722 | local = os.path.join(self.local_manifests, local_file) | 1005 | local = os.path.join(self.local_manifests, local_file) |
723 | # Since local manifests are entirely managed by the user, allow | 1006 | # Since local manifests are entirely managed by the user, allow |
724 | # them to point anywhere the user wants. | 1007 | # them to point anywhere the user wants. |
1008 | local_group = f'{LOCAL_MANIFEST_GROUP_PREFIX}:{local_file[:-4]}' | ||
725 | nodes.append(self._ParseManifestXml( | 1009 | nodes.append(self._ParseManifestXml( |
726 | local, self.repodir, | 1010 | local, self.subdir, |
727 | parent_groups=f'{LOCAL_MANIFEST_GROUP_PREFIX}:{local_file[:-4]}', | 1011 | parent_groups=f'{local_group},{parent_groups}', |
728 | restrict_includes=False)) | 1012 | restrict_includes=False)) |
729 | except OSError: | 1013 | except OSError: |
730 | pass | 1014 | pass |
@@ -743,6 +1027,23 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
743 | 1027 | ||
744 | self._loaded = True | 1028 | self._loaded = True |
745 | 1029 | ||
1030 | # Now that we have loaded this manifest, load any submanifest manifests | ||
1031 | # as well. We need to do this after self._loaded is set to avoid looping. | ||
1032 | if self._outer_client: | ||
1033 | for name in self._submanifests: | ||
1034 | tree = self._submanifests[name] | ||
1035 | spec = tree.ToSubmanifestSpec(self) | ||
1036 | present = os.path.exists(os.path.join(self.subdir, MANIFEST_FILE_NAME)) | ||
1037 | if present and tree.present and not tree.repo_client: | ||
1038 | if initial_client and initial_client.topdir == self.topdir: | ||
1039 | tree.repo_client = self | ||
1040 | tree.present = present | ||
1041 | elif not os.path.exists(self.subdir): | ||
1042 | tree.present = False | ||
1043 | if tree.present: | ||
1044 | tree.repo_client._Load(initial_client=initial_client, | ||
1045 | submanifest_depth=submanifest_depth + 1) | ||
1046 | |||
746 | def _ParseManifestXml(self, path, include_root, parent_groups='', | 1047 | def _ParseManifestXml(self, path, include_root, parent_groups='', |
747 | restrict_includes=True): | 1048 | restrict_includes=True): |
748 | """Parse a manifest XML and return the computed nodes. | 1049 | """Parse a manifest XML and return the computed nodes. |
@@ -832,6 +1133,20 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
832 | if self._default is None: | 1133 | if self._default is None: |
833 | self._default = _Default() | 1134 | self._default = _Default() |
834 | 1135 | ||
1136 | submanifest_paths = set() | ||
1137 | for node in itertools.chain(*node_list): | ||
1138 | if node.nodeName == 'submanifest': | ||
1139 | submanifest = self._ParseSubmanifest(node) | ||
1140 | if submanifest: | ||
1141 | if submanifest.name in self._submanifests: | ||
1142 | if submanifest != self._submanifests[submanifest.name]: | ||
1143 | raise ManifestParseError( | ||
1144 | 'submanifest %s already exists with different attributes' % | ||
1145 | (submanifest.name)) | ||
1146 | else: | ||
1147 | self._submanifests[submanifest.name] = submanifest | ||
1148 | submanifest_paths.add(submanifest.relpath) | ||
1149 | |||
835 | for node in itertools.chain(*node_list): | 1150 | for node in itertools.chain(*node_list): |
836 | if node.nodeName == 'notice': | 1151 | if node.nodeName == 'notice': |
837 | if self._notice is not None: | 1152 | if self._notice is not None: |
@@ -859,6 +1174,11 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
859 | raise ManifestParseError( | 1174 | raise ManifestParseError( |
860 | 'duplicate path %s in %s' % | 1175 | 'duplicate path %s in %s' % |
861 | (project.relpath, self.manifestFile)) | 1176 | (project.relpath, self.manifestFile)) |
1177 | for tree in submanifest_paths: | ||
1178 | if project.relpath.startswith(tree): | ||
1179 | raise ManifestParseError( | ||
1180 | 'project %s conflicts with submanifest path %s' % | ||
1181 | (project.relpath, tree)) | ||
862 | self._paths[project.relpath] = project | 1182 | self._paths[project.relpath] = project |
863 | projects.append(project) | 1183 | projects.append(project) |
864 | for subproject in project.subprojects: | 1184 | for subproject in project.subprojects: |
@@ -883,8 +1203,10 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
883 | if groups: | 1203 | if groups: |
884 | groups = self._ParseList(groups) | 1204 | groups = self._ParseList(groups) |
885 | revision = node.getAttribute('revision') | 1205 | revision = node.getAttribute('revision') |
886 | remote = node.getAttribute('remote') | 1206 | remote_name = node.getAttribute('remote') |
887 | if remote: | 1207 | if not remote_name: |
1208 | remote = self._default.remote | ||
1209 | else: | ||
888 | remote = self._get_remote(node) | 1210 | remote = self._get_remote(node) |
889 | 1211 | ||
890 | named_projects = self._projects[name] | 1212 | named_projects = self._projects[name] |
@@ -899,12 +1221,13 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
899 | if revision: | 1221 | if revision: |
900 | p.SetRevision(revision) | 1222 | p.SetRevision(revision) |
901 | 1223 | ||
902 | if remote: | 1224 | if remote_name: |
903 | p.remote = remote.ToRemoteSpec(name) | 1225 | p.remote = remote.ToRemoteSpec(name) |
904 | 1226 | ||
905 | if dest_path: | 1227 | if dest_path: |
906 | del self._paths[p.relpath] | 1228 | del self._paths[p.relpath] |
907 | relpath, worktree, gitdir, objdir, _ = self.GetProjectPaths(name, dest_path) | 1229 | relpath, worktree, gitdir, objdir, _ = self.GetProjectPaths( |
1230 | name, dest_path, remote.name) | ||
908 | p.UpdatePaths(relpath, worktree, gitdir, objdir) | 1231 | p.UpdatePaths(relpath, worktree, gitdir, objdir) |
909 | self._paths[p.relpath] = p | 1232 | self._paths[p.relpath] = p |
910 | 1233 | ||
@@ -1109,6 +1432,53 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
1109 | 1432 | ||
1110 | return '\n'.join(cleanLines) | 1433 | return '\n'.join(cleanLines) |
1111 | 1434 | ||
1435 | def _ParseSubmanifest(self, node): | ||
1436 | """Reads a <submanifest> element from the manifest file.""" | ||
1437 | name = self._reqatt(node, 'name') | ||
1438 | remote = node.getAttribute('remote') | ||
1439 | if remote == '': | ||
1440 | remote = None | ||
1441 | project = node.getAttribute('project') | ||
1442 | if project == '': | ||
1443 | project = None | ||
1444 | revision = node.getAttribute('revision') | ||
1445 | if revision == '': | ||
1446 | revision = None | ||
1447 | manifestName = node.getAttribute('manifest-name') | ||
1448 | if manifestName == '': | ||
1449 | manifestName = None | ||
1450 | groups = '' | ||
1451 | if node.hasAttribute('groups'): | ||
1452 | groups = node.getAttribute('groups') | ||
1453 | groups = self._ParseList(groups) | ||
1454 | path = node.getAttribute('path') | ||
1455 | if path == '': | ||
1456 | path = None | ||
1457 | if revision: | ||
1458 | msg = self._CheckLocalPath(revision.split('/')[-1]) | ||
1459 | if msg: | ||
1460 | raise ManifestInvalidPathError( | ||
1461 | '<submanifest> invalid "revision": %s: %s' % (revision, msg)) | ||
1462 | else: | ||
1463 | msg = self._CheckLocalPath(name) | ||
1464 | if msg: | ||
1465 | raise ManifestInvalidPathError( | ||
1466 | '<submanifest> invalid "name": %s: %s' % (name, msg)) | ||
1467 | else: | ||
1468 | msg = self._CheckLocalPath(path) | ||
1469 | if msg: | ||
1470 | raise ManifestInvalidPathError( | ||
1471 | '<submanifest> invalid "path": %s: %s' % (path, msg)) | ||
1472 | |||
1473 | submanifest = _XmlSubmanifest(name, remote, project, revision, manifestName, | ||
1474 | groups, path, self) | ||
1475 | |||
1476 | for n in node.childNodes: | ||
1477 | if n.nodeName == 'annotation': | ||
1478 | self._ParseAnnotation(submanifest, n) | ||
1479 | |||
1480 | return submanifest | ||
1481 | |||
1112 | def _JoinName(self, parent_name, name): | 1482 | def _JoinName(self, parent_name, name): |
1113 | return os.path.join(parent_name, name) | 1483 | return os.path.join(parent_name, name) |
1114 | 1484 | ||
@@ -1172,7 +1542,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
1172 | 1542 | ||
1173 | if parent is None: | 1543 | if parent is None: |
1174 | relpath, worktree, gitdir, objdir, use_git_worktrees = \ | 1544 | relpath, worktree, gitdir, objdir, use_git_worktrees = \ |
1175 | self.GetProjectPaths(name, path) | 1545 | self.GetProjectPaths(name, path, remote.name) |
1176 | else: | 1546 | else: |
1177 | use_git_worktrees = False | 1547 | use_git_worktrees = False |
1178 | relpath, worktree, gitdir, objdir = \ | 1548 | relpath, worktree, gitdir, objdir = \ |
@@ -1218,31 +1588,54 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
1218 | 1588 | ||
1219 | return project | 1589 | return project |
1220 | 1590 | ||
1221 | def GetProjectPaths(self, name, path): | 1591 | def GetProjectPaths(self, name, path, remote): |
1592 | """Return the paths for a project. | ||
1593 | |||
1594 | Args: | ||
1595 | name: a string, the name of the project. | ||
1596 | path: a string, the path of the project. | ||
1597 | remote: a string, the remote.name of the project. | ||
1598 | """ | ||
1222 | # The manifest entries might have trailing slashes. Normalize them to avoid | 1599 | # The manifest entries might have trailing slashes. Normalize them to avoid |
1223 | # unexpected filesystem behavior since we do string concatenation below. | 1600 | # unexpected filesystem behavior since we do string concatenation below. |
1224 | path = path.rstrip('/') | 1601 | path = path.rstrip('/') |
1225 | name = name.rstrip('/') | 1602 | name = name.rstrip('/') |
1603 | remote = remote.rstrip('/') | ||
1226 | use_git_worktrees = False | 1604 | use_git_worktrees = False |
1605 | use_remote_name = bool(self._outer_client._submanifests) | ||
1227 | relpath = path | 1606 | relpath = path |
1228 | if self.IsMirror: | 1607 | if self.IsMirror: |
1229 | worktree = None | 1608 | worktree = None |
1230 | gitdir = os.path.join(self.topdir, '%s.git' % name) | 1609 | gitdir = os.path.join(self.topdir, '%s.git' % name) |
1231 | objdir = gitdir | 1610 | objdir = gitdir |
1232 | else: | 1611 | else: |
1612 | if use_remote_name: | ||
1613 | namepath = os.path.join(remote, f'{name}.git') | ||
1614 | else: | ||
1615 | namepath = f'{name}.git' | ||
1233 | worktree = os.path.join(self.topdir, path).replace('\\', '/') | 1616 | worktree = os.path.join(self.topdir, path).replace('\\', '/') |
1234 | gitdir = os.path.join(self.repodir, 'projects', '%s.git' % path) | 1617 | gitdir = os.path.join(self.subdir, 'projects', '%s.git' % path) |
1235 | # We allow people to mix git worktrees & non-git worktrees for now. | 1618 | # We allow people to mix git worktrees & non-git worktrees for now. |
1236 | # This allows for in situ migration of repo clients. | 1619 | # This allows for in situ migration of repo clients. |
1237 | if os.path.exists(gitdir) or not self.UseGitWorktrees: | 1620 | if os.path.exists(gitdir) or not self.UseGitWorktrees: |
1238 | objdir = os.path.join(self.repodir, 'project-objects', '%s.git' % name) | 1621 | objdir = os.path.join(self.subdir, 'project-objects', namepath) |
1239 | else: | 1622 | else: |
1240 | use_git_worktrees = True | 1623 | use_git_worktrees = True |
1241 | gitdir = os.path.join(self.repodir, 'worktrees', '%s.git' % name) | 1624 | gitdir = os.path.join(self.repodir, 'worktrees', namepath) |
1242 | objdir = gitdir | 1625 | objdir = gitdir |
1243 | return relpath, worktree, gitdir, objdir, use_git_worktrees | 1626 | return relpath, worktree, gitdir, objdir, use_git_worktrees |
1244 | 1627 | ||
1245 | def GetProjectsWithName(self, name): | 1628 | def GetProjectsWithName(self, name, all_manifests=False): |
1629 | """All projects with |name|. | ||
1630 | |||
1631 | Args: | ||
1632 | name: a string, the name of the project. | ||
1633 | all_manifests: a boolean, if True, then all manifests are searched. If | ||
1634 | False, then only this manifest is searched. | ||
1635 | """ | ||
1636 | if all_manifests: | ||
1637 | return list(itertools.chain.from_iterable( | ||
1638 | x._projects.get(name, []) for x in self.all_manifests)) | ||
1246 | return self._projects.get(name, []) | 1639 | return self._projects.get(name, []) |
1247 | 1640 | ||
1248 | def GetSubprojectName(self, parent, submodule_path): | 1641 | def GetSubprojectName(self, parent, submodule_path): |
@@ -1498,19 +1891,26 @@ class GitcManifest(XmlManifest): | |||
1498 | class RepoClient(XmlManifest): | 1891 | class RepoClient(XmlManifest): |
1499 | """Manages a repo client checkout.""" | 1892 | """Manages a repo client checkout.""" |
1500 | 1893 | ||
1501 | def __init__(self, repodir, manifest_file=None): | 1894 | def __init__(self, repodir, manifest_file=None, submanifest_path='', **kwargs): |
1502 | self.isGitcClient = False | 1895 | self.isGitcClient = False |
1896 | submanifest_path = submanifest_path or '' | ||
1897 | if submanifest_path: | ||
1898 | self._CheckLocalPath(submanifest_path) | ||
1899 | prefix = os.path.join(repodir, SUBMANIFEST_DIR, submanifest_path) | ||
1900 | else: | ||
1901 | prefix = repodir | ||
1503 | 1902 | ||
1504 | if os.path.exists(os.path.join(repodir, LOCAL_MANIFEST_NAME)): | 1903 | if os.path.exists(os.path.join(prefix, LOCAL_MANIFEST_NAME)): |
1505 | print('error: %s is not supported; put local manifests in `%s` instead' % | 1904 | print('error: %s is not supported; put local manifests in `%s` instead' % |
1506 | (LOCAL_MANIFEST_NAME, os.path.join(repodir, LOCAL_MANIFESTS_DIR_NAME)), | 1905 | (LOCAL_MANIFEST_NAME, os.path.join(prefix, LOCAL_MANIFESTS_DIR_NAME)), |
1507 | file=sys.stderr) | 1906 | file=sys.stderr) |
1508 | sys.exit(1) | 1907 | sys.exit(1) |
1509 | 1908 | ||
1510 | if manifest_file is None: | 1909 | if manifest_file is None: |
1511 | manifest_file = os.path.join(repodir, MANIFEST_FILE_NAME) | 1910 | manifest_file = os.path.join(prefix, MANIFEST_FILE_NAME) |
1512 | local_manifests = os.path.abspath(os.path.join(repodir, LOCAL_MANIFESTS_DIR_NAME)) | 1911 | local_manifests = os.path.abspath(os.path.join(prefix, LOCAL_MANIFESTS_DIR_NAME)) |
1513 | super().__init__(repodir, manifest_file, local_manifests) | 1912 | super().__init__(repodir, manifest_file, local_manifests, |
1913 | submanifest_path=submanifest_path, **kwargs) | ||
1514 | 1914 | ||
1515 | # TODO: Completely separate manifest logic out of the client. | 1915 | # TODO: Completely separate manifest logic out of the client. |
1516 | self.manifest = self | 1916 | self.manifest = self |