From cc879a97c3e2614d19b15b4661c3cab4d33139c9 Mon Sep 17 00:00:00 2001 From: LaMont Jones Date: Thu, 18 Nov 2021 22:40:18 +0000 Subject: Add multi-manifest support with element To be addressed in another change: - a partial `repo sync` (with a list of projects/paths to sync) requires `--this-tree-only`. Change-Id: I6c7400bf001540e9d7694fa70934f8f204cb5f57 Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/322657 Tested-by: LaMont Jones Reviewed-by: Mike Frysinger --- manifest_xml.py | 454 ++++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 427 insertions(+), 27 deletions(-) (limited to 'manifest_xml.py') diff --git a/manifest_xml.py b/manifest_xml.py index 7c5906da..7a4eb1e8 100644 --- a/manifest_xml.py +++ b/manifest_xml.py @@ -33,6 +33,9 @@ from wrapper import Wrapper MANIFEST_FILE_NAME = 'manifest.xml' LOCAL_MANIFEST_NAME = 'local_manifest.xml' LOCAL_MANIFESTS_DIR_NAME = 'local_manifests' +SUBMANIFEST_DIR = 'submanifests' +# Limit submanifests to an arbitrary depth for loop detection. +MAX_SUBMANIFEST_DEPTH = 8 # Add all projects from local manifest into a group. LOCAL_MANIFEST_GROUP_PREFIX = 'local:' @@ -197,10 +200,122 @@ class _XmlRemote(object): self.annotations.append(Annotation(name, value, keep)) +class _XmlSubmanifest: + """Manage the element specified in the manifest. + + Attributes: + name: a string, the name for this submanifest. + remote: a string, the remote.name for this submanifest. + project: a string, the name of the manifest project. + revision: a string, the commitish. + manifestName: a string, the submanifest file name. + groups: a list of strings, the groups to add to all projects in the submanifest. + path: a string, the relative path for the submanifest checkout. + annotations: (derived) a list of annotations. + present: (derived) a boolean, whether the submanifest's manifest file is present. + """ + def __init__(self, + name, + remote=None, + project=None, + revision=None, + manifestName=None, + groups=None, + path=None, + parent=None): + self.name = name + self.remote = remote + self.project = project + self.revision = revision + self.manifestName = manifestName + self.groups = groups + self.path = path + self.annotations = [] + outer_client = parent._outer_client or parent + if self.remote and not self.project: + raise ManifestParseError( + f'Submanifest {name}: must specify project when remote is given.') + rc = self.repo_client = RepoClient( + parent.repodir, manifestName, parent_groups=','.join(groups) or '', + submanifest_path=self.relpath, outer_client=outer_client) + + self.present = os.path.exists(os.path.join(self.repo_client.subdir, + MANIFEST_FILE_NAME)) + + def __eq__(self, other): + if not isinstance(other, _XmlSubmanifest): + return False + return ( + self.name == other.name and + self.remote == other.remote and + self.project == other.project and + self.revision == other.revision and + self.manifestName == other.manifestName and + self.groups == other.groups and + self.path == other.path and + sorted(self.annotations) == sorted(other.annotations)) + + def __ne__(self, other): + return not self.__eq__(other) + + def ToSubmanifestSpec(self, root): + """Return a SubmanifestSpec object, populating attributes""" + mp = root.manifestProject + remote = root.remotes[self.remote or root.default.remote.name] + # If a project was given, generate the url from the remote and project. + # If not, use this manifestProject's url. + if self.project: + manifestUrl = remote.ToRemoteSpec(self.project).url + else: + manifestUrl = mp.GetRemote(mp.remote.name).url + manifestName = self.manifestName or 'default.xml' + revision = self.revision or self.name + path = self.path or revision.split('/')[-1] + groups = self.groups or [] + + return SubmanifestSpec(self.name, manifestUrl, manifestName, revision, path, + groups) + + @property + def relpath(self): + """The path of this submanifest relative to the parent manifest.""" + revision = self.revision or self.name + return self.path or revision.split('/')[-1] + + def GetGroupsStr(self): + """Returns the `groups` given for this submanifest.""" + if self.groups: + return ','.join(self.groups) + return '' + + def AddAnnotation(self, name, value, keep): + """Add annotations to the submanifest.""" + self.annotations.append(Annotation(name, value, keep)) + + +class SubmanifestSpec: + """The submanifest element, with all fields expanded.""" + + def __init__(self, + name, + manifestUrl, + manifestName, + revision, + path, + groups): + self.name = name + self.manifestUrl = manifestUrl + self.manifestName = manifestName + self.revision = revision + self.path = path + self.groups = groups or [] + + class XmlManifest(object): """manages the repo configuration file""" - def __init__(self, repodir, manifest_file, local_manifests=None): + def __init__(self, repodir, manifest_file, local_manifests=None, + outer_client=None, parent_groups='', submanifest_path=''): """Initialize. Args: @@ -210,23 +325,37 @@ class XmlManifest(object): be |repodir|/|MANIFEST_FILE_NAME|. local_manifests: Full path to the directory of local override manifests. This will usually be |repodir|/|LOCAL_MANIFESTS_DIR_NAME|. + outer_client: RepoClient of the outertree. + parent_groups: a string, the groups to apply to this projects. + submanifest_path: The submanifest root relative to the repo root. """ # TODO(vapier): Move this out of this class. self.globalConfig = GitConfig.ForUser() self.repodir = os.path.abspath(repodir) - self.topdir = os.path.dirname(self.repodir) + self._CheckLocalPath(submanifest_path) + self.topdir = os.path.join(os.path.dirname(self.repodir), submanifest_path) self.manifestFile = manifest_file self.local_manifests = local_manifests self._load_local_manifests = True + self.parent_groups = parent_groups + + if outer_client and self.isGitcClient: + raise ManifestParseError('Multi-manifest is incompatible with `gitc-init`') + + if submanifest_path and not outer_client: + # If passing a submanifest_path, there must be an outer_client. + raise ManifestParseError(f'Bad call to {self.__class__.__name__}') + + # If self._outer_client is None, this is not a checkout that supports + # multi-tree. + self._outer_client = outer_client or self self.repoProject = MetaProject(self, 'repo', gitdir=os.path.join(repodir, 'repo/.git'), worktree=os.path.join(repodir, 'repo')) - mp = MetaProject(self, 'manifests', - gitdir=os.path.join(repodir, 'manifests.git'), - worktree=os.path.join(repodir, 'manifests')) + mp = self.SubmanifestProject(self.path_prefix) self.manifestProject = mp # This is a bit hacky, but we're in a chicken & egg situation: all the @@ -311,6 +440,31 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md ae.setAttribute('value', a.value) e.appendChild(ae) + def _SubmanifestToXml(self, r, doc, root): + """Generate XML node.""" + e = doc.createElement('submanifest') + root.appendChild(e) + e.setAttribute('name', r.name) + if r.remote is not None: + e.setAttribute('remote', r.remote) + if r.project is not None: + e.setAttribute('project', r.project) + if r.manifestName is not None: + e.setAttribute('manifest-name', r.manifestName) + if r.revision is not None: + e.setAttribute('revision', r.revision) + if r.path is not None: + e.setAttribute('path', r.path) + if r.groups: + e.setAttribute('groups', r.GetGroupsStr()) + + for a in r.annotations: + if a.keep == 'true': + ae = doc.createElement('annotation') + ae.setAttribute('name', a.name) + ae.setAttribute('value', a.value) + e.appendChild(ae) + def _ParseList(self, field): """Parse fields that contain flattened lists. @@ -329,6 +483,8 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md doc = xml.dom.minidom.Document() root = doc.createElement('manifest') + if self.is_submanifest: + root.setAttribute('path', self.path_prefix) doc.appendChild(root) # Save out the notice. There's a little bit of work here to give it the @@ -383,6 +539,11 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md root.appendChild(e) root.appendChild(doc.createTextNode('')) + for r in sorted(self.submanifests): + self._SubmanifestToXml(self.submanifests[r], doc, root) + if self.submanifests: + root.appendChild(doc.createTextNode('')) + def output_projects(parent, parent_node, projects): for project_name in projects: for project in self._projects[project_name]: @@ -537,6 +698,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md 'project', 'extend-project', 'include', + 'submanifest', # These are children of 'project' nodes. 'annotation', 'project', @@ -574,13 +736,75 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md def _output_manifest_project_extras(self, p, e): """Manifests can modify e if they support extra project attributes.""" + @property + def is_multimanifest(self): + """Whether this is a multimanifest checkout""" + return bool(self.outer_client.submanifests) + + @property + def is_submanifest(self): + """Whether this manifest is a submanifest""" + return self._outer_client and self._outer_client != self + + @property + def outer_client(self): + """The instance of the outermost manifest client""" + self._Load() + return self._outer_client + + @property + def all_manifests(self): + """Generator yielding all (sub)manifests.""" + self._Load() + outer = self._outer_client + yield outer + for tree in outer.all_children: + yield tree + + @property + def all_children(self): + """Generator yielding all child submanifests.""" + self._Load() + for child in self._submanifests.values(): + if child.repo_client: + yield child.repo_client + for tree in child.repo_client.all_children: + yield tree + + @property + def path_prefix(self): + """The path of this submanifest, relative to the outermost manifest.""" + if not self._outer_client or self == self._outer_client: + return '' + return os.path.relpath(self.topdir, self._outer_client.topdir) + + @property + def all_paths(self): + """All project paths for all (sub)manifests. See `paths`.""" + ret = {} + for tree in self.all_manifests: + prefix = tree.path_prefix + ret.update({os.path.join(prefix, k): v for k, v in tree.paths.items()}) + return ret + + @property + def all_projects(self): + """All projects for all (sub)manifests. See `projects`.""" + return list(itertools.chain.from_iterable(x._paths.values() for x in self.all_manifests)) + @property def paths(self): + """Return all paths for this manifest. + + Return: + A dictionary of {path: Project()}. `path` is relative to this manifest. + """ self._Load() return self._paths @property def projects(self): + """Return a list of all Projects in this manifest.""" self._Load() return list(self._paths.values()) @@ -594,6 +818,12 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md self._Load() return self._default + @property + def submanifests(self): + """All submanifests in this manifest.""" + self._Load() + return self._submanifests + @property def repo_hooks_project(self): self._Load() @@ -651,8 +881,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md return self._load_local_manifests and self.local_manifests def IsFromLocalManifest(self, project): - """Is the project from a local manifest? - """ + """Is the project from a local manifest?""" return any(x.startswith(LOCAL_MANIFEST_GROUP_PREFIX) for x in project.groups) @@ -676,6 +905,50 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md def EnableGitLfs(self): return self.manifestProject.config.GetBoolean('repo.git-lfs') + def FindManifestByPath(self, path): + """Returns the manifest containing path.""" + path = os.path.abspath(path) + manifest = self._outer_client or self + old = None + while manifest._submanifests and manifest != old: + old = manifest + for name in manifest._submanifests: + tree = manifest._submanifests[name] + if path.startswith(tree.repo_client.manifest.topdir): + manifest = tree.repo_client + break + return manifest + + @property + def subdir(self): + """Returns the path for per-submanifest objects for this manifest.""" + return self.SubmanifestInfoDir(self.path_prefix) + + def SubmanifestInfoDir(self, submanifest_path, object_path=''): + """Return the path to submanifest-specific info for a submanifest. + + Return the full path of the directory in which to put per-manifest objects. + + Args: + submanifest_path: a string, the path of the submanifest, relative to the + outermost topdir. If empty, then repodir is returned. + object_path: a string, relative path to append to the submanifest info + directory path. + """ + if submanifest_path: + return os.path.join(self.repodir, SUBMANIFEST_DIR, submanifest_path, + object_path) + else: + return os.path.join(self.repodir, object_path) + + def SubmanifestProject(self, submanifest_path): + """Return a manifestProject for a submanifest.""" + subdir = self.SubmanifestInfoDir(submanifest_path) + mp = MetaProject(self, 'manifests', + gitdir=os.path.join(subdir, 'manifests.git'), + worktree=os.path.join(subdir, 'manifests')) + return mp + def GetDefaultGroupsStr(self): """Returns the default group string for the platform.""" return 'default,platform-' + platform.system().lower() @@ -693,6 +966,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md self._paths = {} self._remotes = {} self._default = None + self._submanifests = {} self._repo_hooks_project = None self._superproject = {} self._contactinfo = ContactInfo(Wrapper().BUG_URL) @@ -700,20 +974,29 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md self.branch = None self._manifest_server = None - def _Load(self): + def _Load(self, initial_client=None, submanifest_depth=0): + if submanifest_depth > MAX_SUBMANIFEST_DEPTH: + raise ManifestParseError('maximum submanifest depth %d exceeded.' % + MAX_SUBMANIFEST_DEPTH) if not self._loaded: + if self._outer_client and self._outer_client != self: + # This will load all clients. + self._outer_client._Load(initial_client=self) + m = self.manifestProject b = m.GetBranch(m.CurrentBranch).merge if b is not None and b.startswith(R_HEADS): b = b[len(R_HEADS):] self.branch = b + parent_groups = self.parent_groups + # The manifestFile was specified by the user which is why we allow include # paths to point anywhere. nodes = [] nodes.append(self._ParseManifestXml( self.manifestFile, self.manifestProject.worktree, - restrict_includes=False)) + parent_groups=parent_groups, restrict_includes=False)) if self._load_local_manifests and self.local_manifests: try: @@ -722,9 +1005,10 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md local = os.path.join(self.local_manifests, local_file) # Since local manifests are entirely managed by the user, allow # them to point anywhere the user wants. + local_group = f'{LOCAL_MANIFEST_GROUP_PREFIX}:{local_file[:-4]}' nodes.append(self._ParseManifestXml( - local, self.repodir, - parent_groups=f'{LOCAL_MANIFEST_GROUP_PREFIX}:{local_file[:-4]}', + local, self.subdir, + parent_groups=f'{local_group},{parent_groups}', restrict_includes=False)) except OSError: pass @@ -743,6 +1027,23 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md self._loaded = True + # Now that we have loaded this manifest, load any submanifest manifests + # as well. We need to do this after self._loaded is set to avoid looping. + if self._outer_client: + for name in self._submanifests: + tree = self._submanifests[name] + spec = tree.ToSubmanifestSpec(self) + present = os.path.exists(os.path.join(self.subdir, MANIFEST_FILE_NAME)) + if present and tree.present and not tree.repo_client: + if initial_client and initial_client.topdir == self.topdir: + tree.repo_client = self + tree.present = present + elif not os.path.exists(self.subdir): + tree.present = False + if tree.present: + tree.repo_client._Load(initial_client=initial_client, + submanifest_depth=submanifest_depth + 1) + def _ParseManifestXml(self, path, include_root, parent_groups='', restrict_includes=True): """Parse a manifest XML and return the computed nodes. @@ -832,6 +1133,20 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md if self._default is None: self._default = _Default() + submanifest_paths = set() + for node in itertools.chain(*node_list): + if node.nodeName == 'submanifest': + submanifest = self._ParseSubmanifest(node) + if submanifest: + if submanifest.name in self._submanifests: + if submanifest != self._submanifests[submanifest.name]: + raise ManifestParseError( + 'submanifest %s already exists with different attributes' % + (submanifest.name)) + else: + self._submanifests[submanifest.name] = submanifest + submanifest_paths.add(submanifest.relpath) + for node in itertools.chain(*node_list): if node.nodeName == 'notice': if self._notice is not None: @@ -859,6 +1174,11 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md raise ManifestParseError( 'duplicate path %s in %s' % (project.relpath, self.manifestFile)) + for tree in submanifest_paths: + if project.relpath.startswith(tree): + raise ManifestParseError( + 'project %s conflicts with submanifest path %s' % + (project.relpath, tree)) self._paths[project.relpath] = project projects.append(project) for subproject in project.subprojects: @@ -883,8 +1203,10 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md if groups: groups = self._ParseList(groups) revision = node.getAttribute('revision') - remote = node.getAttribute('remote') - if remote: + remote_name = node.getAttribute('remote') + if not remote_name: + remote = self._default.remote + else: remote = self._get_remote(node) named_projects = self._projects[name] @@ -899,12 +1221,13 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md if revision: p.SetRevision(revision) - if remote: + if remote_name: p.remote = remote.ToRemoteSpec(name) if dest_path: del self._paths[p.relpath] - relpath, worktree, gitdir, objdir, _ = self.GetProjectPaths(name, dest_path) + relpath, worktree, gitdir, objdir, _ = self.GetProjectPaths( + name, dest_path, remote.name) p.UpdatePaths(relpath, worktree, gitdir, objdir) self._paths[p.relpath] = p @@ -1109,6 +1432,53 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md return '\n'.join(cleanLines) + def _ParseSubmanifest(self, node): + """Reads a element from the manifest file.""" + name = self._reqatt(node, 'name') + remote = node.getAttribute('remote') + if remote == '': + remote = None + project = node.getAttribute('project') + if project == '': + project = None + revision = node.getAttribute('revision') + if revision == '': + revision = None + manifestName = node.getAttribute('manifest-name') + if manifestName == '': + manifestName = None + groups = '' + if node.hasAttribute('groups'): + groups = node.getAttribute('groups') + groups = self._ParseList(groups) + path = node.getAttribute('path') + if path == '': + path = None + if revision: + msg = self._CheckLocalPath(revision.split('/')[-1]) + if msg: + raise ManifestInvalidPathError( + ' invalid "revision": %s: %s' % (revision, msg)) + else: + msg = self._CheckLocalPath(name) + if msg: + raise ManifestInvalidPathError( + ' invalid "name": %s: %s' % (name, msg)) + else: + msg = self._CheckLocalPath(path) + if msg: + raise ManifestInvalidPathError( + ' invalid "path": %s: %s' % (path, msg)) + + submanifest = _XmlSubmanifest(name, remote, project, revision, manifestName, + groups, path, self) + + for n in node.childNodes: + if n.nodeName == 'annotation': + self._ParseAnnotation(submanifest, n) + + return submanifest + def _JoinName(self, parent_name, name): return os.path.join(parent_name, name) @@ -1172,7 +1542,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md if parent is None: relpath, worktree, gitdir, objdir, use_git_worktrees = \ - self.GetProjectPaths(name, path) + self.GetProjectPaths(name, path, remote.name) else: use_git_worktrees = False relpath, worktree, gitdir, objdir = \ @@ -1218,31 +1588,54 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md return project - def GetProjectPaths(self, name, path): + def GetProjectPaths(self, name, path, remote): + """Return the paths for a project. + + Args: + name: a string, the name of the project. + path: a string, the path of the project. + remote: a string, the remote.name of the project. + """ # The manifest entries might have trailing slashes. Normalize them to avoid # unexpected filesystem behavior since we do string concatenation below. path = path.rstrip('/') name = name.rstrip('/') + remote = remote.rstrip('/') use_git_worktrees = False + use_remote_name = bool(self._outer_client._submanifests) relpath = path if self.IsMirror: worktree = None gitdir = os.path.join(self.topdir, '%s.git' % name) objdir = gitdir else: + if use_remote_name: + namepath = os.path.join(remote, f'{name}.git') + else: + namepath = f'{name}.git' worktree = os.path.join(self.topdir, path).replace('\\', '/') - gitdir = os.path.join(self.repodir, 'projects', '%s.git' % path) + gitdir = os.path.join(self.subdir, 'projects', '%s.git' % path) # We allow people to mix git worktrees & non-git worktrees for now. # This allows for in situ migration of repo clients. if os.path.exists(gitdir) or not self.UseGitWorktrees: - objdir = os.path.join(self.repodir, 'project-objects', '%s.git' % name) + objdir = os.path.join(self.subdir, 'project-objects', namepath) else: use_git_worktrees = True - gitdir = os.path.join(self.repodir, 'worktrees', '%s.git' % name) + gitdir = os.path.join(self.repodir, 'worktrees', namepath) objdir = gitdir return relpath, worktree, gitdir, objdir, use_git_worktrees - def GetProjectsWithName(self, name): + def GetProjectsWithName(self, name, all_manifests=False): + """All projects with |name|. + + Args: + name: a string, the name of the project. + all_manifests: a boolean, if True, then all manifests are searched. If + False, then only this manifest is searched. + """ + if all_manifests: + return list(itertools.chain.from_iterable( + x._projects.get(name, []) for x in self.all_manifests)) return self._projects.get(name, []) def GetSubprojectName(self, parent, submodule_path): @@ -1498,19 +1891,26 @@ class GitcManifest(XmlManifest): class RepoClient(XmlManifest): """Manages a repo client checkout.""" - def __init__(self, repodir, manifest_file=None): + def __init__(self, repodir, manifest_file=None, submanifest_path='', **kwargs): self.isGitcClient = False + submanifest_path = submanifest_path or '' + if submanifest_path: + self._CheckLocalPath(submanifest_path) + prefix = os.path.join(repodir, SUBMANIFEST_DIR, submanifest_path) + else: + prefix = repodir - if os.path.exists(os.path.join(repodir, LOCAL_MANIFEST_NAME)): + if os.path.exists(os.path.join(prefix, LOCAL_MANIFEST_NAME)): print('error: %s is not supported; put local manifests in `%s` instead' % - (LOCAL_MANIFEST_NAME, os.path.join(repodir, LOCAL_MANIFESTS_DIR_NAME)), + (LOCAL_MANIFEST_NAME, os.path.join(prefix, LOCAL_MANIFESTS_DIR_NAME)), file=sys.stderr) sys.exit(1) if manifest_file is None: - manifest_file = os.path.join(repodir, MANIFEST_FILE_NAME) - local_manifests = os.path.abspath(os.path.join(repodir, LOCAL_MANIFESTS_DIR_NAME)) - super().__init__(repodir, manifest_file, local_manifests) + manifest_file = os.path.join(prefix, MANIFEST_FILE_NAME) + local_manifests = os.path.abspath(os.path.join(prefix, LOCAL_MANIFESTS_DIR_NAME)) + super().__init__(repodir, manifest_file, local_manifests, + submanifest_path=submanifest_path, **kwargs) # TODO: Completely separate manifest logic out of the client. self.manifest = self -- cgit v1.2.3-54-g00ecf