manifest_xml.py

# Copyright (C) 2008 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import collections
import itertools
import os
import platform
import re
import sys
import xml.dom.minidom
import urllib.parse

import gitc_utils
from git_config import GitConfig
from git_refs import R_HEADS, HEAD
from git_superproject import Superproject
import platform_utils
from project import (
    Annotation,
    RemoteSpec,
    Project,
    RepoProject,
    ManifestProject,
)
from error import (
    ManifestParseError,
    ManifestInvalidPathError,
    ManifestInvalidRevisionError,
)
from wrapper import Wrapper

MANIFEST_FILE_NAME = "manifest.xml"
LOCAL_MANIFEST_NAME = "local_manifest.xml"
LOCAL_MANIFESTS_DIR_NAME = "local_manifests"
SUBMANIFEST_DIR = "submanifests"
# Limit submanifests to an arbitrary depth for loop detection.
MAX_SUBMANIFEST_DEPTH = 8
# Add all projects from sub manifest into a group.
SUBMANIFEST_GROUP_PREFIX = "submanifest:"

# Add all projects from local manifest into a group.
LOCAL_MANIFEST_GROUP_PREFIX = "local:"

# ContactInfo has the self-registered bug url, supplied by the manifest authors.
ContactInfo = collections.namedtuple("ContactInfo", "bugurl")

# urljoin gets confused if the scheme is not known.
urllib.parse.uses_relative.extend(
    ["ssh", "git", "persistent-https", "sso", "rpc"]
)
urllib.parse.uses_netloc.extend(
    ["ssh", "git", "persistent-https", "sso", "rpc"]
)


def XmlBool(node, attr, default=None):
    """Determine boolean value of |node|'s |attr|.

    Invalid values will issue a non-fatal warning.

    Args:
        node: XML node whose attributes we access.
        attr: The attribute to access.
        default: If the attribute is not set (value is empty), then use this.

    Returns:
        True if the attribute is a valid string representing true.
        False if the attribute is a valid string representing false.
        |default| otherwise.
    """
    value = node.getAttribute(attr)
    s = value.lower()
    if s == "":
        return default
    elif s in {"yes", "true", "1"}:
        return True
    elif s in {"no", "false", "0"}:
        return False
    else:
        print(
            'warning: manifest: %s="%s": ignoring invalid XML boolean'
            % (attr, value),
            file=sys.stderr,
        )
        return default


def XmlInt(node, attr, default=None):
    """Determine integer value of |node|'s |attr|.

    Args:
        node: XML node whose attributes we access.
        attr: The attribute to access.
        default: If the attribute is not set (value is empty), then use this.

    Returns:
        The number if the attribute is a valid number.

    Raises:
        ManifestParseError: The number is invalid.
    """
    value = node.getAttribute(attr)
    if not value:
        return default

    try:
        return int(value)
    except ValueError:
        raise ManifestParseError(
            'manifest: invalid %s="%s" integer' % (attr, value)
        )


class _Default(object):
    """Project defaults within the manifest."""

    revisionExpr = None
    destBranchExpr = None
    upstreamExpr = None
    remote = None
    sync_j = None
    sync_c = False
    sync_s = False
    sync_tags = True

    def __eq__(self, other):
        if not isinstance(other, _Default):
            return False
        return self.__dict__ == other.__dict__

    def __ne__(self, other):
        if not isinstance(other, _Default):
            return True
        return self.__dict__ != other.__dict__


class _XmlRemote(object):
    def __init__(
        self,
        name,
        alias=None,
        fetch=None,
        pushUrl=None,
        manifestUrl=None,
        review=None,
        revision=None,
    ):
        self.name = name
        self.fetchUrl = fetch
        self.pushUrl = pushUrl
        self.manifestUrl = manifestUrl
        self.remoteAlias = alias
        self.reviewUrl = review
        self.revision = revision
        self.resolvedFetchUrl = self._resolveFetchUrl()
        self.annotations = []

    def __eq__(self, other):
        if not isinstance(other, _XmlRemote):
            return False
        return (
            sorted(self.annotations) == sorted(other.annotations)
            and self.name == other.name
            and self.fetchUrl == other.fetchUrl
            and self.pushUrl == other.pushUrl
            and self.remoteAlias == other.remoteAlias
            and self.reviewUrl == other.reviewUrl
            and self.revision == other.revision
        )

    def __ne__(self, other):
        return not self.__eq__(other)

    def _resolveFetchUrl(self):
        if self.fetchUrl is None:
            return ""
        url = self.fetchUrl.rstrip("/")
        manifestUrl = self.manifestUrl.rstrip("/")
        # urljoin will gets confused over quite a few things.  The ones we care
        # about here are:
        # * no scheme in the base url, like <hostname:port>
        # We handle no scheme by replacing it with an obscure protocol, gopher
        # and then replacing it with the original when we are done.

        if manifestUrl.find(":") != manifestUrl.find("/") - 1:
            url = urllib.parse.urljoin("gopher://" + manifestUrl, url)
            url = re.sub(r"^gopher://", "", url)
        else:
            url = urllib.parse.urljoin(manifestUrl, url)
        return url

    def ToRemoteSpec(self, projectName):
        fetchUrl = self.resolvedFetchUrl.rstrip("/")
        url = fetchUrl + "/" + projectName
        remoteName = self.name
        if self.remoteAlias:
            remoteName = self.remoteAlias
        return RemoteSpec(
            remoteName,
            url=url,
            pushUrl=self.pushUrl,
            review=self.reviewUrl,
            orig_name=self.name,
            fetchUrl=self.fetchUrl,
        )

    def AddAnnotation(self, name, value, keep):
        self.annotations.append(Annotation(name, value, keep))


class _XmlSubmanifest:
    """Manage the <submanifest> element specified in the manifest.

    Attributes:
      name: a string, the name for this submanifest.
      remote: a string, the remote.name for this submanifest.
      project: a string, the name of the manifest project.
      revision: a string, the commitish.
      manifestName: a string, the submanifest file name.
      groups: a list of strings, the groups to add to all projects in the
          submanifest.
      default_groups: a list of strings, the default groups to sync.
      path: a string, the relative path for the submanifest checkout.
      parent: an XmlManifest, the parent manifest.
      annotations: (derived) a list of annotations.
      present: (derived) a boolean, whether the sub manifest file is present.
    """

    def __init__(
        self,
        name,
        remote=None,
        project=None,
        revision=None,
        manifestName=None,
        groups=None,
        default_groups=None,
        path=None,
        parent=None,
    ):
        self.name = name
        self.remote = remote
        self.project = project
        self.revision = revision
        self.manifestName = manifestName
        self.groups = groups
        self.default_groups = default_groups
        self.path = path
        self.parent = parent
        self.annotations = []
        outer_client = parent._outer_client or parent
        if self.remote and not self.project:
            raise ManifestParseError(
                f"Submanifest {name}: must specify project when remote is "
                "given."
            )
        # Construct the absolute path to the manifest file using the parent's
        # method, so that we can correctly create our repo_client.
        manifestFile = parent.SubmanifestInfoDir(
            os.path.join(parent.path_prefix, self.relpath),
            os.path.join("manifests", manifestName or "default.xml"),
        )
        linkFile = parent.SubmanifestInfoDir(
            os.path.join(parent.path_prefix, self.relpath), MANIFEST_FILE_NAME
        )
        self.repo_client = RepoClient(
            parent.repodir,
            linkFile,
            parent_groups=",".join(groups) or "",
            submanifest_path=self.relpath,
            outer_client=outer_client,
            default_groups=default_groups,
        )

        self.present = os.path.exists(manifestFile)

    def __eq__(self, other):
        if not isinstance(other, _XmlSubmanifest):
            return False
        return (
            self.name == other.name
            and self.remote == other.remote
            and self.project == other.project
            and self.revision == other.revision
            and self.manifestName == other.manifestName
            and self.groups == other.groups
            and self.default_groups == other.default_groups
            and self.path == other.path
            and sorted(self.annotations) == sorted(other.annotations)
        )

    def __ne__(self, other):
        return not self.__eq__(other)

    def ToSubmanifestSpec(self):
        """Return a SubmanifestSpec object, populating attributes"""
        mp = self.parent.manifestProject
        remote = self.parent.remotes[
            self.remote or self.parent.default.remote.name
        ]
        # If a project was given, generate the url from the remote and project.
        # If not, use this manifestProject's url.
        if self.project:
            manifestUrl = remote.ToRemoteSpec(self.project).url
        else:
            manifestUrl = mp.GetRemote().url
        manifestName = self.manifestName or "default.xml"
        revision = self.revision or self.name
        path = self.path or revision.split("/")[-1]
        groups = self.groups or []

        return SubmanifestSpec(
            self.name, manifestUrl, manifestName, revision, path, groups
        )

    @property
    def relpath(self):
        """The path of this submanifest relative to the parent manifest."""
        revision = self.revision or self.name
        return self.path or revision.split("/")[-1]

    def GetGroupsStr(self):
        """Returns the `groups` given for this submanifest."""
        if self.groups:
            return ",".join(self.groups)
        return ""

    def GetDefaultGroupsStr(self):
        """Returns the `default-groups` given for this submanifest."""
        return ",".join(self.default_groups or [])

    def AddAnnotation(self, name, value, keep):
        """Add annotations to the submanifest."""
        self.annotations.append(Annotation(name, value, keep))


class SubmanifestSpec:
    """The submanifest element, with all fields expanded."""

    def __init__(self, name, manifestUrl, manifestName, revision, path, groups):
        self.name = name
        self.manifestUrl = manifestUrl
        self.manifestName = manifestName
        self.revision = revision
        self.path = path
        self.groups = groups or []


class XmlManifest(object):
    """manages the repo configuration file"""

    def __init__(
        self,
        repodir,
        manifest_file,
        local_manifests=None,
        outer_client=None,
        parent_groups="",
        submanifest_path="",
        default_groups=None,
    ):
        """Initialize.

        Args:
            repodir: Path to the .repo/ dir for holding all internal checkout
                state. It must be in the top directory of the repo client
                checkout.
            manifest_file: Full path to the manifest file to parse.  This will
                usually be |repodir|/|MANIFEST_FILE_NAME|.
            local_manifests: Full path to the directory of local override
                manifests. This will usually be
                |repodir|/|LOCAL_MANIFESTS_DIR_NAME|.
            outer_client: RepoClient of the outer manifest.
            parent_groups: a string, the groups to apply to this projects.
            submanifest_path: The submanifest root relative to the repo root.
            default_groups: a string, the default manifest groups to use.
        """
        # TODO(vapier): Move this out of this class.
        self.globalConfig = GitConfig.ForUser()

        self.repodir = os.path.abspath(repodir)
        self._CheckLocalPath(submanifest_path)
        self.topdir = os.path.dirname(self.repodir)
        if submanifest_path:
            # This avoids a trailing os.path.sep when submanifest_path is empty.
            self.topdir = os.path.join(self.topdir, submanifest_path)
        if manifest_file != os.path.abspath(manifest_file):
            raise ManifestParseError("manifest_file must be abspath")
        self.manifestFile = manifest_file
        if not outer_client or outer_client == self:
            # manifestFileOverrides only exists in the outer_client's manifest,
            # since that is the only instance left when Unload() is called on
            # the outer manifest.
            self.manifestFileOverrides = {}
        self.local_manifests = local_manifests
        self._load_local_manifests = True
        self.parent_groups = parent_groups
        self.default_groups = default_groups

        if outer_client and self.isGitcClient:
            raise ManifestParseError(
                "Multi-manifest is incompatible with `gitc-init`"
            )

        if submanifest_path and not outer_client:
            # If passing a submanifest_path, there must be an outer_client.
            raise ManifestParseError(f"Bad call to {self.__class__.__name__}")

        # If self._outer_client is None, this is not a checkout that supports
        # multi-tree.
        self._outer_client = outer_client or self

        self.repoProject = RepoProject(
            self,
            "repo",
            gitdir=os.path.join(repodir, "repo/.git"),
            worktree=os.path.join(repodir, "repo"),
        )

        mp = self.SubmanifestProject(self.path_prefix)
        self.manifestProject = mp

        # This is a bit hacky, but we're in a chicken & egg situation: all the
        # normal repo settings live in the manifestProject which we just setup
        # above, so we couldn't easily query before that.  We assume Project()
        # init doesn't care if this changes afterwards.
        if os.path.exists(mp.gitdir) and mp.use_worktree:
            mp.use_git_worktrees = True

        self.Unload()

    def Override(self, name, load_local_manifests=True):
        """Use a different manifest, just for the current instantiation."""
        path = None

        # Look for a manifest by path in the filesystem (including the cwd).
        if not load_local_manifests:
            local_path = os.path.abspath(name)
            if os.path.isfile(local_path):
                path = local_path

        # Look for manifests by name from the manifests repo.
        if path is None:
            path = os.path.join(self.manifestProject.worktree, name)
            if not os.path.isfile(path):
                raise ManifestParseError("manifest %s not found" % name)

        self._load_local_manifests = load_local_manifests
        self._outer_client.manifestFileOverrides[self.path_prefix] = path
        self.Unload()
        self._Load()

    def Link(self, name):
        """Update the repo metadata to use a different manifest."""
        self.Override(name)

        # Old versions of repo would generate symlinks we need to clean up.
        platform_utils.remove(self.manifestFile, missing_ok=True)
        # This file is interpreted as if it existed inside the manifest repo.
        # That allows us to use <include> with the relative file name.
        with open(self.manifestFile, "w") as fp:
            fp.write(
                """<?xml version="1.0" encoding="UTF-8"?>
<!--
DO NOT EDIT THIS FILE!  It is generated by repo and changes will be discarded.
If you want to use a different manifest, use `repo init -m <file>` instead.

If you want to customize your checkout by overriding manifest settings, use
the local_manifests/ directory instead.

For more information on repo manifests, check out:
https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
-->
<manifest>
  <include name="%s" />
</manifest>
"""
                % (name,)
            )

    def _RemoteToXml(self, r, doc, root):
        e = doc.createElement("remote")
        root.appendChild(e)
        e.setAttribute("name", r.name)
        e.setAttribute("fetch", r.fetchUrl)
        if r.pushUrl is not None:
            e.setAttribute("pushurl", r.pushUrl)
        if r.remoteAlias is not None:
            e.setAttribute("alias", r.remoteAlias)
        if r.reviewUrl is not None:
            e.setAttribute("review", r.reviewUrl)
        if r.revision is not None:
            e.setAttribute("revision", r.revision)

        for a in r.annotations:
            if a.keep == "true":
                ae = doc.createElement("annotation")
                ae.setAttribute("name", a.name)
                ae.setAttribute("value", a.value)
                e.appendChild(ae)

    def _SubmanifestToXml(self, r, doc, root):
        """Generate XML <submanifest/> node."""
        e = doc.createElement("submanifest")
        root.appendChild(e)
        e.setAttribute("name", r.name)
        if r.remote is not None:
            e.setAttribute("remote", r.remote)
        if r.project is not None:
            e.setAttribute("project", r.project)
        if r.manifestName is not None:
            e.setAttribute("manifest-name", r.manifestName)
        if r.revision is not None:
            e.setAttribute("revision", r.revision)
        if r.path is not None:
            e.setAttribute("path", r.path)
        if r.groups:
            e.setAttribute("groups", r.GetGroupsStr())
        if r.default_groups:
            e.setAttribute("default-groups", r.GetDefaultGroupsStr())

        for a in r.annotations:
            if a.keep == "true":
                ae = doc.createElement("annotation")
                ae.setAttribute("name", a.name)
                ae.setAttribute("value", a.value)
                e.appendChild(ae)

    def _ParseList(self, field):
        """Parse fields that contain flattened lists.

        These are whitespace & comma separated.  Empty elements will be
        discarded.
        """
        return [x for x in re.split(r"[,\s]+", field) if x]

    def ToXml(
        self,
        peg_rev=False,
        peg_rev_upstream=True,
        peg_rev_dest_branch=True,
        groups=None,
        omit_local=False,
    ):
        """Return the current manifest XML."""
        mp = self.manifestProject

        if groups is None:
            groups = mp.manifest_groups
        if groups:
            groups = self._ParseList(groups)

        doc = xml.dom.minidom.Document()
        root = doc.createElement("manifest")
        if self.is_submanifest:
            root.setAttribute("path", self.path_prefix)
        doc.appendChild(root)

        # Save out the notice.  There's a little bit of work here to give it the
        # right whitespace, which assumes that the notice is automatically
        # indented by 4 by minidom.
        if self.notice:
            notice_element = root.appendChild(doc.createElement("notice"))
            notice_lines = self.notice.splitlines()
            indented_notice = (
                "\n".join(" " * 4 + line for line in notice_lines)
            )[4:]
            notice_element.appendChild(doc.createTextNode(indented_notice))

        d = self.default

        for r in sorted(self.remotes):
            self._RemoteToXml(self.remotes[r], doc, root)
        if self.remotes:
            root.appendChild(doc.createTextNode(""))

        have_default = False
        e = doc.createElement("default")
        if d.remote:
            have_default = True
            e.setAttribute("remote", d.remote.name)
        if d.revisionExpr:
            have_default = True
            e.setAttribute("revision", d.revisionExpr)
        if d.destBranchExpr:
            have_default = True
            e.setAttribute("dest-branch", d.destBranchExpr)
        if d.upstreamExpr:
            have_default = True
            e.setAttribute("upstream", d.upstreamExpr)
        if d.sync_j is not None:
            have_default = True
            e.setAttribute("sync-j", "%d" % d.sync_j)
        if d.sync_c:
            have_default = True
            e.setAttribute("sync-c", "true")
        if d.sync_s:
            have_default = True
            e.setAttribute("sync-s", "true")
        if not d.sync_tags:
            have_default = True
            e.setAttribute("sync-tags", "false")
        if have_default:
            root.appendChild(e)
            root.appendChild(doc.createTextNode(""))

        if self._manifest_server:
            e = doc.createElement("manifest-server")
            e.setAttribute("url", self._manifest_server)
            root.appendChild(e)
            root.appendChild(doc.createTextNode(""))

        for r in sorted(self.submanifests):
            self._SubmanifestToXml(self.submanifests[r], doc, root)
        if self.submanifests:
            root.appendChild(doc.createTextNode(""))

        def output_projects(parent, parent_node, projects):
            for project_name in projects:
                for project in self._projects[project_name]:
                    output_project(parent, parent_node, project)

        def output_project(parent, parent_node, p):
            if not p.MatchesGroups(groups):
                return

            if omit_local and self.IsFromLocalManifest(p):
                return

            name = p.name
            relpath = p.relpath
            if parent:
                name = self._UnjoinName(parent.name, name)
                relpath = self._UnjoinRelpath(parent.relpath, relpath)

            e = doc.createElement("project")
            parent_node.appendChild(e)
            e.setAttribute("name", name)
            if relpath != name:
                e.setAttribute("path", relpath)
            remoteName = None
            if d.remote:
                remoteName = d.remote.name
            if not d.remote or p.remote.orig_name != remoteName:
                remoteName = p.remote.orig_name
                e.setAttribute("remote", remoteName)
            if peg_rev:
                if self.IsMirror:
                    value = p.bare_git.rev_parse(p.revisionExpr + "^0")
                else:
                    value = p.work_git.rev_parse(HEAD + "^0")
                e.setAttribute("revision", value)
                if peg_rev_upstream:
                    if p.upstream:
                        e.setAttribute("upstream", p.upstream)
                    elif value != p.revisionExpr:
                        # Only save the origin if the origin is not a sha1, and
                        # the default isn't our value
                        e.setAttribute("upstream", p.revisionExpr)

                if peg_rev_dest_branch:
                    if p.dest_branch:
                        e.setAttribute("dest-branch", p.dest_branch)
                    elif value != p.revisionExpr:
                        e.setAttribute("dest-branch", p.revisionExpr)

            else:
                revision = (
                    self.remotes[p.remote.orig_name].revision or d.revisionExpr
                )
                if not revision or revision != p.revisionExpr:
                    e.setAttribute("revision", p.revisionExpr)
                elif p.revisionId:
                    e.setAttribute("revision", p.revisionId)
                if p.upstream and (
                    p.upstream != p.revisionExpr or p.upstream != d.upstreamExpr
                ):
                    e.setAttribute("upstream", p.upstream)

            if p.dest_branch and p.dest_branch != d.destBranchExpr:
                e.setAttribute("dest-branch", p.dest_branch)

            for c in p.copyfiles:
                ce = doc.createElement("copyfile")
                ce.setAttribute("src", c.src)
                ce.setAttribute("dest", c.dest)
                e.appendChild(ce)

            for lf in p.linkfiles:
                le = doc.createElement("linkfile")
                le.setAttribute("src", lf.src)
                le.setAttribute("dest", lf.dest)
                e.appendChild(le)

            default_groups = ["all", "name:%s" % p.name, "path:%s" % p.relpath]
            egroups = [g for g in p.groups if g not in default_groups]
            if egroups:
                e.setAttribute("groups", ",".join(egroups))

            for a in p.annotations:
                if a.keep == "true":
                    ae = doc.createElement("annotation")
                    ae.setAttribute("name", a.name)
                    ae.setAttribute("value", a.value)
                    e.appendChild(ae)

            if p.sync_c:
                e.setAttribute("sync-c", "true")

            if p.sync_s:
                e.setAttribute("sync-s", "true")

            if not p.sync_tags:
                e.setAttribute("sync-tags", "false")

            if p.clone_depth:
                e.setAttribute("clone-depth", str(p.clone_depth))

            self._output_manifest_project_extras(p, e)

            if p.subprojects:
                subprojects = set(subp.name for subp in p.subprojects)
                output_projects(p, e, list(sorted(subprojects)))

        projects = set(p.name for p in self._paths.values() if not p.parent)
        output_projects(None, root, list(sorted(projects)))

        if self._repo_hooks_project:
            root.appendChild(doc.createTextNode(""))
            e = doc.createElement("repo-hooks")
            e.setAttribute("in-project", self._repo_hooks_project.name)
            e.setAttribute(
                "enabled-list",
                " ".join(self._repo_hooks_project.enabled_repo_hooks),
            )
            root.appendChild(e)

        if self._superproject:
            root.appendChild(doc.createTextNode(""))
            e = doc.createElement("superproject")
            e.setAttribute("name", self._superproject.name)
            remoteName = None
            if d.remote:
                remoteName = d.remote.name
            remote = self._superproject.remote
            if not d.remote or remote.orig_name != remoteName:
                remoteName = remote.orig_name
                e.setAttribute("remote", remoteName)
            revision = remote.revision or d.revisionExpr
            if not revision or revision != self._superproject.revision:
                e.setAttribute("revision", self._superproject.revision)
            root.appendChild(e)

        if self._contactinfo.bugurl != Wrapper().BUG_URL:
            root.appendChild(doc.createTextNode(""))
            e = doc.createElement("contactinfo")
            e.setAttribute("bugurl", self._contactinfo.bugurl)
            root.appendChild(e)

        return doc

    def ToDict(self, **kwargs):
        """Return the current manifest as a dictionary."""
        # Elements that may only appear once.
        SINGLE_ELEMENTS = {
            "notice",
            "default",
            "manifest-server",
            "repo-hooks",
            "superproject",
            "contactinfo",
        }
        # Elements that may be repeated.
        MULTI_ELEMENTS = {
            "remote",
            "remove-project",
            "project",
            "extend-project",
            "include",
            "submanifest",
            # These are children of 'project' nodes.
            "annotation",
            "project",
            "copyfile",
            "linkfile",
        }

        doc = self.ToXml(**kwargs)
        ret = {}

        def append_children(ret, node):
            for child in node.childNodes:
                if child.nodeType == xml.dom.Node.ELEMENT_NODE:
                    attrs = child.attributes
                    element = dict(
                        (attrs.item(i).localName, attrs.item(i).value)
                        for i in range(attrs.length)
                    )
                    if child.nodeName in SINGLE_ELEMENTS:
                        ret[child.nodeName] = element
                    elif child.nodeName in MULTI_ELEMENTS:
                        ret.setdefault(child.nodeName, []).append(element)
                    else:
                        raise ManifestParseError(
                            'Unhandled element "%s"' % (child.nodeName,)
                        )

                    append_children(element, child)

        append_children(ret, doc.firstChild)

        return ret

    def Save(self, fd, **kwargs):
        """Write the current manifest out to the given file descriptor."""
        doc = self.ToXml(**kwargs)
        doc.writexml(fd, "", "  ", "\n", "UTF-8")

    def _output_manifest_project_extras(self, p, e):
        """Manifests can modify e if they support extra project attributes."""

    @property
    def is_multimanifest(self):
        """Whether this is a multimanifest checkout.

        This is safe to use as long as the outermost manifest XML has been
        parsed.
        """
        return bool(self._outer_client._submanifests)

    @property
    def is_submanifest(self):
        """Whether this manifest is a submanifest.

        This is safe to use as long as the outermost manifest XML has been
        parsed.
        """
        return self._outer_client and self._outer_client != self

    @property
    def outer_client(self):
        """The instance of the outermost manifest client."""
        self._Load()
        return self._outer_client

    @property
    def all_manifests(self):
        """Generator yielding all (sub)manifests, in depth-first order."""
        self._Load()
        outer = self._outer_client
        yield outer
        for tree in outer.all_children:
            yield tree

    @property
    def all_children(self):
        """Generator yielding all (present) child submanifests."""
        self._Load()
        for child in self._submanifests.values():
            if child.repo_client:
                yield child.repo_client
                for tree in child.repo_client.all_children:
                    yield tree

    @property
    def path_prefix(self):
        """The path of this submanifest, relative to the outermost manifest."""
        if not self._outer_client or self == self._outer_client:
            return ""
        return os.path.relpath(self.topdir, self._outer_client.topdir)

    @property
    def all_paths(self):
        """All project paths for all (sub)manifests.

        See also `paths`.

        Returns:
            A dictionary of {path: Project()}.  `path` is relative to the outer
            manifest.
        """
        ret = {}
        for tree in self.all_manifests:
            prefix = tree.path_prefix
            ret.update(
                {os.path.join(prefix, k): v for k, v in tree.paths.items()}
            )
        return ret

    @property
    def all_projects(self):
        """All projects for all (sub)manifests.  See `projects`."""
        return list(
            itertools.chain.from_iterable(
                x._paths.values() for x in self.all_manifests
            )
        )

    @property
    def paths(self):
        """Return all paths for this manifest.

        Returns:
            A dictionary of {path: Project()}.  `path` is relative to this
            manifest.
        """
        self._Load()
        return self._paths

    @property
    def projects(self):
        """Return a list of all Projects in this manifest."""
        self._Load()
        return list(self._paths.values())

    @property
    def remotes(self):
        """Return a list of remotes for this manifest."""
        self._Load()
        return self._remotes

    @property
    def default(self):
        """Return default values for this manifest."""
        self._Load()
        return self._default

    @property
    def submanifests(self):
        """All submanifests in this manifest."""
        self._Load()
        return self._submanifests

    @property
    def repo_hooks_project(self):
        self._Load()
        return self._repo_hooks_project

    @property
    def superproject(self):
        self._Load()
        return self._superproject

    @property
    def contactinfo(self):
        self._Load()
        return self._contactinfo

    @property
    def notice(self):
        self._Load()
        return self._notice

    @property
    def manifest_server(self):
        self._Load()
        return self._manifest_server

    @property
    def CloneBundle(self):
        clone_bundle = self.manifestProject.clone_bundle
        if clone_bundle is None:
            return False if self.manifestProject.partial_clone else True
        else:
            return clone_bundle

    @property
    def CloneFilter(self):
        if self.manifestProject.partial_clone:
            return self.manifestProject.clone_filter
        return None

    @property
    def CloneFilterForDepth(self):
        if self.manifestProject.clone_filter_for_depth:
            return self.manifestProject.clone_filter_for_depth
        return None

    @property
    def PartialCloneExclude(self):
        exclude = self.manifest.manifestProject.partial_clone_exclude or ""
        return set(x.strip() for x in exclude.split(","))

    def SetManifestOverride(self, path):
        """Override manifestFile.  The caller must call Unload()"""
        self._outer_client.manifest.manifestFileOverrides[
            self.path_prefix
        ] = path

    @property
    def UseLocalManifests(self):
        return self._load_local_manifests

    def SetUseLocalManifests(self, value):
        self._load_local_manifests = value

    @property
    def HasLocalManifests(self):
        return self._load_local_manifests and self.local_manifests

    def IsFromLocalManifest(self, project):
        """Is the project from a local manifest?"""
        return any(
            x.startswith(LOCAL_MANIFEST_GROUP_PREFIX) for x in project.groups
        )

    @property
    def IsMirror(self):
        return self.manifestProject.mirror

    @property
    def UseGitWorktrees(self):
        return self.manifestProject.use_worktree

    @property
    def IsArchive(self):
        return self.manifestProject.archive

    @property
    def HasSubmodules(self):
        return self.manifestProject.submodules

    @property
    def EnableGitLfs(self):
        return self.manifestProject.git_lfs

    def FindManifestByPath(self, path):
        """Returns the manifest containing path."""
        path = os.path.abspath(path)
        manifest = self._outer_client or self
        old = None
        while manifest._submanifests and manifest != old:
            old = manifest
            for name in manifest._submanifests:
                tree = manifest._submanifests[name]
                if path.startswith(tree.repo_client.manifest.topdir):
                    manifest = tree.repo_client
                    break
        return manifest

    @property
    def subdir(self):
        """Returns the path for per-submanifest objects for this manifest."""
        return self.SubmanifestInfoDir(self.path_prefix)

    def SubmanifestInfoDir(self, submanifest_path, object_path=""):
        """Return the path to submanifest-specific info for a submanifest.

        Return the full path of the directory in which to put per-manifest
        objects.

        Args:
            submanifest_path: a string, the path of the submanifest, relative to
                the outermost topdir.  If empty, then repodir is returned.
            object_path: a string, relative path to append to the submanifest
                info directory path.
        """
        if submanifest_path:
            return os.path.join(
                self.repodir, SUBMANIFEST_DIR, submanifest_path, object_path
            )
        else:
            return os.path.join(self.repodir, object_path)

    def SubmanifestProject(self, submanifest_path):
        """Return a manifestProject for a submanifest."""
        subdir = self.SubmanifestInfoDir(submanifest_path)
        mp = ManifestProject(
            self,
            "manifests",
            gitdir=os.path.join(subdir, "manifests.git"),
            worktree=os.path.join(subdir, "manifests"),
        )
        return mp

    def GetDefaultGroupsStr(self, with_platform=True):
        """Returns the default group string to use.

        Args:
            with_platform: a boolean, whether to include the group for the
                          underlying platform.
        """
        groups = ",".join(self.default_groups or ["default"])
        if with_platform:
            groups += f",platform-{platform.system().lower()}"
        return groups

    def GetGroupsStr(self):
        """Returns the manifest group string that should be synced."""
        return (
            self.manifestProject.manifest_groups or self.GetDefaultGroupsStr()
        )

    def Unload(self):
        """Unload the manifest.

        If the manifest files have been changed since Load() was called, this
        will cause the new/updated manifest to be used.

        """
        self._loaded = False
        self._projects = {}
        self._paths = {}
        self._remotes = {}
        self._default = None
        self._submanifests = {}
        self._repo_hooks_project = None
        self._superproject = None
        self._contactinfo = ContactInfo(Wrapper().BUG_URL)
        self._notice = None
        self.branch = None
        self._manifest_server = None

    def Load(self):
        """Read the manifest into memory."""
        # Do not expose internal arguments.
        self._Load()

    def _Load(self, initial_client=None, submanifest_depth=0):
        if submanifest_depth > MAX_SUBMANIFEST_DEPTH:
            raise ManifestParseError(
                "maximum submanifest depth %d exceeded." % MAX_SUBMANIFEST_DEPTH
            )
        if not self._loaded:
            if self._outer_client and self._outer_client != self:
                # This will load all clients.
                self._outer_client._Load(initial_client=self)

            savedManifestFile = self.manifestFile
            override = self._outer_client.manifestFileOverrides.get(
                self.path_prefix
            )
            if override:
                self.manifestFile = override

            try:
                m = self.manifestProject
                b = m.GetBranch(m.CurrentBranch).merge
                if b is not None and b.startswith(R_HEADS):
                    b = b[len(R_HEADS) :]
                self.branch = b

                parent_groups = self.parent_groups
                if self.path_prefix:
                    parent_groups = (
                        f"{SUBMANIFEST_GROUP_PREFIX}:path:"
                        f"{self.path_prefix},{parent_groups}"
                    )

                # The manifestFile was specified by the user which is why we
                # allow include paths to point anywhere.
                nodes = []
                nodes.append(
                    self._ParseManifestXml(
                        self.manifestFile,
                        self.manifestProject.worktree,
                        parent_groups=parent_groups,
                        restrict_includes=False,
                    )
                )

                if self._load_local_manifests and self.local_manifests:
                    try:
                        for local_file in sorted(
                            platform_utils.listdir(self.local_manifests)
                        ):
                            if local_file.endswith(".xml"):
                                local = os.path.join(
                                    self.local_manifests, local_file
                                )
                                # Since local manifests are entirely managed by
                                # the user, allow them to point anywhere the
                                # user wants.
                                local_group = (
                                    f"{LOCAL_MANIFEST_GROUP_PREFIX}:"
                                    f"{local_file[:-4]}"
                                )
                                nodes.append(
                                    self._ParseManifestXml(
                                        local,
                                        self.subdir,
                                        parent_groups=(
                                            f"{local_group},{parent_groups}"
                                        ),
                                        restrict_includes=False,
                                    )
                                )
                    except OSError:
                        pass

                try:
                    self._ParseManifest(nodes)
                except ManifestParseError as e:
                    # There was a problem parsing, unload ourselves in case they
                    # catch this error and try again later, we will show the
                    # correct error
                    self.Unload()
                    raise e

                if self.IsMirror:
                    self._AddMetaProjectMirror(self.repoProject)
                    self._AddMetaProjectMirror(self.manifestProject)

                self._loaded = True
            finally:
                if override:
                    self.manifestFile = savedManifestFile

            # Now that we have loaded this manifest, load any submanifests as
            # well. We need to do this after self._loaded is set to avoid
            # looping.
            for name in self._submanifests:
                tree = self._submanifests[name]
                tree.ToSubmanifestSpec()
                present = os.path.exists(
                    os.path.join(self.subdir, MANIFEST_FILE_NAME)
                )
                if present and tree.present and not tree.repo_client:
                    if initial_client and initial_client.topdir == self.topdir:
                        tree.repo_client = self
                        tree.present = present
                    elif not os.path.exists(self.subdir):
                        tree.present = False
                if present and tree.present:
                    tree.repo_client._Load(
                        initial_client=initial_client,
                        submanifest_depth=submanifest_depth + 1,
                    )

    def _ParseManifestXml(
        self,
        path,
        include_root,
        parent_groups="",
        restrict_includes=True,
        parent_node=None,
    ):
        """Parse a manifest XML and return the computed nodes.

        Args:
            path: The XML file to read & parse.
            include_root: The path to interpret include "name"s relative to.
            parent_groups: The groups to apply to this projects.
            restrict_includes: Whether to constrain the "name" attribute of
                includes.
            parent_node: The parent include node, to apply attribute to this
                projects.

        Returns:
            List of XML nodes.
        """
        try:
            root = xml.dom.minidom.parse(path)
        except (OSError, xml.parsers.expat.ExpatError) as e:
            raise ManifestParseError(
                "error parsing manifest %s: %s" % (path, e)
            )

        if not root or not root.childNodes:
            raise ManifestParseError("no root node in %s" % (path,))

        for manifest in root.childNodes:
            if manifest.nodeName == "manifest":
                break
        else:
            raise ManifestParseError("no <manifest> in %s" % (path,))

        nodes = []
        for node in manifest.childNodes:
            if node.nodeName == "include":
                name = self._reqatt(node, "name")
                if restrict_includes:
                    msg = self._CheckLocalPath(name)
                    if msg:
                        raise ManifestInvalidPathError(
                            '<include> invalid "name": %s: %s' % (name, msg)
                        )
                include_groups = ""
                if parent_groups:
                    include_groups = parent_groups
                if node.hasAttribute("groups"):
                    include_groups = (
                        node.getAttribute("groups") + "," + include_groups
                    )
                fp = os.path.join(include_root, name)
                if not os.path.isfile(fp):
                    raise ManifestParseError(
                        "include [%s/]%s doesn't exist or isn't a file"
                        % (include_root, name)
                    )
                try:
                    nodes.extend(
                        self._ParseManifestXml(
                            fp, include_root, include_groups, parent_node=node
                        )
                    )
                # should isolate this to the exact exception, but that's
                # tricky.  actual parsing implementation may vary.
                except (
                    KeyboardInterrupt,
                    RuntimeError,
                    SystemExit,
                    ManifestParseError,
                ):
                    raise
                except Exception as e:
                    raise ManifestParseError(
                        "failed parsing included manifest %s: %s" % (name, e)
                    )
            else:
                if parent_groups and node.nodeName == "project":
                    nodeGroups = parent_groups
                    if node.hasAttribute("groups"):
                        nodeGroups = (
                            node.getAttribute("groups") + "," + nodeGroups
                        )
                    node.setAttribute("groups", nodeGroups)
                if (
                    parent_node
                    and node.nodeName == "project"
                    and not node.hasAttribute("revision")
                ):
                    node.setAttribute(
                        "revision", parent_node.getAttribute("revision")
                    )
                nodes.append(node)
        return nodes

    def _ParseManifest(self, node_list):
        for node in itertools.chain(*node_list):
            if node.nodeName == "remote":
                remote = self._ParseRemote(node)
                if remote:
                    if remote.name in self._remotes:
                        if remote != self._remotes[remote.name]:
                            raise ManifestParseError(
                                "remote %s already exists with different "
                                "attributes" % (remote.name)
                            )
                    else:
                        self._remotes[remote.name] = remote

        for node in itertools.chain(*node_list):
            if node.nodeName == "default":
                new_default = self._ParseDefault(node)
                emptyDefault = (
                    not node.hasAttributes() and not node.hasChildNodes()
                )
                if self._default is None:
                    self._default = new_default
                elif not emptyDefault and new_default != self._default:
                    raise ManifestParseError(
                        "duplicate default in %s" % (self.manifestFile)
                    )

        if self._default is None:
            self._default = _Default()

        submanifest_paths = set()
        for node in itertools.chain(*node_list):
            if node.nodeName == "submanifest":
                submanifest = self._ParseSubmanifest(node)
                if submanifest:
                    if submanifest.name in self._submanifests:
                        if submanifest != self._submanifests[submanifest.name]:
                            raise ManifestParseError(
                                "submanifest %s already exists with different "
                                "attributes" % (submanifest.name)
                            )
                    else:
                        self._submanifests[submanifest.name] = submanifest
                        submanifest_paths.add(submanifest.relpath)

        for node in itertools.chain(*node_list):
            if node.nodeName == "notice":
                if self._notice is not None:
                    raise ManifestParseError(
                        "duplicate notice in %s" % (self.manifestFile)
                    )
                self._notice = self._ParseNotice(node)

        for node in itertools.chain(*node_list):
            if node.nodeName == "manifest-server":
                url = self._reqatt(node, "url")
                if self._manifest_server is not None:
                    raise ManifestParseError(
                        "duplicate manifest-server in %s" % (self.manifestFile)
                    )
                self._manifest_server = url

        def recursively_add_projects(project):
            projects = self._projects.setdefault(project.name, [])
            if project.relpath is None:
                raise ManifestParseError(
                    "missing path for %s in %s"
                    % (project.name, self.manifestFile)
                )
            if project.relpath in self._paths:
                raise ManifestParseError(
                    "duplicate path %s in %s"
                    % (project.relpath, self.manifestFile)
                )
            for tree in submanifest_paths:
                if project.relpath.startswith(tree):
                    raise ManifestParseError(
                        "project %s conflicts with submanifest path %s"
                        % (project.relpath, tree)
                    )
            self._paths[project.relpath] = project
            projects.append(project)
            for subproject in project.subprojects:
                recursively_add_projects(subproject)

        repo_hooks_project = None
        enabled_repo_hooks = None
        for node in itertools.chain(*node_list):
            if node.nodeName == "project":
                project = self._ParseProject(node)
                recursively_add_projects(project)
            if node.nodeName == "extend-project":
                name = self._reqatt(node, "name")

                if name not in self._projects:
                    raise ManifestParseError(
                        "extend-project element specifies non-existent "
                        "project: %s" % name
                    )

                path = node.getAttribute("path")
                dest_path = node.getAttribute("dest-path")
                groups = node.getAttribute("groups")
                if groups:
                    groups = self._ParseList(groups)
                revision = node.getAttribute("revision")
                remote_name = node.getAttribute("remote")
                if not remote_name:
                    remote = self._default.remote
                else:
                    remote = self._get_remote(node)
                dest_branch = node.getAttribute("dest-branch")
                upstream = node.getAttribute("upstream")

                named_projects = self._projects[name]
                if dest_path and not path and len(named_projects) > 1:
                    raise ManifestParseError(
                        "extend-project cannot use dest-path when "
                        "matching multiple projects: %s" % name
                    )
                for p in self._projects[name]:
                    if path and p.relpath != path:
                        continue
                    if groups:
                        p.groups.extend(groups)
                    if revision:
                        p.SetRevision(revision)

                    if remote_name:
                        p.remote = remote.ToRemoteSpec(name)
                    if dest_branch:
                        p.dest_branch = dest_branch
                    if upstream:
                        p.upstream = upstream

                    if dest_path:
                        del self._paths[p.relpath]
                        (
                            relpath,
                            worktree,
                            gitdir,
                            objdir,
                            _,
                        ) = self.GetProjectPaths(name, dest_path, remote.name)
                        p.UpdatePaths(relpath, worktree, gitdir, objdir)
                        self._paths[p.relpath] = p

            if node.nodeName == "repo-hooks":
                # Only one project can be the hooks project
                if repo_hooks_project is not None:
                    raise ManifestParseError(
                        "duplicate repo-hooks in %s" % (self.manifestFile)
                    )

                # Get the name of the project and the (space-separated) list of
                # enabled.
                repo_hooks_project = self._reqatt(node, "in-project")
                enabled_repo_hooks = self._ParseList(
                    self._reqatt(node, "enabled-list")
                )
            if node.nodeName == "superproject":
                name = self._reqatt(node, "name")
                # There can only be one superproject.
                if self._superproject:
                    raise ManifestParseError(
                        "duplicate superproject in %s" % (self.manifestFile)
                    )
                remote_name = node.getAttribute("remote")
                if not remote_name:
                    remote = self._default.remote
                else:
                    remote = self._get_remote(node)
                if remote is None:
                    raise ManifestParseError(
                        "no remote for superproject %s within %s"
                        % (name, self.manifestFile)
                    )
                revision = node.getAttribute("revision") or remote.revision
                if not revision:
                    revision = self._default.revisionExpr
                if not revision:
                    raise ManifestParseError(
                        "no revision for superproject %s within %s"
                        % (name, self.manifestFile)
                    )
                self._superproject = Superproject(
                    self,
                    name=name,
                    remote=remote.ToRemoteSpec(name),
                    revision=revision,
                )
            if node.nodeName == "contactinfo":
                bugurl = self._reqatt(node, "bugurl")
                # This element can be repeated, later entries will clobber
                # earlier ones.
                self._contactinfo = ContactInfo(bugurl)

            if node.nodeName == "remove-project":
                name = node.getAttribute("name")
                path = node.getAttribute("path")

                # Name or path needed.
                if not name and not path:
                    raise ManifestParseError(
                        "remove-project must have name and/or path"
                    )

                removed_project = ""

                # Find and remove projects based on name and/or path.
                for projname, projects in list(self._projects.items()):
                    for p in projects:
                        if name == projname and not path:
                            del self._paths[p.relpath]
                            if not removed_project:
                                del self._projects[name]
                                removed_project = name
                        elif path == p.relpath and (
                            name == projname or not name
                        ):
                            self._projects[projname].remove(p)
                            del self._paths[p.relpath]
                            removed_project = p.name

                # If the manifest removes the hooks project, treat it as if
                # it deleted the repo-hooks element too.
                if (
                    removed_project
                    and removed_project not in self._projects
                    and repo_hooks_project == removed_project
                ):
                    repo_hooks_project = None

                if not removed_project and not XmlBool(node, "optional", False):
                    raise ManifestParseError(
                        "remove-project element specifies non-existent "
                        "project: %s" % node.toxml()
                    )

        # Store repo hooks project information.
        if repo_hooks_project:
            # Store a reference to the Project.
            try:
                repo_hooks_projects = self._projects[repo_hooks_project]
            except KeyError:
                raise ManifestParseError(
                    "project %s not found for repo-hooks" % (repo_hooks_project)
                )

            if len(repo_hooks_projects) != 1:
                raise ManifestParseError(
                    "internal error parsing repo-hooks in %s"
                    % (self.manifestFile)
                )
            self._repo_hooks_project = repo_hooks_projects[0]
            # Store the enabled hooks in the Project object.
            self._repo_hooks_project.enabled_repo_hooks = enabled_repo_hooks

    def _AddMetaProjectMirror(self, m):
        name = None
        m_url = m.GetRemote().url
        if m_url.endswith("/.git"):
            raise ManifestParseError("refusing to mirror %s" % m_url)

        if self._default and self._default.remote:
            url = self._default.remote.resolvedFetchUrl
            if not url.endswith("/"):
                url += "/"
            if m_url.startswith(url):
                remote = self._default.remote
                name = m_url[len(url) :]

        if name is None:
            s = m_url.rindex("/") + 1
            manifestUrl = self.manifestProject.config.GetString(
                "remote.origin.url"
            )
            remote = _XmlRemote(
                "origin", fetch=m_url[:s], manifestUrl=manifestUrl
            )
            name = m_url[s:]

        if name.endswith(".git"):
            name = name[:-4]

        if name not in self._projects:
            m.PreSync()
            gitdir = os.path.join(self.topdir, "%s.git" % name)
            project = Project(
                manifest=self,
                name=name,
                remote=remote.ToRemoteSpec(name),
                gitdir=gitdir,
                objdir=gitdir,
                worktree=None,
                relpath=name or None,
                revisionExpr=m.revisionExpr,
                revisionId=None,
            )
            self._projects[project.name] = [project]
            self._paths[project.relpath] = project

    def _ParseRemote(self, node):
        """
        reads a <remote> element from the manifest file
        """
        name = self._reqatt(node, "name")
        alias = node.getAttribute("alias")
        if alias == "":
            alias = None
        fetch = self._reqatt(node, "fetch")
        pushUrl = node.getAttribute("pushurl")
        if pushUrl == "":
            pushUrl = None
        review = node.getAttribute("review")
        if review == "":
            review = None
        revision = node.getAttribute("revision")
        if revision == "":
            revision = None
        manifestUrl = self.manifestProject.config.GetString("remote.origin.url")

        remote = _XmlRemote(
            name, alias, fetch, pushUrl, manifestUrl, review, revision
        )

        for n in node.childNodes:
            if n.nodeName == "annotation":
                self._ParseAnnotation(remote, n)

        return remote

    def _ParseDefault(self, node):
        """
        reads a <default> element from the manifest file
        """
        d = _Default()
        d.remote = self._get_remote(node)
        d.revisionExpr = node.getAttribute("revision")
        if d.revisionExpr == "":
            d.revisionExpr = None

        d.destBranchExpr = node.getAttribute("dest-branch") or None
        d.upstreamExpr = node.getAttribute("upstream") or None

        d.sync_j = XmlInt(node, "sync-j", None)
        if d.sync_j is not None and d.sync_j <= 0:
            raise ManifestParseError(
                '%s: sync-j must be greater than 0, not "%s"'
                % (self.manifestFile, d.sync_j)
            )

        d.sync_c = XmlBool(node, "sync-c", False)
        d.sync_s = XmlBool(node, "sync-s", False)
        d.sync_tags = XmlBool(node, "sync-tags", True)
        return d

    def _ParseNotice(self, node):
        """
        reads a <notice> element from the manifest file

        The <notice> element is distinct from other tags in the XML in that the
        data is conveyed between the start and end tag (it's not an
        empty-element tag).

        The white space (carriage returns, indentation) for the notice element
        is relevant and is parsed in a way that is based on how python
        docstrings work. In fact, the code is remarkably similar to here:
          http://www.python.org/dev/peps/pep-0257/
        """
        # Get the data out of the node...
        notice = node.childNodes[0].data

        # Figure out minimum indentation, skipping the first line (the same line
        # as the <notice> tag)...
        minIndent = sys.maxsize
        lines = notice.splitlines()
        for line in lines[1:]:
            lstrippedLine = line.lstrip()
            if lstrippedLine:
                indent = len(line) - len(lstrippedLine)
                minIndent = min(indent, minIndent)

        # Strip leading / trailing blank lines and also indentation.
        cleanLines = [lines[0].strip()]
        for line in lines[1:]:
            cleanLines.append(line[minIndent:].rstrip())

        # Clear completely blank lines from front and back...
        while cleanLines and not cleanLines[0]:
            del cleanLines[0]
        while cleanLines and not cleanLines[-1]:
            del cleanLines[-1]

        return "\n".join(cleanLines)

    def _ParseSubmanifest(self, node):
        """Reads a <submanifest> element from the manifest file."""
        name = self._reqatt(node, "name")
        remote = node.getAttribute("remote")
        if remote == "":
            remote = None
        project = node.getAttribute("project")
        if project == "":
            project = None
        revision = node.getAttribute("revision")
        if revision == "":
            revision = None
        manifestName = node.getAttribute("manifest-name")
        if manifestName == "":
            manifestName = None
        groups = ""
        if node.hasAttribute("groups"):
            groups = node.getAttribute("groups")
        groups = self._ParseList(groups)
        default_groups = self._ParseList(node.getAttribute("default-groups"))
        path = node.getAttribute("path")
        if path == "":
            path = None
            if revision:
                msg = self._CheckLocalPath(revision.split("/")[-1])
                if msg:
                    raise ManifestInvalidPathError(
                        '<submanifest> invalid "revision": %s: %s'
                        % (revision, msg)
                    )
            else:
                msg = self._CheckLocalPath(name)
                if msg:
                    raise ManifestInvalidPathError(
                        '<submanifest> invalid "name": %s: %s' % (name, msg)
                    )
        else:
            msg = self._CheckLocalPath(path)
            if msg:
                raise ManifestInvalidPathError(
                    '<submanifest> invalid "path": %s: %s' % (path, msg)
                )

        submanifest = _XmlSubmanifest(
            name,
            remote,
            project,
            revision,
            manifestName,
            groups,
            default_groups,
            path,
            self,
        )

        for n in node.childNodes:
            if n.nodeName == "annotation":
                self._ParseAnnotation(submanifest, n)

        return submanifest

    def _JoinName(self, parent_name, name):
        return os.path.join(parent_name, name)

    def _UnjoinName(self, parent_name, name):
        return os.path.relpath(name, parent_name)

    def _ParseProject(self, node, parent=None, **extra_proj_attrs):
        """
        reads a <project> element from the manifest file
        """
        name = self._reqatt(node, "name")
        msg = self._CheckLocalPath(name, dir_ok=True)
        if msg:
            raise ManifestInvalidPathError(
                '<project> invalid "name": %s: %s' % (name, msg)
            )
        if parent:
            name = self._JoinName(parent.name, name)

        remote = self._get_remote(node)
        if remote is None:
            remote = self._default.remote
        if remote is None:
            raise ManifestParseError(
                "no remote for project %s within %s" % (name, self.manifestFile)
            )

        revisionExpr = node.getAttribute("revision") or remote.revision
        if not revisionExpr:
            revisionExpr = self._default.revisionExpr
        if not revisionExpr:
            raise ManifestParseError(
                "no revision for project %s within %s"
                % (name, self.manifestFile)
            )

        path = node.getAttribute("path")
        if not path:
            path = name
        else:
            # NB: The "." project is handled specially in
            # Project.Sync_LocalHalf.
            msg = self._CheckLocalPath(path, dir_ok=True, cwd_dot_ok=True)
            if msg:
                raise ManifestInvalidPathError(
                    '<project> invalid "path": %s: %s' % (path, msg)
                )

        rebase = XmlBool(node, "rebase", True)
        sync_c = XmlBool(node, "sync-c", False)
        sync_s = XmlBool(node, "sync-s", self._default.sync_s)
        sync_tags = XmlBool(node, "sync-tags", self._default.sync_tags)

        clone_depth = XmlInt(node, "clone-depth")
        if clone_depth is not None and clone_depth <= 0:
            raise ManifestParseError(
                '%s: clone-depth must be greater than 0, not "%s"'
                % (self.manifestFile, clone_depth)
            )

        dest_branch = (
            node.getAttribute("dest-branch") or self._default.destBranchExpr
        )

        upstream = node.getAttribute("upstream") or self._default.upstreamExpr

        groups = ""
        if node.hasAttribute("groups"):
            groups = node.getAttribute("groups")
        groups = self._ParseList(groups)

        if parent is None:
            (
                relpath,
                worktree,
                gitdir,
                objdir,
                use_git_worktrees,
            ) = self.GetProjectPaths(name, path, remote.name)
        else:
            use_git_worktrees = False
            relpath, worktree, gitdir, objdir = self.GetSubprojectPaths(
                parent, name, path
            )

        default_groups = ["all", "name:%s" % name, "path:%s" % relpath]
        groups.extend(set(default_groups).difference(groups))

        if self.IsMirror and node.hasAttribute("force-path"):
            if XmlBool(node, "force-path", False):
                gitdir = os.path.join(self.topdir, "%s.git" % path)

        project = Project(
            manifest=self,
            name=name,
            remote=remote.ToRemoteSpec(name),
            gitdir=gitdir,
            objdir=objdir,
            worktree=worktree,
            relpath=relpath,
            revisionExpr=revisionExpr,
            revisionId=None,
            rebase=rebase,
            groups=groups,
            sync_c=sync_c,
            sync_s=sync_s,
            sync_tags=sync_tags,
            clone_depth=clone_depth,
            upstream=upstream,
            parent=parent,
            dest_branch=dest_branch,
            use_git_worktrees=use_git_worktrees,
            **extra_proj_attrs,
        )

        for n in node.childNodes:
            if n.nodeName == "copyfile":
                self._ParseCopyFile(project, n)
            if n.nodeName == "linkfile":
                self._ParseLinkFile(project, n)
            if n.nodeName == "annotation":
                self._ParseAnnotation(project, n)
            if n.nodeName == "project":
                project.subprojects.append(
                    self._ParseProject(n, parent=project)
                )

        return project

    def GetProjectPaths(self, name, path, remote):
        """Return the paths for a project.

        Args:
            name: a string, the name of the project.
            path: a string, the path of the project.
            remote: a string, the remote.name of the project.

        Returns:
            A tuple of (relpath, worktree, gitdir, objdir, use_git_worktrees)
            for the project with |name| and |path|.
        """
        # The manifest entries might have trailing slashes.  Normalize them to
        # avoid unexpected filesystem behavior since we do string concatenation
        # below.
        path = path.rstrip("/")
        name = name.rstrip("/")
        remote = remote.rstrip("/")
        use_git_worktrees = False
        use_remote_name = self.is_multimanifest
        relpath = path
        if self.IsMirror:
            worktree = None
            gitdir = os.path.join(self.topdir, "%s.git" % name)
            objdir = gitdir
        else:
            if use_remote_name:
                namepath = os.path.join(remote, f"{name}.git")
            else:
                namepath = f"{name}.git"
            worktree = os.path.join(self.topdir, path).replace("\\", "/")
            gitdir = os.path.join(self.subdir, "projects", "%s.git" % path)
            # We allow people to mix git worktrees & non-git worktrees for now.
            # This allows for in situ migration of repo clients.
            if os.path.exists(gitdir) or not self.UseGitWorktrees:
                objdir = os.path.join(self.repodir, "project-objects", namepath)
            else:
                use_git_worktrees = True
                gitdir = os.path.join(self.repodir, "worktrees", namepath)
                objdir = gitdir
        return relpath, worktree, gitdir, objdir, use_git_worktrees

    def GetProjectsWithName(self, name, all_manifests=False):
        """All projects with |name|.

        Args:
            name: a string, the name of the project.
            all_manifests: a boolean, if True, then all manifests are searched.
                If False, then only this manifest is searched.

        Returns:
            A list of Project instances with name |name|.
        """
        if all_manifests:
            return list(
                itertools.chain.from_iterable(
                    x._projects.get(name, []) for x in self.all_manifests
                )
            )
        return self._projects.get(name, [])

    def GetSubprojectName(self, parent, submodule_path):
        return os.path.join(parent.name, submodule_path)

    def _JoinRelpath(self, parent_relpath, relpath):
        return os.path.join(parent_relpath, relpath)

    def _UnjoinRelpath(self, parent_relpath, relpath):
        return os.path.relpath(relpath, parent_relpath)

    def GetSubprojectPaths(self, parent, name, path):
        # The manifest entries might have trailing slashes.  Normalize them to
        # avoid unexpected filesystem behavior since we do string concatenation
        # below.
        path = path.rstrip("/")
        name = name.rstrip("/")
        relpath = self._JoinRelpath(parent.relpath, path)
        gitdir = os.path.join(parent.gitdir, "subprojects", "%s.git" % path)
        objdir = os.path.join(
            parent.gitdir, "subproject-objects", "%s.git" % name
        )
        if self.IsMirror:
            worktree = None
        else:
            worktree = os.path.join(parent.worktree, path).replace("\\", "/")
        return relpath, worktree, gitdir, objdir

    @staticmethod
    def _CheckLocalPath(path, dir_ok=False, cwd_dot_ok=False):
        """Verify |path| is reasonable for use in filesystem paths.

        Used with <copyfile> & <linkfile> & <project> elements.

        This only validates the |path| in isolation: it does not check against
        the current filesystem state.  Thus it is suitable as a first-past in a
        parser.

        It enforces a number of constraints:
        * No empty paths.
        * No "~" in paths.
        * No Unicode codepoints that filesystems might elide when normalizing.
        * No relative path components like "." or "..".
        * No absolute paths.
        * No ".git" or ".repo*" path components.

        Args:
            path: The path name to validate.
            dir_ok: Whether |path| may force a directory (e.g. end in a /).
            cwd_dot_ok: Whether |path| may be just ".".

        Returns:
            None if |path| is OK, a failure message otherwise.
        """
        if not path:
            return "empty paths not allowed"

        if "~" in path:
            return "~ not allowed (due to 8.3 filenames on Windows filesystems)"

        path_codepoints = set(path)

        # Some filesystems (like Apple's HFS+) try to normalize Unicode
        # codepoints which means there are alternative names for ".git".  Reject
        # paths with these in it as there shouldn't be any reasonable need for
        # them here. The set of codepoints here was cribbed from jgit's
        # implementation:
        # https://eclipse.googlesource.com/jgit/jgit/+/9110037e3e9461ff4dac22fee84ef3694ed57648/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectChecker.java#884
        BAD_CODEPOINTS = {
            "\u200C",  # ZERO WIDTH NON-JOINER
            "\u200D",  # ZERO WIDTH JOINER
            "\u200E",  # LEFT-TO-RIGHT MARK
            "\u200F",  # RIGHT-TO-LEFT MARK
            "\u202A",  # LEFT-TO-RIGHT EMBEDDING
            "\u202B",  # RIGHT-TO-LEFT EMBEDDING
            "\u202C",  # POP DIRECTIONAL FORMATTING
            "\u202D",  # LEFT-TO-RIGHT OVERRIDE
            "\u202E",  # RIGHT-TO-LEFT OVERRIDE
            "\u206A",  # INHIBIT SYMMETRIC SWAPPING
            "\u206B",  # ACTIVATE SYMMETRIC SWAPPING
            "\u206C",  # INHIBIT ARABIC FORM SHAPING
            "\u206D",  # ACTIVATE ARABIC FORM SHAPING
            "\u206E",  # NATIONAL DIGIT SHAPES
            "\u206F",  # NOMINAL DIGIT SHAPES
            "\uFEFF",  # ZERO WIDTH NO-BREAK SPACE
        }
        if BAD_CODEPOINTS & path_codepoints:
            # This message is more expansive than reality, but should be fine.
            return "Unicode combining characters not allowed"

        # Reject newlines as there shouldn't be any legitmate use for them,
        # they'll be confusing to users, and they can easily break tools that
        # expect to be able to iterate over newline delimited lists.  This even
        # applies to our own code like .repo/project.list.
        if {"\r", "\n"} & path_codepoints:
            return "Newlines not allowed"

        # Assume paths might be used on case-insensitive filesystems.
        path = path.lower()

        # Split up the path by its components.  We can't use os.path.sep
        # exclusively as some platforms (like Windows) will convert / to \ and
        # that bypasses all our constructed logic here.  Especially since
        # manifest authors only use / in their paths.
        resep = re.compile(r"[/%s]" % re.escape(os.path.sep))
        # Strip off trailing slashes as those only produce '' elements, and we
        # use parts to look for individual bad components.
        parts = resep.split(path.rstrip("/"))

        # Some people use src="." to create stable links to projects.  Lets
        # allow that but reject all other uses of "." to keep things simple.
        if not cwd_dot_ok or parts != ["."]:
            for part in set(parts):
                if part in {".", "..", ".git"} or part.startswith(".repo"):
                    return "bad component: %s" % (part,)

        if not dir_ok and resep.match(path[-1]):
            return "dirs not allowed"

        # NB: The two abspath checks here are to handle platforms with multiple
        # filesystem path styles (e.g. Windows).
        norm = os.path.normpath(path)
        if (
            norm == ".."
            or (
                len(norm) >= 3
                and norm.startswith("..")
                and resep.match(norm[0])
            )
            or os.path.isabs(norm)
            or norm.startswith("/")
        ):
            return "path cannot be outside"

    @classmethod
    def _ValidateFilePaths(cls, element, src, dest):
        """Verify |src| & |dest| are reasonable for <copyfile> & <linkfile>.

        We verify the path independent of any filesystem state as we won't have
        a checkout available to compare to.  i.e. This is for parsing validation
        purposes only.

        We'll do full/live sanity checking before we do the actual filesystem
        modifications in _CopyFile/_LinkFile/etc...
        """
        # |dest| is the file we write to or symlink we create.
        # It is relative to the top of the repo client checkout.
        msg = cls._CheckLocalPath(dest)
        if msg:
            raise ManifestInvalidPathError(
                '<%s> invalid "dest": %s: %s' % (element, dest, msg)
            )

        # |src| is the file we read from or path we point to for symlinks.
        # It is relative to the top of the git project checkout.
        is_linkfile = element == "linkfile"
        msg = cls._CheckLocalPath(
            src, dir_ok=is_linkfile, cwd_dot_ok=is_linkfile
        )
        if msg:
            raise ManifestInvalidPathError(
                '<%s> invalid "src": %s: %s' % (element, src, msg)
            )

    def _ParseCopyFile(self, project, node):
        src = self._reqatt(node, "src")
        dest = self._reqatt(node, "dest")
        if not self.IsMirror:
            # src is project relative;
            # dest is relative to the top of the tree.
            # We only validate paths if we actually plan to process them.
            self._ValidateFilePaths("copyfile", src, dest)
            project.AddCopyFile(src, dest, self.topdir)

    def _ParseLinkFile(self, project, node):
        src = self._reqatt(node, "src")
        dest = self._reqatt(node, "dest")
        if not self.IsMirror:
            # src is project relative;
            # dest is relative to the top of the tree.
            # We only validate paths if we actually plan to process them.
            self._ValidateFilePaths("linkfile", src, dest)
            project.AddLinkFile(src, dest, self.topdir)

    def _ParseAnnotation(self, element, node):
        name = self._reqatt(node, "name")
        value = self._reqatt(node, "value")
        try:
            keep = self._reqatt(node, "keep").lower()
        except ManifestParseError:
            keep = "true"
        if keep != "true" and keep != "false":
            raise ManifestParseError(
                'optional "keep" attribute must be ' '"true" or "false"'
            )
        element.AddAnnotation(name, value, keep)

    def _get_remote(self, node):
        name = node.getAttribute("remote")
        if not name:
            return None

        v = self._remotes.get(name)
        if not v:
            raise ManifestParseError(
                "remote %s not defined in %s" % (name, self.manifestFile)
            )
        return v

    def _reqatt(self, node, attname):
        """
        reads a required attribute from the node.
        """
        v = node.getAttribute(attname)
        if not v:
            raise ManifestParseError(
                "no %s in <%s> within %s"
                % (attname, node.nodeName, self.manifestFile)
            )
        return v

    def projectsDiff(self, manifest):
        """return the projects differences between two manifests.

        The diff will be from self to given manifest.

        """
        fromProjects = self.paths
        toProjects = manifest.paths

        fromKeys = sorted(fromProjects.keys())
        toKeys = sorted(toProjects.keys())

        diff = {
            "added": [],
            "removed": [],
            "missing": [],
            "changed": [],
            "unreachable": [],
        }

        for proj in fromKeys:
            if proj not in toKeys:
                diff["removed"].append(fromProjects[proj])
            elif not fromProjects[proj].Exists:
                diff["missing"].append(toProjects[proj])
                toKeys.remove(proj)
            else:
                fromProj = fromProjects[proj]
                toProj = toProjects[proj]
                try:
                    fromRevId = fromProj.GetCommitRevisionId()
                    toRevId = toProj.GetCommitRevisionId()
                except ManifestInvalidRevisionError:
                    diff["unreachable"].append((fromProj, toProj))
                else:
                    if fromRevId != toRevId:
                        diff["changed"].append((fromProj, toProj))
                toKeys.remove(proj)

        for proj in toKeys:
            diff["added"].append(toProjects[proj])

        return diff


class GitcManifest(XmlManifest):
    """Parser for GitC (git-in-the-cloud) manifests."""

    def _ParseProject(self, node, parent=None):
        """Override _ParseProject and add support for GITC specific attributes."""  # noqa: E501
        return super()._ParseProject(
            node, parent=parent, old_revision=node.getAttribute("old-revision")
        )

    def _output_manifest_project_extras(self, p, e):
        """Output GITC Specific Project attributes"""
        if p.old_revision:
            e.setAttribute("old-revision", str(p.old_revision))


class RepoClient(XmlManifest):
    """Manages a repo client checkout."""

    def __init__(
        self, repodir, manifest_file=None, submanifest_path="", **kwargs
    ):
        """Initialize.

        Args:
            repodir: Path to the .repo/ dir for holding all internal checkout
                state. It must be in the top directory of the repo client
                checkout.
            manifest_file: Full path to the manifest file to parse.  This will
                usually be |repodir|/|MANIFEST_FILE_NAME|.
            submanifest_path: The submanifest root relative to the repo root.
            **kwargs: Additional keyword arguments, passed to XmlManifest.
        """
        self.isGitcClient = False
        submanifest_path = submanifest_path or ""
        if submanifest_path:
            self._CheckLocalPath(submanifest_path)
            prefix = os.path.join(repodir, SUBMANIFEST_DIR, submanifest_path)
        else:
            prefix = repodir

        if os.path.exists(os.path.join(prefix, LOCAL_MANIFEST_NAME)):
            print(
                "error: %s is not supported; put local manifests in `%s` "
                "instead"
                % (
                    LOCAL_MANIFEST_NAME,
                    os.path.join(prefix, LOCAL_MANIFESTS_DIR_NAME),
                ),
                file=sys.stderr,
            )
            sys.exit(1)

        if manifest_file is None:
            manifest_file = os.path.join(prefix, MANIFEST_FILE_NAME)
        local_manifests = os.path.abspath(
            os.path.join(prefix, LOCAL_MANIFESTS_DIR_NAME)
        )
        super().__init__(
            repodir,
            manifest_file,
            local_manifests,
            submanifest_path=submanifest_path,
            **kwargs,
        )

        # TODO: Completely separate manifest logic out of the client.
        self.manifest = self


class GitcClient(RepoClient, GitcManifest):
    """Manages a GitC client checkout."""

    def __init__(self, repodir, gitc_client_name):
        """Initialize the GitcManifest object."""
        self.gitc_client_name = gitc_client_name
        self.gitc_client_dir = os.path.join(
            gitc_utils.get_gitc_manifest_dir(), gitc_client_name
        )

        super().__init__(
            repodir, os.path.join(self.gitc_client_dir, ".manifest")
        )
        self.isGitcClient = True