Merge tag 'v2.33'
Resolved merge conflicts in error.py, project.py, and subcmds/sync.py.
Conflicts were caused by upstream changes to style.
* tag 'v2.33':
sync: Fix how sync times for shared projects are recorded
manifest: add support for revision in include
sync: Display total elapsed fetch time
[SyncAnalysisState] Preserve synctime µs
tests: do not allow underscores in cli options
upload: Add `--no-follow-tags` by default to git push
run_tests: Check flake8
Update abandon to support multiple branches
run_tests: Always check black and check it last
Format codebase with black and check formatting in CQ
Make black with line length 80 repo's code style
docs: update Focal Python version
diff --git a/project.py b/project.py
index df627d4..80169da 100644
--- a/project.py
+++ b/project.py
@@ -32,8 +32,14 @@
from color import Coloring
import fetch
from git_command import GitCommand, git_require
-from git_config import GitConfig, IsId, GetSchemeFromUrl, GetUrlCookieFile, \
- ID_RE, RefSpec
+from git_config import (
+ GitConfig,
+ IsId,
+ GetSchemeFromUrl,
+ GetUrlCookieFile,
+ ID_RE,
+ RefSpec,
+)
import git_superproject
from git_trace2_event_log import EventLog
from error import GitError, UploadError, DownloadError
@@ -48,12 +54,13 @@
class SyncNetworkHalfResult(NamedTuple):
- """Sync_NetworkHalf return value."""
- # True if successful.
- success: bool
- # Did we query the remote? False when optimized_fetch is True and we have the
- # commit already present.
- remote_fetched: bool
+ """Sync_NetworkHalf return value."""
+
+ # True if successful.
+ success: bool
+ # Did we query the remote? False when optimized_fetch is True and we have
+ # the commit already present.
+ remote_fetched: bool
# Maximum sleep time allowed during retries.
@@ -63,3982 +70,4461 @@
# Whether to use alternates. Switching back and forth is *NOT* supported.
# TODO(vapier): Remove knob once behavior is verified.
-_ALTERNATES = os.environ.get('REPO_USE_ALTERNATES') == '1'
+_ALTERNATES = os.environ.get("REPO_USE_ALTERNATES") == "1"
def _lwrite(path, content):
- lock = '%s.lock' % path
+ lock = "%s.lock" % path
- # Maintain Unix line endings on all OS's to match git behavior.
- with open(lock, 'w', newline='\n') as fd:
- fd.write(content)
+ # Maintain Unix line endings on all OS's to match git behavior.
+ with open(lock, "w", newline="\n") as fd:
+ fd.write(content)
- try:
- platform_utils.rename(lock, path)
- except OSError:
- platform_utils.remove(lock)
- raise
+ try:
+ platform_utils.rename(lock, path)
+ except OSError:
+ platform_utils.remove(lock)
+ raise
def _error(fmt, *args):
- msg = fmt % args
- print('error: %s' % msg, file=sys.stderr)
+ msg = fmt % args
+ print("error: %s" % msg, file=sys.stderr)
def _warn(fmt, *args):
- msg = fmt % args
- print('warn: %s' % msg, file=sys.stderr)
+ msg = fmt % args
+ print("warn: %s" % msg, file=sys.stderr)
def not_rev(r):
- return '^' + r
+ return "^" + r
def sq(r):
- return "'" + r.replace("'", "'\''") + "'"
+ return "'" + r.replace("'", "'''") + "'"
_project_hook_list = None
def _ProjectHooks():
- """List the hooks present in the 'hooks' directory.
+ """List the hooks present in the 'hooks' directory.
- These hooks are project hooks and are copied to the '.git/hooks' directory
- of all subprojects.
+ These hooks are project hooks and are copied to the '.git/hooks' directory
+ of all subprojects.
- This function caches the list of hooks (based on the contents of the
- 'repo/hooks' directory) on the first call.
+ This function caches the list of hooks (based on the contents of the
+ 'repo/hooks' directory) on the first call.
- Returns:
- A list of absolute paths to all of the files in the hooks directory.
- """
- global _project_hook_list
- if _project_hook_list is None:
- d = platform_utils.realpath(os.path.abspath(os.path.dirname(__file__)))
- d = os.path.join(d, 'hooks')
- _project_hook_list = [os.path.join(d, x) for x in platform_utils.listdir(d)]
- return _project_hook_list
+ Returns:
+ A list of absolute paths to all of the files in the hooks directory.
+ """
+ global _project_hook_list
+ if _project_hook_list is None:
+ d = platform_utils.realpath(os.path.abspath(os.path.dirname(__file__)))
+ d = os.path.join(d, "hooks")
+ _project_hook_list = [
+ os.path.join(d, x) for x in platform_utils.listdir(d)
+ ]
+ return _project_hook_list
class DownloadedChange(object):
- _commit_cache = None
+ _commit_cache = None
- def __init__(self, project, base, change_id, ps_id, commit):
- self.project = project
- self.base = base
- self.change_id = change_id
- self.ps_id = ps_id
- self.commit = commit
+ def __init__(self, project, base, change_id, ps_id, commit):
+ self.project = project
+ self.base = base
+ self.change_id = change_id
+ self.ps_id = ps_id
+ self.commit = commit
- @property
- def commits(self):
- if self._commit_cache is None:
- self._commit_cache = self.project.bare_git.rev_list('--abbrev=8',
- '--abbrev-commit',
- '--pretty=oneline',
- '--reverse',
- '--date-order',
- not_rev(self.base),
- self.commit,
- '--')
- return self._commit_cache
+ @property
+ def commits(self):
+ if self._commit_cache is None:
+ self._commit_cache = self.project.bare_git.rev_list(
+ "--abbrev=8",
+ "--abbrev-commit",
+ "--pretty=oneline",
+ "--reverse",
+ "--date-order",
+ not_rev(self.base),
+ self.commit,
+ "--",
+ )
+ return self._commit_cache
class ReviewableBranch(object):
- _commit_cache = None
- _base_exists = None
+ _commit_cache = None
+ _base_exists = None
- def __init__(self, project, branch, base):
- self.project = project
- self.branch = branch
- self.base = base
+ def __init__(self, project, branch, base):
+ self.project = project
+ self.branch = branch
+ self.base = base
- @property
- def name(self):
- return self.branch.name
+ @property
+ def name(self):
+ return self.branch.name
- @property
- def commits(self):
- if self._commit_cache is None:
- args = ('--abbrev=8', '--abbrev-commit', '--pretty=oneline', '--reverse',
- '--date-order', not_rev(self.base), R_HEADS + self.name, '--')
- try:
- self._commit_cache = self.project.bare_git.rev_list(*args)
- except GitError:
- # We weren't able to probe the commits for this branch. Was it tracking
- # a branch that no longer exists? If so, return no commits. Otherwise,
- # rethrow the error as we don't know what's going on.
- if self.base_exists:
- raise
+ @property
+ def commits(self):
+ if self._commit_cache is None:
+ args = (
+ "--abbrev=8",
+ "--abbrev-commit",
+ "--pretty=oneline",
+ "--reverse",
+ "--date-order",
+ not_rev(self.base),
+ R_HEADS + self.name,
+ "--",
+ )
+ try:
+ self._commit_cache = self.project.bare_git.rev_list(*args)
+ except GitError:
+ # We weren't able to probe the commits for this branch. Was it
+ # tracking a branch that no longer exists? If so, return no
+ # commits. Otherwise, rethrow the error as we don't know what's
+ # going on.
+ if self.base_exists:
+ raise
- self._commit_cache = []
+ self._commit_cache = []
- return self._commit_cache
+ return self._commit_cache
- @property
- def unabbrev_commits(self):
- r = dict()
- for commit in self.project.bare_git.rev_list(not_rev(self.base),
- R_HEADS + self.name,
- '--'):
- r[commit[0:8]] = commit
- return r
+ @property
+ def unabbrev_commits(self):
+ r = dict()
+ for commit in self.project.bare_git.rev_list(
+ not_rev(self.base), R_HEADS + self.name, "--"
+ ):
+ r[commit[0:8]] = commit
+ return r
- @property
- def date(self):
- return self.project.bare_git.log('--pretty=format:%cd',
- '-n', '1',
- R_HEADS + self.name,
- '--')
+ @property
+ def date(self):
+ return self.project.bare_git.log(
+ "--pretty=format:%cd", "-n", "1", R_HEADS + self.name, "--"
+ )
- @property
- def base_exists(self):
- """Whether the branch we're tracking exists.
+ @property
+ def base_exists(self):
+ """Whether the branch we're tracking exists.
- Normally it should, but sometimes branches we track can get deleted.
- """
- if self._base_exists is None:
- try:
- self.project.bare_git.rev_parse('--verify', not_rev(self.base))
- # If we're still here, the base branch exists.
- self._base_exists = True
- except GitError:
- # If we failed to verify, the base branch doesn't exist.
- self._base_exists = False
+ Normally it should, but sometimes branches we track can get deleted.
+ """
+ if self._base_exists is None:
+ try:
+ self.project.bare_git.rev_parse("--verify", not_rev(self.base))
+ # If we're still here, the base branch exists.
+ self._base_exists = True
+ except GitError:
+ # If we failed to verify, the base branch doesn't exist.
+ self._base_exists = False
- return self._base_exists
+ return self._base_exists
- def UploadForReview(self, people,
- dryrun=False,
- auto_topic=False,
- hashtags=(),
- labels=(),
- private=False,
- notify=None,
- wip=False,
- ready=False,
- dest_branch=None,
- validate_certs=True,
- push_options=None):
- self.project.UploadForReview(branch=self.name,
- people=people,
- dryrun=dryrun,
- auto_topic=auto_topic,
- hashtags=hashtags,
- labels=labels,
- private=private,
- notify=notify,
- wip=wip,
- ready=ready,
- dest_branch=dest_branch,
- validate_certs=validate_certs,
- push_options=push_options)
+ def UploadForReview(
+ self,
+ people,
+ dryrun=False,
+ auto_topic=False,
+ hashtags=(),
+ labels=(),
+ private=False,
+ notify=None,
+ wip=False,
+ ready=False,
+ dest_branch=None,
+ validate_certs=True,
+ push_options=None,
+ ):
+ self.project.UploadForReview(
+ branch=self.name,
+ people=people,
+ dryrun=dryrun,
+ auto_topic=auto_topic,
+ hashtags=hashtags,
+ labels=labels,
+ private=private,
+ notify=notify,
+ wip=wip,
+ ready=ready,
+ dest_branch=dest_branch,
+ validate_certs=validate_certs,
+ push_options=push_options,
+ )
- def GetPublishedRefs(self):
- refs = {}
- output = self.project.bare_git.ls_remote(
- self.branch.remote.SshReviewUrl(self.project.UserEmail),
- 'refs/changes/*')
- for line in output.split('\n'):
- try:
- (sha, ref) = line.split()
- refs[sha] = ref
- except ValueError:
- pass
+ def GetPublishedRefs(self):
+ refs = {}
+ output = self.project.bare_git.ls_remote(
+ self.branch.remote.SshReviewUrl(self.project.UserEmail),
+ "refs/changes/*",
+ )
+ for line in output.split("\n"):
+ try:
+ (sha, ref) = line.split()
+ refs[sha] = ref
+ except ValueError:
+ pass
- return refs
+ return refs
class StatusColoring(Coloring):
+ def __init__(self, config):
+ super().__init__(config, "status")
+ self.project = self.printer("header", attr="bold")
+ self.branch = self.printer("header", attr="bold")
+ self.nobranch = self.printer("nobranch", fg="red")
+ self.important = self.printer("important", fg="red")
- def __init__(self, config):
- super().__init__(config, 'status')
- self.project = self.printer('header', attr='bold')
- self.branch = self.printer('header', attr='bold')
- self.nobranch = self.printer('nobranch', fg='red')
- self.important = self.printer('important', fg='red')
-
- self.added = self.printer('added', fg='green')
- self.changed = self.printer('changed', fg='red')
- self.untracked = self.printer('untracked', fg='red')
+ self.added = self.printer("added", fg="green")
+ self.changed = self.printer("changed", fg="red")
+ self.untracked = self.printer("untracked", fg="red")
class DiffColoring(Coloring):
-
- def __init__(self, config):
- super().__init__(config, 'diff')
- self.project = self.printer('header', attr='bold')
- self.fail = self.printer('fail', fg='red')
+ def __init__(self, config):
+ super().__init__(config, "diff")
+ self.project = self.printer("header", attr="bold")
+ self.fail = self.printer("fail", fg="red")
class Annotation(object):
+ def __init__(self, name, value, keep):
+ self.name = name
+ self.value = value
+ self.keep = keep
- def __init__(self, name, value, keep):
- self.name = name
- self.value = value
- self.keep = keep
+ def __eq__(self, other):
+ if not isinstance(other, Annotation):
+ return False
+ return self.__dict__ == other.__dict__
- def __eq__(self, other):
- if not isinstance(other, Annotation):
- return False
- return self.__dict__ == other.__dict__
-
- def __lt__(self, other):
- # This exists just so that lists of Annotation objects can be sorted, for
- # use in comparisons.
- if not isinstance(other, Annotation):
- raise ValueError('comparison is not between two Annotation objects')
- if self.name == other.name:
- if self.value == other.value:
- return self.keep < other.keep
- return self.value < other.value
- return self.name < other.name
+ def __lt__(self, other):
+ # This exists just so that lists of Annotation objects can be sorted,
+ # for use in comparisons.
+ if not isinstance(other, Annotation):
+ raise ValueError("comparison is not between two Annotation objects")
+ if self.name == other.name:
+ if self.value == other.value:
+ return self.keep < other.keep
+ return self.value < other.value
+ return self.name < other.name
def _SafeExpandPath(base, subpath, skipfinal=False):
- """Make sure |subpath| is completely safe under |base|.
+ """Make sure |subpath| is completely safe under |base|.
- We make sure no intermediate symlinks are traversed, and that the final path
- is not a special file (e.g. not a socket or fifo).
+ We make sure no intermediate symlinks are traversed, and that the final path
+ is not a special file (e.g. not a socket or fifo).
- NB: We rely on a number of paths already being filtered out while parsing the
- manifest. See the validation logic in manifest_xml.py for more details.
- """
- # Split up the path by its components. We can't use os.path.sep exclusively
- # as some platforms (like Windows) will convert / to \ and that bypasses all
- # our constructed logic here. Especially since manifest authors only use
- # / in their paths.
- resep = re.compile(r'[/%s]' % re.escape(os.path.sep))
- components = resep.split(subpath)
- if skipfinal:
- # Whether the caller handles the final component itself.
- finalpart = components.pop()
+ NB: We rely on a number of paths already being filtered out while parsing
+ the manifest. See the validation logic in manifest_xml.py for more details.
+ """
+ # Split up the path by its components. We can't use os.path.sep exclusively
+ # as some platforms (like Windows) will convert / to \ and that bypasses all
+ # our constructed logic here. Especially since manifest authors only use
+ # / in their paths.
+ resep = re.compile(r"[/%s]" % re.escape(os.path.sep))
+ components = resep.split(subpath)
+ if skipfinal:
+ # Whether the caller handles the final component itself.
+ finalpart = components.pop()
- path = base
- for part in components:
- if part in {'.', '..'}:
- raise ManifestInvalidPathError(
- '%s: "%s" not allowed in paths' % (subpath, part))
+ path = base
+ for part in components:
+ if part in {".", ".."}:
+ raise ManifestInvalidPathError(
+ '%s: "%s" not allowed in paths' % (subpath, part)
+ )
- path = os.path.join(path, part)
- if platform_utils.islink(path):
- raise ManifestInvalidPathError(
- '%s: traversing symlinks not allow' % (path,))
+ path = os.path.join(path, part)
+ if platform_utils.islink(path):
+ raise ManifestInvalidPathError(
+ "%s: traversing symlinks not allow" % (path,)
+ )
- if os.path.exists(path):
- if not os.path.isfile(path) and not platform_utils.isdir(path):
- raise ManifestInvalidPathError(
- '%s: only regular files & directories allowed' % (path,))
+ if os.path.exists(path):
+ if not os.path.isfile(path) and not platform_utils.isdir(path):
+ raise ManifestInvalidPathError(
+ "%s: only regular files & directories allowed" % (path,)
+ )
- if skipfinal:
- path = os.path.join(path, finalpart)
+ if skipfinal:
+ path = os.path.join(path, finalpart)
- return path
+ return path
class _CopyFile(object):
- """Container for <copyfile> manifest element."""
+ """Container for <copyfile> manifest element."""
- def __init__(self, git_worktree, src, topdir, dest):
- """Register a <copyfile> request.
+ def __init__(self, git_worktree, src, topdir, dest):
+ """Register a <copyfile> request.
- Args:
- git_worktree: Absolute path to the git project checkout.
- src: Relative path under |git_worktree| of file to read.
- topdir: Absolute path to the top of the repo client checkout.
- dest: Relative path under |topdir| of file to write.
- """
- self.git_worktree = git_worktree
- self.topdir = topdir
- self.src = src
- self.dest = dest
+ Args:
+ git_worktree: Absolute path to the git project checkout.
+ src: Relative path under |git_worktree| of file to read.
+ topdir: Absolute path to the top of the repo client checkout.
+ dest: Relative path under |topdir| of file to write.
+ """
+ self.git_worktree = git_worktree
+ self.topdir = topdir
+ self.src = src
+ self.dest = dest
- def _Copy(self):
- src = _SafeExpandPath(self.git_worktree, self.src)
- dest = _SafeExpandPath(self.topdir, self.dest)
+ def _Copy(self):
+ src = _SafeExpandPath(self.git_worktree, self.src)
+ dest = _SafeExpandPath(self.topdir, self.dest)
- if platform_utils.isdir(src):
- raise ManifestInvalidPathError(
- '%s: copying from directory not supported' % (self.src,))
- if platform_utils.isdir(dest):
- raise ManifestInvalidPathError(
- '%s: copying to directory not allowed' % (self.dest,))
+ if platform_utils.isdir(src):
+ raise ManifestInvalidPathError(
+ "%s: copying from directory not supported" % (self.src,)
+ )
+ if platform_utils.isdir(dest):
+ raise ManifestInvalidPathError(
+ "%s: copying to directory not allowed" % (self.dest,)
+ )
- # copy file if it does not exist or is out of date
- if not os.path.exists(dest) or not filecmp.cmp(src, dest):
- try:
- # remove existing file first, since it might be read-only
- if os.path.exists(dest):
- platform_utils.remove(dest)
- else:
- dest_dir = os.path.dirname(dest)
- if not platform_utils.isdir(dest_dir):
- os.makedirs(dest_dir)
- shutil.copy(src, dest)
- # make the file read-only
- mode = os.stat(dest)[stat.ST_MODE]
- mode = mode & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
- os.chmod(dest, mode)
- except IOError:
- _error('Cannot copy file %s to %s', src, dest)
+ # Copy file if it does not exist or is out of date.
+ if not os.path.exists(dest) or not filecmp.cmp(src, dest):
+ try:
+ # Remove existing file first, since it might be read-only.
+ if os.path.exists(dest):
+ platform_utils.remove(dest)
+ else:
+ dest_dir = os.path.dirname(dest)
+ if not platform_utils.isdir(dest_dir):
+ os.makedirs(dest_dir)
+ shutil.copy(src, dest)
+ # Make the file read-only.
+ mode = os.stat(dest)[stat.ST_MODE]
+ mode = mode & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
+ os.chmod(dest, mode)
+ except IOError:
+ _error("Cannot copy file %s to %s", src, dest)
class _LinkFile(object):
- """Container for <linkfile> manifest element."""
+ """Container for <linkfile> manifest element."""
- def __init__(self, git_worktree, src, topdir, dest):
- """Register a <linkfile> request.
+ def __init__(self, git_worktree, src, topdir, dest):
+ """Register a <linkfile> request.
- Args:
- git_worktree: Absolute path to the git project checkout.
- src: Target of symlink relative to path under |git_worktree|.
- topdir: Absolute path to the top of the repo client checkout.
- dest: Relative path under |topdir| of symlink to create.
- """
- self.git_worktree = git_worktree
- self.topdir = topdir
- self.src = src
- self.dest = dest
+ Args:
+ git_worktree: Absolute path to the git project checkout.
+ src: Target of symlink relative to path under |git_worktree|.
+ topdir: Absolute path to the top of the repo client checkout.
+ dest: Relative path under |topdir| of symlink to create.
+ """
+ self.git_worktree = git_worktree
+ self.topdir = topdir
+ self.src = src
+ self.dest = dest
- def __linkIt(self, relSrc, absDest):
- # link file if it does not exist or is out of date
- if not platform_utils.islink(absDest) or (platform_utils.readlink(absDest) != relSrc):
- try:
- # remove existing file first, since it might be read-only
- if os.path.lexists(absDest):
- platform_utils.remove(absDest)
+ def __linkIt(self, relSrc, absDest):
+ # Link file if it does not exist or is out of date.
+ if not platform_utils.islink(absDest) or (
+ platform_utils.readlink(absDest) != relSrc
+ ):
+ try:
+ # Remove existing file first, since it might be read-only.
+ if os.path.lexists(absDest):
+ platform_utils.remove(absDest)
+ else:
+ dest_dir = os.path.dirname(absDest)
+ if not platform_utils.isdir(dest_dir):
+ os.makedirs(dest_dir)
+ platform_utils.symlink(relSrc, absDest)
+ except IOError:
+ _error("Cannot link file %s to %s", relSrc, absDest)
+
+ def _Link(self):
+ """Link the self.src & self.dest paths.
+
+ Handles wild cards on the src linking all of the files in the source in
+ to the destination directory.
+ """
+ # Some people use src="." to create stable links to projects. Let's
+ # allow that but reject all other uses of "." to keep things simple.
+ if self.src == ".":
+ src = self.git_worktree
else:
- dest_dir = os.path.dirname(absDest)
- if not platform_utils.isdir(dest_dir):
- os.makedirs(dest_dir)
- platform_utils.symlink(relSrc, absDest)
- except IOError:
- _error('Cannot link file %s to %s', relSrc, absDest)
+ src = _SafeExpandPath(self.git_worktree, self.src)
- def _Link(self):
- """Link the self.src & self.dest paths.
+ if not glob.has_magic(src):
+ # Entity does not contain a wild card so just a simple one to one
+ # link operation.
+ dest = _SafeExpandPath(self.topdir, self.dest, skipfinal=True)
+ # dest & src are absolute paths at this point. Make sure the target
+ # of the symlink is relative in the context of the repo client
+ # checkout.
+ relpath = os.path.relpath(src, os.path.dirname(dest))
+ self.__linkIt(relpath, dest)
+ else:
+ dest = _SafeExpandPath(self.topdir, self.dest)
+ # Entity contains a wild card.
+ if os.path.exists(dest) and not platform_utils.isdir(dest):
+ _error(
+ "Link error: src with wildcard, %s must be a directory",
+ dest,
+ )
+ else:
+ for absSrcFile in glob.glob(src):
+ # Create a releative path from source dir to destination
+ # dir.
+ absSrcDir = os.path.dirname(absSrcFile)
+ relSrcDir = os.path.relpath(absSrcDir, dest)
- Handles wild cards on the src linking all of the files in the source in to
- the destination directory.
- """
- # Some people use src="." to create stable links to projects. Lets allow
- # that but reject all other uses of "." to keep things simple.
- if self.src == '.':
- src = self.git_worktree
- else:
- src = _SafeExpandPath(self.git_worktree, self.src)
+ # Get the source file name.
+ srcFile = os.path.basename(absSrcFile)
- if not glob.has_magic(src):
- # Entity does not contain a wild card so just a simple one to one link operation.
- dest = _SafeExpandPath(self.topdir, self.dest, skipfinal=True)
- # dest & src are absolute paths at this point. Make sure the target of
- # the symlink is relative in the context of the repo client checkout.
- relpath = os.path.relpath(src, os.path.dirname(dest))
- self.__linkIt(relpath, dest)
- else:
- dest = _SafeExpandPath(self.topdir, self.dest)
- # Entity contains a wild card.
- if os.path.exists(dest) and not platform_utils.isdir(dest):
- _error('Link error: src with wildcard, %s must be a directory', dest)
- else:
- for absSrcFile in glob.glob(src):
- # Create a releative path from source dir to destination dir
- absSrcDir = os.path.dirname(absSrcFile)
- relSrcDir = os.path.relpath(absSrcDir, dest)
-
- # Get the source file name
- srcFile = os.path.basename(absSrcFile)
-
- # Now form the final full paths to srcFile. They will be
- # absolute for the desintaiton and relative for the srouce.
- absDest = os.path.join(dest, srcFile)
- relSrc = os.path.join(relSrcDir, srcFile)
- self.__linkIt(relSrc, absDest)
+ # Now form the final full paths to srcFile. They will be
+ # absolute for the desintaiton and relative for the source.
+ absDest = os.path.join(dest, srcFile)
+ relSrc = os.path.join(relSrcDir, srcFile)
+ self.__linkIt(relSrc, absDest)
class RemoteSpec(object):
-
- def __init__(self,
- name,
- url=None,
- pushUrl=None,
- review=None,
- revision=None,
- orig_name=None,
- fetchUrl=None):
- self.name = name
- self.url = url
- self.pushUrl = pushUrl
- self.review = review
- self.revision = revision
- self.orig_name = orig_name
- self.fetchUrl = fetchUrl
+ def __init__(
+ self,
+ name,
+ url=None,
+ pushUrl=None,
+ review=None,
+ revision=None,
+ orig_name=None,
+ fetchUrl=None,
+ ):
+ self.name = name
+ self.url = url
+ self.pushUrl = pushUrl
+ self.review = review
+ self.revision = revision
+ self.orig_name = orig_name
+ self.fetchUrl = fetchUrl
class Project(object):
- # These objects can be shared between several working trees.
- @property
- def shareable_dirs(self):
- """Return the shareable directories"""
- if self.UseAlternates:
- return ['hooks', 'rr-cache']
- else:
- return ['hooks', 'objects', 'rr-cache']
+ # These objects can be shared between several working trees.
+ @property
+ def shareable_dirs(self):
+ """Return the shareable directories"""
+ if self.UseAlternates:
+ return ["hooks", "rr-cache"]
+ else:
+ return ["hooks", "objects", "rr-cache"]
- def __init__(self,
- manifest,
- name,
- remote,
- gitdir,
- objdir,
- worktree,
- relpath,
- revisionExpr,
- revisionId,
- rebase=True,
- groups=None,
- sync_c=False,
- sync_s=False,
- sync_tags=True,
- clone_depth=None,
- upstream=None,
- parent=None,
- use_git_worktrees=False,
- is_derived=False,
- dest_branch=None,
- optimized_fetch=False,
- retry_fetches=0,
- old_revision=None):
- """Init a Project object.
-
- Args:
- manifest: The XmlManifest object.
- name: The `name` attribute of manifest.xml's project element.
- remote: RemoteSpec object specifying its remote's properties.
- gitdir: Absolute path of git directory.
- objdir: Absolute path of directory to store git objects.
- worktree: Absolute path of git working tree.
- relpath: Relative path of git working tree to repo's top directory.
- revisionExpr: The `revision` attribute of manifest.xml's project element.
- revisionId: git commit id for checking out.
- rebase: The `rebase` attribute of manifest.xml's project element.
- groups: The `groups` attribute of manifest.xml's project element.
- sync_c: The `sync-c` attribute of manifest.xml's project element.
- sync_s: The `sync-s` attribute of manifest.xml's project element.
- sync_tags: The `sync-tags` attribute of manifest.xml's project element.
- upstream: The `upstream` attribute of manifest.xml's project element.
- parent: The parent Project object.
- use_git_worktrees: Whether to use `git worktree` for this project.
- is_derived: False if the project was explicitly defined in the manifest;
- True if the project is a discovered submodule.
- dest_branch: The branch to which to push changes for review by default.
- optimized_fetch: If True, when a project is set to a sha1 revision, only
- fetch from the remote if the sha1 is not present locally.
- retry_fetches: Retry remote fetches n times upon receiving transient error
- with exponential backoff and jitter.
- old_revision: saved git commit id for open GITC projects.
- """
- self.client = self.manifest = manifest
- self.name = name
- self.remote = remote
- self.UpdatePaths(relpath, worktree, gitdir, objdir)
- self.SetRevision(revisionExpr, revisionId=revisionId)
-
- self.rebase = rebase
- self.groups = groups
- self.sync_c = sync_c
- self.sync_s = sync_s
- self.sync_tags = sync_tags
- self.clone_depth = clone_depth
- self.upstream = upstream
- self.parent = parent
- # NB: Do not use this setting in __init__ to change behavior so that the
- # manifest.git checkout can inspect & change it after instantiating. See
- # the XmlManifest init code for more info.
- self.use_git_worktrees = use_git_worktrees
- self.is_derived = is_derived
- self.optimized_fetch = optimized_fetch
- self.retry_fetches = max(0, retry_fetches)
- self.subprojects = []
-
- self.snapshots = {}
- self.copyfiles = []
- self.linkfiles = []
- self.annotations = []
- self.dest_branch = dest_branch
- self.old_revision = old_revision
-
- # This will be filled in if a project is later identified to be the
- # project containing repo hooks.
- self.enabled_repo_hooks = []
-
- def RelPath(self, local=True):
- """Return the path for the project relative to a manifest.
-
- Args:
- local: a boolean, if True, the path is relative to the local
- (sub)manifest. If false, the path is relative to the
- outermost manifest.
- """
- if local:
- return self.relpath
- return os.path.join(self.manifest.path_prefix, self.relpath)
-
- def SetRevision(self, revisionExpr, revisionId=None):
- """Set revisionId based on revision expression and id"""
- self.revisionExpr = revisionExpr
- if revisionId is None and revisionExpr and IsId(revisionExpr):
- self.revisionId = self.revisionExpr
- else:
- self.revisionId = revisionId
-
- def UpdatePaths(self, relpath, worktree, gitdir, objdir):
- """Update paths used by this project"""
- self.gitdir = gitdir.replace('\\', '/')
- self.objdir = objdir.replace('\\', '/')
- if worktree:
- self.worktree = os.path.normpath(worktree).replace('\\', '/')
- else:
- self.worktree = None
- self.relpath = relpath
-
- self.config = GitConfig.ForRepository(gitdir=self.gitdir,
- defaults=self.manifest.globalConfig)
-
- if self.worktree:
- self.work_git = self._GitGetByExec(self, bare=False, gitdir=self.gitdir)
- else:
- self.work_git = None
- self.bare_git = self._GitGetByExec(self, bare=True, gitdir=self.gitdir)
- self.bare_ref = GitRefs(self.gitdir)
- self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=self.objdir)
-
- @property
- def UseAlternates(self):
- """Whether git alternates are in use.
-
- This will be removed once migration to alternates is complete.
- """
- return _ALTERNATES or self.manifest.is_multimanifest
-
- @property
- def Derived(self):
- return self.is_derived
-
- @property
- def Exists(self):
- return platform_utils.isdir(self.gitdir) and platform_utils.isdir(self.objdir)
-
- @property
- def CurrentBranch(self):
- """Obtain the name of the currently checked out branch.
-
- The branch name omits the 'refs/heads/' prefix.
- None is returned if the project is on a detached HEAD, or if the work_git is
- otheriwse inaccessible (e.g. an incomplete sync).
- """
- try:
- b = self.work_git.GetHead()
- except NoManifestException:
- # If the local checkout is in a bad state, don't barf. Let the callers
- # process this like the head is unreadable.
- return None
- if b.startswith(R_HEADS):
- return b[len(R_HEADS):]
- return None
-
- def IsRebaseInProgress(self):
- return (os.path.exists(self.work_git.GetDotgitPath('rebase-apply')) or
- os.path.exists(self.work_git.GetDotgitPath('rebase-merge')) or
- os.path.exists(os.path.join(self.worktree, '.dotest')))
-
- def IsDirty(self, consider_untracked=True):
- """Is the working directory modified in some way?
- """
- self.work_git.update_index('-q',
- '--unmerged',
- '--ignore-missing',
- '--refresh')
- if self.work_git.DiffZ('diff-index', '-M', '--cached', HEAD):
- return True
- if self.work_git.DiffZ('diff-files'):
- return True
- if consider_untracked and self.UntrackedFiles():
- return True
- return False
-
- _userident_name = None
- _userident_email = None
-
- @property
- def UserName(self):
- """Obtain the user's personal name.
- """
- if self._userident_name is None:
- self._LoadUserIdentity()
- return self._userident_name
-
- @property
- def UserEmail(self):
- """Obtain the user's email address. This is very likely
- to be their Gerrit login.
- """
- if self._userident_email is None:
- self._LoadUserIdentity()
- return self._userident_email
-
- def _LoadUserIdentity(self):
- u = self.bare_git.var('GIT_COMMITTER_IDENT')
- m = re.compile("^(.*) <([^>]*)> ").match(u)
- if m:
- self._userident_name = m.group(1)
- self._userident_email = m.group(2)
- else:
- self._userident_name = ''
- self._userident_email = ''
-
- def GetRemote(self, name=None):
- """Get the configuration for a single remote.
-
- Defaults to the current project's remote.
- """
- if name is None:
- name = self.remote.name
- return self.config.GetRemote(name)
-
- def GetBranch(self, name):
- """Get the configuration for a single branch.
- """
- return self.config.GetBranch(name)
-
- def GetBranches(self):
- """Get all existing local branches.
- """
- current = self.CurrentBranch
- all_refs = self._allrefs
- heads = {}
-
- for name, ref_id in all_refs.items():
- if name.startswith(R_HEADS):
- name = name[len(R_HEADS):]
- b = self.GetBranch(name)
- b.current = name == current
- b.published = None
- b.revision = ref_id
- heads[name] = b
-
- for name, ref_id in all_refs.items():
- if name.startswith(R_PUB):
- name = name[len(R_PUB):]
- b = heads.get(name)
- if b:
- b.published = ref_id
-
- return heads
-
- def MatchesGroups(self, manifest_groups):
- """Returns true if the manifest groups specified at init should cause
- this project to be synced.
- Prefixing a manifest group with "-" inverts the meaning of a group.
- All projects are implicitly labelled with "all".
-
- labels are resolved in order. In the example case of
- project_groups: "all,group1,group2"
- manifest_groups: "-group1,group2"
- the project will be matched.
-
- The special manifest group "default" will match any project that
- does not have the special project group "notdefault"
- """
- default_groups = self.manifest.default_groups or ['default']
- expanded_manifest_groups = manifest_groups or default_groups
- expanded_project_groups = ['all'] + (self.groups or [])
- if 'notdefault' not in expanded_project_groups:
- expanded_project_groups += ['default']
-
- matched = False
- for group in expanded_manifest_groups:
- if group.startswith('-') and group[1:] in expanded_project_groups:
- matched = False
- elif group in expanded_project_groups:
- matched = True
-
- return matched
-
-# Status Display ##
- def UncommitedFiles(self, get_all=True):
- """Returns a list of strings, uncommitted files in the git tree.
-
- Args:
- get_all: a boolean, if True - get information about all different
- uncommitted files. If False - return as soon as any kind of
- uncommitted files is detected.
- """
- details = []
- self.work_git.update_index('-q',
- '--unmerged',
- '--ignore-missing',
- '--refresh')
- if self.IsRebaseInProgress():
- details.append("rebase in progress")
- if not get_all:
- return details
-
- changes = self.work_git.DiffZ('diff-index', '--cached', HEAD).keys()
- if changes:
- details.extend(changes)
- if not get_all:
- return details
-
- changes = self.work_git.DiffZ('diff-files').keys()
- if changes:
- details.extend(changes)
- if not get_all:
- return details
-
- changes = self.UntrackedFiles()
- if changes:
- details.extend(changes)
-
- return details
-
- def UntrackedFiles(self):
- """Returns a list of strings, untracked files in the git tree."""
- return self.work_git.LsOthers()
-
- def HasChanges(self):
- """Returns true if there are uncommitted changes.
- """
- return bool(self.UncommitedFiles(get_all=False))
-
- def PrintWorkTreeStatus(self, output_redir=None, quiet=False, local=False):
- """Prints the status of the repository to stdout.
-
- Args:
- output_redir: If specified, redirect the output to this object.
- quiet: If True then only print the project name. Do not print
- the modified files, branch name, etc.
- local: a boolean, if True, the path is relative to the local
- (sub)manifest. If false, the path is relative to the
- outermost manifest.
- """
- if not platform_utils.isdir(self.worktree):
- if output_redir is None:
- output_redir = sys.stdout
- print(file=output_redir)
- print('project %s/' % self.RelPath(local), file=output_redir)
- print(' missing (run "repo sync")', file=output_redir)
- return
-
- self.work_git.update_index('-q',
- '--unmerged',
- '--ignore-missing',
- '--refresh')
- rb = self.IsRebaseInProgress()
- di = self.work_git.DiffZ('diff-index', '-M', '--cached', HEAD)
- df = self.work_git.DiffZ('diff-files')
- do = self.work_git.LsOthers()
- if not rb and not di and not df and not do and not self.CurrentBranch:
- return 'CLEAN'
-
- out = StatusColoring(self.config)
- if output_redir is not None:
- out.redirect(output_redir)
- out.project('project %-40s', self.RelPath(local) + '/ ')
-
- if quiet:
- out.nl()
- return 'DIRTY'
-
- branch = self.CurrentBranch
- if branch is None:
- out.nobranch('(*** NO BRANCH ***)')
- else:
- out.branch('branch %s', branch)
- out.nl()
-
- if rb:
- out.important('prior sync failed; rebase still in progress')
- out.nl()
-
- paths = list()
- paths.extend(di.keys())
- paths.extend(df.keys())
- paths.extend(do)
-
- for p in sorted(set(paths)):
- try:
- i = di[p]
- except KeyError:
- i = None
-
- try:
- f = df[p]
- except KeyError:
- f = None
-
- if i:
- i_status = i.status.upper()
- else:
- i_status = '-'
-
- if f:
- f_status = f.status.lower()
- else:
- f_status = '-'
-
- if i and i.src_path:
- line = ' %s%s\t%s => %s (%s%%)' % (i_status, f_status,
- i.src_path, p, i.level)
- else:
- line = ' %s%s\t%s' % (i_status, f_status, p)
-
- if i and not f:
- out.added('%s', line)
- elif (i and f) or (not i and f):
- out.changed('%s', line)
- elif not i and not f:
- out.untracked('%s', line)
- else:
- out.write('%s', line)
- out.nl()
-
- return 'DIRTY'
-
- def PrintWorkTreeDiff(self, absolute_paths=False, output_redir=None,
- local=False):
- """Prints the status of the repository to stdout.
- """
- out = DiffColoring(self.config)
- if output_redir:
- out.redirect(output_redir)
- cmd = ['diff']
- if out.is_on:
- cmd.append('--color')
- cmd.append(HEAD)
- if absolute_paths:
- cmd.append('--src-prefix=a/%s/' % self.RelPath(local))
- cmd.append('--dst-prefix=b/%s/' % self.RelPath(local))
- cmd.append('--')
- try:
- p = GitCommand(self,
- cmd,
- capture_stdout=True,
- capture_stderr=True)
- p.Wait()
- except GitError as e:
- out.nl()
- out.project('project %s/' % self.RelPath(local))
- out.nl()
- out.fail('%s', str(e))
- out.nl()
- return False
- if p.stdout:
- out.nl()
- out.project('project %s/' % self.RelPath(local))
- out.nl()
- out.write('%s', p.stdout)
- return p.Wait() == 0
-
-# Publish / Upload ##
- def WasPublished(self, branch, all_refs=None):
- """Was the branch published (uploaded) for code review?
- If so, returns the SHA-1 hash of the last published
- state for the branch.
- """
- key = R_PUB + branch
- if all_refs is None:
- try:
- return self.bare_git.rev_parse(key)
- except GitError:
- return None
- else:
- try:
- return all_refs[key]
- except KeyError:
- return None
-
- def CleanPublishedCache(self, all_refs=None):
- """Prunes any stale published refs.
- """
- if all_refs is None:
- all_refs = self._allrefs
- heads = set()
- canrm = {}
- for name, ref_id in all_refs.items():
- if name.startswith(R_HEADS):
- heads.add(name)
- elif name.startswith(R_PUB):
- canrm[name] = ref_id
-
- for name, ref_id in canrm.items():
- n = name[len(R_PUB):]
- if R_HEADS + n not in heads:
- self.bare_git.DeleteRef(name, ref_id)
-
- def GetUploadableBranches(self, selected_branch=None):
- """List any branches which can be uploaded for review.
- """
- heads = {}
- pubed = {}
-
- for name, ref_id in self._allrefs.items():
- if name.startswith(R_HEADS):
- heads[name[len(R_HEADS):]] = ref_id
- elif name.startswith(R_PUB):
- pubed[name[len(R_PUB):]] = ref_id
-
- ready = []
- for branch, ref_id in heads.items():
- if branch in pubed and pubed[branch] == ref_id:
- continue
- if selected_branch and branch != selected_branch:
- continue
-
- rb = self.GetUploadableBranch(branch)
- if rb:
- ready.append(rb)
- return ready
-
- def GetUploadableBranch(self, branch_name):
- """Get a single uploadable branch, or None.
- """
- branch = self.GetBranch(branch_name)
- base = branch.LocalMerge
- if branch.LocalMerge:
- rb = ReviewableBranch(self, branch, base)
- if rb.commits:
- return rb
- return None
-
- def UploadForReview(self, branch=None,
- people=([], []),
- dryrun=False,
- auto_topic=False,
- hashtags=(),
- labels=(),
- private=False,
- notify=None,
- wip=False,
- ready=False,
- dest_branch=None,
- validate_certs=True,
- push_options=None):
- """Uploads the named branch for code review.
- """
- if branch is None:
- branch = self.CurrentBranch
- if branch is None:
- raise GitError('not currently on a branch')
-
- branch = self.GetBranch(branch)
- if not branch.LocalMerge:
- raise GitError('branch %s does not track a remote' % branch.name)
- if not branch.remote.review:
- raise GitError('remote %s has no review url' % branch.remote.name)
-
- # Basic validity check on label syntax.
- for label in labels:
- if not re.match(r'^.+[+-][0-9]+$', label):
- raise UploadError(
- f'invalid label syntax "{label}": labels use forms like '
- 'CodeReview+1 or Verified-1')
-
- if dest_branch is None:
- dest_branch = self.dest_branch
- if dest_branch is None:
- dest_branch = branch.merge
- if not dest_branch.startswith(R_HEADS):
- dest_branch = R_HEADS + dest_branch
-
- if not branch.remote.projectname:
- branch.remote.projectname = self.name
- branch.remote.Save()
-
- url = branch.remote.ReviewUrl(self.UserEmail, validate_certs)
- if url is None:
- raise UploadError('review not configured')
- cmd = ['push']
- if dryrun:
- cmd.append('-n')
-
- if url.startswith('ssh://'):
- cmd.append('--receive-pack=gerrit receive-pack')
-
- for push_option in (push_options or []):
- cmd.append('-o')
- cmd.append(push_option)
-
- cmd.append(url)
-
- if dest_branch.startswith(R_HEADS):
- dest_branch = dest_branch[len(R_HEADS):]
-
- ref_spec = '%s:refs/for/%s' % (R_HEADS + branch.name, dest_branch)
- opts = []
- if auto_topic:
- opts += ['topic=' + branch.name]
- opts += ['t=%s' % p for p in hashtags]
- # NB: No need to encode labels as they've been validated above.
- opts += ['l=%s' % p for p in labels]
-
- opts += ['r=%s' % p for p in people[0]]
- opts += ['cc=%s' % p for p in people[1]]
- if notify:
- opts += ['notify=' + notify]
- if private:
- opts += ['private']
- if wip:
- opts += ['wip']
- if ready:
- opts += ['ready']
- if opts:
- ref_spec = ref_spec + '%' + ','.join(opts)
- cmd.append(ref_spec)
-
- if GitCommand(self, cmd, bare=True).Wait() != 0:
- raise UploadError('Upload failed')
-
- if not dryrun:
- msg = "posted to %s for %s" % (branch.remote.review, dest_branch)
- self.bare_git.UpdateRef(R_PUB + branch.name,
- R_HEADS + branch.name,
- message=msg)
-
-# Sync ##
- def _ExtractArchive(self, tarpath, path=None):
- """Extract the given tar on its current location
-
- Args:
- - tarpath: The path to the actual tar file
-
- """
- try:
- with tarfile.open(tarpath, 'r') as tar:
- tar.extractall(path=path)
- return True
- except (IOError, tarfile.TarError) as e:
- _error("Cannot extract archive %s: %s", tarpath, str(e))
- return False
-
- def CachePopulate(self, cache_dir, url):
- """Populate cache in the cache_dir.
-
- Args:
- cache_dir: Directory to cache git files from Google Storage.
- url: Git url of current repository.
-
- Raises:
- CacheApplyError if it fails to populate the git cache.
- """
- cmd = ['cache', 'populate', '--ignore_locks', '-v',
- '--cache-dir', cache_dir, url]
-
- if GitCommand(self, cmd, cwd=cache_dir).Wait() != 0:
- raise CacheApplyError('Failed to populate cache. cache_dir: %s '
- 'url: %s' % (cache_dir, url))
-
- def CacheExists(self, cache_dir, url):
- """Check the existence of the cache files.
-
- Args:
- cache_dir: Directory to cache git files.
- url: Git url of current repository.
-
- Raises:
- CacheApplyError if the cache files do not exist.
- """
- cmd = ['cache', 'exists', '--quiet', '--cache-dir', cache_dir, url]
-
- exist = GitCommand(self, cmd, cwd=self.gitdir, capture_stdout=True)
- if exist.Wait() != 0:
- raise CacheApplyError('Failed to execute git cache exists cmd. '
- 'cache_dir: %s url: %s' % (cache_dir, url))
-
- if not exist.stdout or not exist.stdout.strip():
- raise CacheApplyError('Failed to find cache. cache_dir: %s '
- 'url: %s' % (cache_dir, url))
- return exist.stdout.strip()
-
- def CacheApply(self, cache_dir):
- """Apply git cache files populated from Google Storage buckets.
-
- Args:
- cache_dir: Directory to cache git files.
-
- Raises:
- CacheApplyError if it fails to apply git caches.
- """
- remote = self.GetRemote(self.remote.name)
-
- self.CachePopulate(cache_dir, remote.url)
-
- mirror_dir = self.CacheExists(cache_dir, remote.url)
-
- refspec = RefSpec(True, 'refs/heads/*',
- 'refs/remotes/%s/*' % remote.name)
-
- fetch_cache_cmd = ['fetch', mirror_dir, str(refspec)]
- if GitCommand(self, fetch_cache_cmd, self.gitdir).Wait() != 0:
- raise CacheApplyError('Failed to fetch refs %s from %s' %
- (mirror_dir, str(refspec)))
-
- def Sync_NetworkHalf(self,
- quiet=False,
- verbose=False,
- output_redir=None,
- is_new=None,
- current_branch_only=None,
- force_sync=False,
- clone_bundle=True,
- tags=None,
- archive=False,
- optimized_fetch=False,
- retry_fetches=0,
- prune=False,
- submodules=False,
- cache_dir=None,
- ssh_proxy=None,
- clone_filter=None,
- partial_clone_exclude=set()):
- """Perform only the network IO portion of the sync process.
- Local working directory/branch state is not affected.
- """
- if archive and not isinstance(self, MetaProject):
- if self.remote.url.startswith(('http://', 'https://')):
- _error("%s: Cannot fetch archives from http/https remotes.", self.name)
- return SyncNetworkHalfResult(False, False)
-
- name = self.relpath.replace('\\', '/')
- name = name.replace('/', '_')
- tarpath = '%s.tar' % name
- topdir = self.manifest.topdir
-
- try:
- self._FetchArchive(tarpath, cwd=topdir)
- except GitError as e:
- _error('%s', e)
- return SyncNetworkHalfResult(False, False)
-
- # From now on, we only need absolute tarpath
- tarpath = os.path.join(topdir, tarpath)
-
- if not self._ExtractArchive(tarpath, path=topdir):
- return SyncNetworkHalfResult(False, True)
- try:
- platform_utils.remove(tarpath)
- except OSError as e:
- _warn("Cannot remove archive %s: %s", tarpath, str(e))
- self._CopyAndLinkFiles()
- return SyncNetworkHalfResult(True, True)
-
- # If the shared object dir already exists, don't try to rebootstrap with a
- # clone bundle download. We should have the majority of objects already.
- if clone_bundle and os.path.exists(self.objdir):
- clone_bundle = False
-
- if self.name in partial_clone_exclude:
- clone_bundle = True
- clone_filter = None
-
- if is_new is None:
- is_new = not self.Exists
- if is_new:
- self._InitGitDir(force_sync=force_sync, quiet=quiet)
- else:
- self._UpdateHooks(quiet=quiet)
- self._InitRemote()
-
- if self.UseAlternates:
- # If gitdir/objects is a symlink, migrate it from the old layout.
- gitdir_objects = os.path.join(self.gitdir, 'objects')
- if platform_utils.islink(gitdir_objects):
- platform_utils.remove(gitdir_objects, missing_ok=True)
- gitdir_alt = os.path.join(self.gitdir, 'objects/info/alternates')
- if not os.path.exists(gitdir_alt):
- os.makedirs(os.path.dirname(gitdir_alt), exist_ok=True)
- _lwrite(gitdir_alt, os.path.join(
- os.path.relpath(self.objdir, gitdir_objects), 'objects') + '\n')
-
- if is_new:
- alt = os.path.join(self.objdir, 'objects/info/alternates')
- try:
- with open(alt) as fd:
- # This works for both absolute and relative alternate directories.
- alt_dir = os.path.join(self.objdir, 'objects', fd.readline().rstrip())
- except IOError:
- alt_dir = None
- else:
- alt_dir = None
-
- applied_cache = False
- # If cache_dir is provided, and it's a new repository without
- # alternative_dir, bootstrap this project repo with the git
- # cache files.
- if cache_dir is not None and is_new and alt_dir is None:
- try:
- self.CacheApply(cache_dir)
- applied_cache = True
- is_new = False
- except CacheApplyError as e:
- _error('Could not apply git cache: %s', e)
- _error('Please check if you have the right GS credentials.')
- _error('Please check if the cache files exist in GS.')
-
- if (clone_bundle
- and not applied_cache
- and alt_dir is None
- and self._ApplyCloneBundle(initial=is_new, quiet=quiet, verbose=verbose)):
- is_new = False
-
- if current_branch_only is None:
- if self.sync_c:
- current_branch_only = True
- elif not self.manifest._loaded:
- # Manifest cannot check defaults until it syncs.
- current_branch_only = False
- elif self.manifest.default.sync_c:
- current_branch_only = True
-
- if tags is None:
- tags = self.sync_tags
-
- if self.clone_depth:
- depth = self.clone_depth
- else:
- depth = self.manifest.manifestProject.depth
-
- # See if we can skip the network fetch entirely.
- remote_fetched = False
- if not (optimized_fetch and
- (ID_RE.match(self.revisionExpr) and
- self._CheckForImmutableRevision())):
- remote_fetched = True
- if not self._RemoteFetch(
- initial=is_new,
- quiet=quiet, verbose=verbose, output_redir=output_redir,
- alt_dir=alt_dir, current_branch_only=current_branch_only,
- tags=tags, prune=prune, depth=depth,
- submodules=submodules, force_sync=force_sync,
- ssh_proxy=ssh_proxy,
- clone_filter=clone_filter, retry_fetches=retry_fetches):
- return SyncNetworkHalfResult(False, remote_fetched)
-
- mp = self.manifest.manifestProject
- dissociate = mp.dissociate
- if dissociate:
- alternates_file = os.path.join(self.objdir, 'objects/info/alternates')
- if os.path.exists(alternates_file):
- cmd = ['repack', '-a', '-d']
- p = GitCommand(self, cmd, bare=True, capture_stdout=bool(output_redir),
- merge_output=bool(output_redir))
- if p.stdout and output_redir:
- output_redir.write(p.stdout)
- if p.Wait() != 0:
- return SyncNetworkHalfResult(False, remote_fetched)
- platform_utils.remove(alternates_file)
-
- if self.worktree:
- self._InitMRef()
- else:
- self._InitMirrorHead()
- platform_utils.remove(os.path.join(self.gitdir, 'FETCH_HEAD'),
- missing_ok=True)
- return SyncNetworkHalfResult(True, remote_fetched)
-
- def PostRepoUpgrade(self):
- self._InitHooks()
-
- def _CopyAndLinkFiles(self):
- if self.client.isGitcClient:
- return
- for copyfile in self.copyfiles:
- copyfile._Copy()
- for linkfile in self.linkfiles:
- linkfile._Link()
-
- def GetCommitRevisionId(self):
- """Get revisionId of a commit.
-
- Use this method instead of GetRevisionId to get the id of the commit rather
- than the id of the current git object (for example, a tag)
-
- """
- if not self.revisionExpr.startswith(R_TAGS):
- return self.GetRevisionId(self._allrefs)
-
- try:
- return self.bare_git.rev_list(self.revisionExpr, '-1')[0]
- except GitError:
- raise ManifestInvalidRevisionError('revision %s in %s not found' %
- (self.revisionExpr, self.name))
-
- def GetRevisionId(self, all_refs=None):
- if self.revisionId:
- return self.revisionId
-
- rem = self.GetRemote()
- rev = rem.ToLocal(self.revisionExpr)
-
- if all_refs is not None and rev in all_refs:
- return all_refs[rev]
-
- try:
- return self.bare_git.rev_parse('--verify', '%s^0' % rev)
- except GitError:
- raise ManifestInvalidRevisionError('revision %s in %s not found' %
- (self.revisionExpr, self.name))
-
- def SetRevisionId(self, revisionId):
- if self.revisionExpr:
- self.upstream = self.revisionExpr
-
- self.revisionId = revisionId
-
- def Sync_LocalHalf(self, syncbuf, force_sync=False, submodules=False):
- """Perform only the local IO portion of the sync process.
- Network access is not required.
- """
- if not os.path.exists(self.gitdir):
- syncbuf.fail(self,
- 'Cannot checkout %s due to missing network sync; Run '
- '`repo sync -n %s` first.' %
- (self.name, self.name))
- return
-
- self._InitWorkTree(force_sync=force_sync, submodules=submodules)
- all_refs = self.bare_ref.all
- self.CleanPublishedCache(all_refs)
- revid = self.GetRevisionId(all_refs)
-
- # Special case the root of the repo client checkout. Make sure it doesn't
- # contain files being checked out to dirs we don't allow.
- if self.relpath == '.':
- PROTECTED_PATHS = {'.repo'}
- paths = set(self.work_git.ls_tree('-z', '--name-only', '--', revid).split('\0'))
- bad_paths = paths & PROTECTED_PATHS
- if bad_paths:
- syncbuf.fail(self,
- 'Refusing to checkout project that writes to protected '
- 'paths: %s' % (', '.join(bad_paths),))
- return
-
- def _doff():
- self._FastForward(revid)
- self._CopyAndLinkFiles()
-
- def _dosubmodules():
- self._SyncSubmodules(quiet=True)
-
- head = self.work_git.GetHead()
- if head.startswith(R_HEADS):
- branch = head[len(R_HEADS):]
- try:
- head = all_refs[head]
- except KeyError:
- head = None
- else:
- branch = None
-
- if branch is None or syncbuf.detach_head:
- # Currently on a detached HEAD. The user is assumed to
- # not have any local modifications worth worrying about.
- #
- if self.IsRebaseInProgress():
- syncbuf.fail(self, _PriorSyncFailedError())
- return
-
- if head == revid:
- # No changes; don't do anything further.
- # Except if the head needs to be detached
- #
- if not syncbuf.detach_head:
- # The copy/linkfile config may have changed.
- self._CopyAndLinkFiles()
- return
- else:
- lost = self._revlist(not_rev(revid), HEAD)
- if lost:
- syncbuf.info(self, "discarding %d commits", len(lost))
-
- try:
- self._Checkout(revid, quiet=True)
- if submodules:
- self._SyncSubmodules(quiet=True)
- except GitError as e:
- syncbuf.fail(self, e)
- return
- self._CopyAndLinkFiles()
- return
-
- if head == revid:
- # No changes; don't do anything further.
- #
- # The copy/linkfile config may have changed.
- self._CopyAndLinkFiles()
- return
-
- branch = self.GetBranch(branch)
-
- if not branch.LocalMerge:
- # The current branch has no tracking configuration.
- # Jump off it to a detached HEAD.
- #
- syncbuf.info(self,
- "leaving %s; does not track upstream",
- branch.name)
- try:
- self._Checkout(revid, quiet=True)
- if submodules:
- self._SyncSubmodules(quiet=True)
- except GitError as e:
- syncbuf.fail(self, e)
- return
- self._CopyAndLinkFiles()
- return
-
- upstream_gain = self._revlist(not_rev(HEAD), revid)
-
- # See if we can perform a fast forward merge. This can happen if our
- # branch isn't in the exact same state as we last published.
- try:
- self.work_git.merge_base('--is-ancestor', HEAD, revid)
- # Skip the published logic.
- pub = False
- except GitError:
- pub = self.WasPublished(branch.name, all_refs)
-
- if pub:
- not_merged = self._revlist(not_rev(revid), pub)
- if not_merged:
- if upstream_gain:
- # The user has published this branch and some of those
- # commits are not yet merged upstream. We do not want
- # to rewrite the published commits so we punt.
- #
- syncbuf.fail(self,
- "branch %s is published (but not merged) and is now "
- "%d commits behind" % (branch.name, len(upstream_gain)))
- return
- elif pub == head:
- # All published commits are merged, and thus we are a
- # strict subset. We can fast-forward safely.
- #
- syncbuf.later1(self, _doff)
- if submodules:
- syncbuf.later1(self, _dosubmodules)
- return
-
- # Examine the local commits not in the remote. Find the
- # last one attributed to this user, if any.
- #
- local_changes = self._revlist(not_rev(revid), HEAD, format='%H %ce')
- last_mine = None
- cnt_mine = 0
- for commit in local_changes:
- commit_id, committer_email = commit.split(' ', 1)
- if committer_email == self.UserEmail:
- last_mine = commit_id
- cnt_mine += 1
-
- if not upstream_gain and cnt_mine == len(local_changes):
- # The copy/linkfile config may have changed.
- self._CopyAndLinkFiles()
- return
-
- if self.IsDirty(consider_untracked=False):
- syncbuf.fail(self, _DirtyError())
- return
-
- # If the upstream switched on us, warn the user.
- #
- if branch.merge != self.revisionExpr:
- if branch.merge and self.revisionExpr:
- syncbuf.info(self,
- 'manifest switched %s...%s',
- branch.merge,
- self.revisionExpr)
- elif branch.merge:
- syncbuf.info(self,
- 'manifest no longer tracks %s',
- branch.merge)
-
- if cnt_mine < len(local_changes):
- # Upstream rebased. Not everything in HEAD
- # was created by this user.
- #
- syncbuf.info(self,
- "discarding %d commits removed from upstream",
- len(local_changes) - cnt_mine)
-
- branch.remote = self.GetRemote()
- if not ID_RE.match(self.revisionExpr):
- # in case of manifest sync the revisionExpr might be a SHA1
- branch.merge = self.revisionExpr
- if not branch.merge.startswith('refs/'):
- branch.merge = R_HEADS + branch.merge
- branch.Save()
-
- if cnt_mine > 0 and self.rebase:
- def _docopyandlink():
- self._CopyAndLinkFiles()
-
- def _dorebase():
- self._Rebase(upstream='%s^1' % last_mine, onto=revid)
- syncbuf.later2(self, _dorebase)
- if submodules:
- syncbuf.later2(self, _dosubmodules)
- syncbuf.later2(self, _docopyandlink)
- elif local_changes:
- try:
- self._ResetHard(revid)
- if submodules:
- self._SyncSubmodules(quiet=True)
- self._CopyAndLinkFiles()
- except GitError as e:
- syncbuf.fail(self, e)
- return
- else:
- syncbuf.later1(self, _doff)
- if submodules:
- syncbuf.later1(self, _dosubmodules)
-
- def AddCopyFile(self, src, dest, topdir):
- """Mark |src| for copying to |dest| (relative to |topdir|).
-
- No filesystem changes occur here. Actual copying happens later on.
-
- Paths should have basic validation run on them before being queued.
- Further checking will be handled when the actual copy happens.
- """
- self.copyfiles.append(_CopyFile(self.worktree, src, topdir, dest))
-
- def AddLinkFile(self, src, dest, topdir):
- """Mark |dest| to create a symlink (relative to |topdir|) pointing to |src|.
-
- No filesystem changes occur here. Actual linking happens later on.
-
- Paths should have basic validation run on them before being queued.
- Further checking will be handled when the actual link happens.
- """
- self.linkfiles.append(_LinkFile(self.worktree, src, topdir, dest))
-
- def AddAnnotation(self, name, value, keep):
- self.annotations.append(Annotation(name, value, keep))
-
- def DownloadPatchSet(self, change_id, patch_id):
- """Download a single patch set of a single change to FETCH_HEAD.
- """
- remote = self.GetRemote()
-
- cmd = ['fetch', remote.name]
- cmd.append('refs/changes/%2.2d/%d/%d'
- % (change_id % 100, change_id, patch_id))
- if GitCommand(self, cmd, bare=True).Wait() != 0:
- return None
- return DownloadedChange(self,
- self.GetRevisionId(),
- change_id,
- patch_id,
- self.bare_git.rev_parse('FETCH_HEAD'))
-
- def DeleteWorktree(self, quiet=False, force=False):
- """Delete the source checkout and any other housekeeping tasks.
-
- This currently leaves behind the internal .repo/ cache state. This helps
- when switching branches or manifest changes get reverted as we don't have
- to redownload all the git objects. But we should do some GC at some point.
-
- Args:
- quiet: Whether to hide normal messages.
- force: Always delete tree even if dirty.
-
- Returns:
- True if the worktree was completely cleaned out.
- """
- if self.IsDirty():
- if force:
- print('warning: %s: Removing dirty project: uncommitted changes lost.' %
- (self.RelPath(local=False),), file=sys.stderr)
- else:
- print('error: %s: Cannot remove project: uncommitted changes are '
- 'present.\n' % (self.RelPath(local=False),), file=sys.stderr)
- return False
-
- if not quiet:
- print('%s: Deleting obsolete checkout.' % (self.RelPath(local=False),))
-
- # Unlock and delink from the main worktree. We don't use git's worktree
- # remove because it will recursively delete projects -- we handle that
- # ourselves below. https://crbug.com/git/48
- if self.use_git_worktrees:
- needle = platform_utils.realpath(self.gitdir)
- # Find the git worktree commondir under .repo/worktrees/.
- output = self.bare_git.worktree('list', '--porcelain').splitlines()[0]
- assert output.startswith('worktree '), output
- commondir = output[9:]
- # Walk each of the git worktrees to see where they point.
- configs = os.path.join(commondir, 'worktrees')
- for name in os.listdir(configs):
- gitdir = os.path.join(configs, name, 'gitdir')
- with open(gitdir) as fp:
- relpath = fp.read().strip()
- # Resolve the checkout path and see if it matches this project.
- fullpath = platform_utils.realpath(os.path.join(configs, name, relpath))
- if fullpath == needle:
- platform_utils.rmtree(os.path.join(configs, name))
-
- # Delete the .git directory first, so we're less likely to have a partially
- # working git repository around. There shouldn't be any git projects here,
- # so rmtree works.
-
- # Try to remove plain files first in case of git worktrees. If this fails
- # for any reason, we'll fall back to rmtree, and that'll display errors if
- # it can't remove things either.
- try:
- platform_utils.remove(self.gitdir)
- except OSError:
- pass
- try:
- platform_utils.rmtree(self.gitdir)
- except OSError as e:
- if e.errno != errno.ENOENT:
- print('error: %s: %s' % (self.gitdir, e), file=sys.stderr)
- print('error: %s: Failed to delete obsolete checkout; remove manually, '
- 'then run `repo sync -l`.' % (self.RelPath(local=False),),
- file=sys.stderr)
- return False
-
- # Delete everything under the worktree, except for directories that contain
- # another git project.
- dirs_to_remove = []
- failed = False
- for root, dirs, files in platform_utils.walk(self.worktree):
- for f in files:
- path = os.path.join(root, f)
- try:
- platform_utils.remove(path)
- except OSError as e:
- if e.errno != errno.ENOENT:
- print('error: %s: Failed to remove: %s' % (path, e), file=sys.stderr)
- failed = True
- dirs[:] = [d for d in dirs
- if not os.path.lexists(os.path.join(root, d, '.git'))]
- dirs_to_remove += [os.path.join(root, d) for d in dirs
- if os.path.join(root, d) not in dirs_to_remove]
- for d in reversed(dirs_to_remove):
- if platform_utils.islink(d):
- try:
- platform_utils.remove(d)
- except OSError as e:
- if e.errno != errno.ENOENT:
- print('error: %s: Failed to remove: %s' % (d, e), file=sys.stderr)
- failed = True
- elif not platform_utils.listdir(d):
- try:
- platform_utils.rmdir(d)
- except OSError as e:
- if e.errno != errno.ENOENT:
- print('error: %s: Failed to remove: %s' % (d, e), file=sys.stderr)
- failed = True
- if failed:
- print('error: %s: Failed to delete obsolete checkout.' % (self.RelPath(local=False),),
- file=sys.stderr)
- print(' Remove manually, then run `repo sync -l`.', file=sys.stderr)
- return False
-
- # Try deleting parent dirs if they are empty.
- path = self.worktree
- while path != self.manifest.topdir:
- try:
- platform_utils.rmdir(path)
- except OSError as e:
- if e.errno != errno.ENOENT:
- break
- path = os.path.dirname(path)
-
- return True
-
-# Branch Management ##
- def StartBranch(self, name, branch_merge='', revision=None):
- """Create a new branch off the manifest's revision.
- """
- if not branch_merge:
- branch_merge = self.revisionExpr
- head = self.work_git.GetHead()
- if head == (R_HEADS + name):
- return True
-
- all_refs = self.bare_ref.all
- if R_HEADS + name in all_refs:
- return GitCommand(self, ['checkout', '-q', name, '--']).Wait() == 0
-
- branch = self.GetBranch(name)
- branch.remote = self.GetRemote()
- branch.merge = branch_merge
- if not branch.merge.startswith('refs/') and not ID_RE.match(branch_merge):
- branch.merge = R_HEADS + branch_merge
-
- if revision is None:
- revid = self.GetRevisionId(all_refs)
- else:
- revid = self.work_git.rev_parse(revision)
-
- if head.startswith(R_HEADS):
- try:
- head = all_refs[head]
- except KeyError:
- head = None
- if revid and head and revid == head:
- ref = R_HEADS + name
- self.work_git.update_ref(ref, revid)
- self.work_git.symbolic_ref(HEAD, ref)
- branch.Save()
- return True
-
- if GitCommand(self, ['checkout', '-q', '-b', branch.name, revid]).Wait() == 0:
- branch.Save()
- return True
- return False
-
- def CheckoutBranch(self, name):
- """Checkout a local topic branch.
+ def __init__(
+ self,
+ manifest,
+ name,
+ remote,
+ gitdir,
+ objdir,
+ worktree,
+ relpath,
+ revisionExpr,
+ revisionId,
+ rebase=True,
+ groups=None,
+ sync_c=False,
+ sync_s=False,
+ sync_tags=True,
+ clone_depth=None,
+ upstream=None,
+ parent=None,
+ use_git_worktrees=False,
+ is_derived=False,
+ dest_branch=None,
+ optimized_fetch=False,
+ retry_fetches=0,
+ old_revision=None,
+ ):
+ """Init a Project object.
Args:
- name: The name of the branch to checkout.
+ manifest: The XmlManifest object.
+ name: The `name` attribute of manifest.xml's project element.
+ remote: RemoteSpec object specifying its remote's properties.
+ gitdir: Absolute path of git directory.
+ objdir: Absolute path of directory to store git objects.
+ worktree: Absolute path of git working tree.
+ relpath: Relative path of git working tree to repo's top directory.
+ revisionExpr: The `revision` attribute of manifest.xml's project
+ element.
+ revisionId: git commit id for checking out.
+ rebase: The `rebase` attribute of manifest.xml's project element.
+ groups: The `groups` attribute of manifest.xml's project element.
+ sync_c: The `sync-c` attribute of manifest.xml's project element.
+ sync_s: The `sync-s` attribute of manifest.xml's project element.
+ sync_tags: The `sync-tags` attribute of manifest.xml's project
+ element.
+ upstream: The `upstream` attribute of manifest.xml's project
+ element.
+ parent: The parent Project object.
+ use_git_worktrees: Whether to use `git worktree` for this project.
+ is_derived: False if the project was explicitly defined in the
+ manifest; True if the project is a discovered submodule.
+ dest_branch: The branch to which to push changes for review by
+ default.
+ optimized_fetch: If True, when a project is set to a sha1 revision,
+ only fetch from the remote if the sha1 is not present locally.
+ retry_fetches: Retry remote fetches n times upon receiving transient
+ error with exponential backoff and jitter.
+ old_revision: saved git commit id for open GITC projects.
+ """
+ self.client = self.manifest = manifest
+ self.name = name
+ self.remote = remote
+ self.UpdatePaths(relpath, worktree, gitdir, objdir)
+ self.SetRevision(revisionExpr, revisionId=revisionId)
+
+ self.rebase = rebase
+ self.groups = groups
+ self.sync_c = sync_c
+ self.sync_s = sync_s
+ self.sync_tags = sync_tags
+ self.clone_depth = clone_depth
+ self.upstream = upstream
+ self.parent = parent
+ # NB: Do not use this setting in __init__ to change behavior so that the
+ # manifest.git checkout can inspect & change it after instantiating.
+ # See the XmlManifest init code for more info.
+ self.use_git_worktrees = use_git_worktrees
+ self.is_derived = is_derived
+ self.optimized_fetch = optimized_fetch
+ self.retry_fetches = max(0, retry_fetches)
+ self.subprojects = []
+
+ self.snapshots = {}
+ self.copyfiles = []
+ self.linkfiles = []
+ self.annotations = []
+ self.dest_branch = dest_branch
+ self.old_revision = old_revision
+
+ # This will be filled in if a project is later identified to be the
+ # project containing repo hooks.
+ self.enabled_repo_hooks = []
+
+ def RelPath(self, local=True):
+ """Return the path for the project relative to a manifest.
+
+ Args:
+ local: a boolean, if True, the path is relative to the local
+ (sub)manifest. If false, the path is relative to the outermost
+ manifest.
+ """
+ if local:
+ return self.relpath
+ return os.path.join(self.manifest.path_prefix, self.relpath)
+
+ def SetRevision(self, revisionExpr, revisionId=None):
+ """Set revisionId based on revision expression and id"""
+ self.revisionExpr = revisionExpr
+ if revisionId is None and revisionExpr and IsId(revisionExpr):
+ self.revisionId = self.revisionExpr
+ else:
+ self.revisionId = revisionId
+
+ def UpdatePaths(self, relpath, worktree, gitdir, objdir):
+ """Update paths used by this project"""
+ self.gitdir = gitdir.replace("\\", "/")
+ self.objdir = objdir.replace("\\", "/")
+ if worktree:
+ self.worktree = os.path.normpath(worktree).replace("\\", "/")
+ else:
+ self.worktree = None
+ self.relpath = relpath
+
+ self.config = GitConfig.ForRepository(
+ gitdir=self.gitdir, defaults=self.manifest.globalConfig
+ )
+
+ if self.worktree:
+ self.work_git = self._GitGetByExec(
+ self, bare=False, gitdir=self.gitdir
+ )
+ else:
+ self.work_git = None
+ self.bare_git = self._GitGetByExec(self, bare=True, gitdir=self.gitdir)
+ self.bare_ref = GitRefs(self.gitdir)
+ self.bare_objdir = self._GitGetByExec(
+ self, bare=True, gitdir=self.objdir
+ )
+
+ @property
+ def UseAlternates(self):
+ """Whether git alternates are in use.
+
+ This will be removed once migration to alternates is complete.
+ """
+ return _ALTERNATES or self.manifest.is_multimanifest
+
+ @property
+ def Derived(self):
+ return self.is_derived
+
+ @property
+ def Exists(self):
+ return platform_utils.isdir(self.gitdir) and platform_utils.isdir(
+ self.objdir
+ )
+
+ @property
+ def CurrentBranch(self):
+ """Obtain the name of the currently checked out branch.
+
+ The branch name omits the 'refs/heads/' prefix.
+ None is returned if the project is on a detached HEAD, or if the
+ work_git is otheriwse inaccessible (e.g. an incomplete sync).
+ """
+ try:
+ b = self.work_git.GetHead()
+ except NoManifestException:
+ # If the local checkout is in a bad state, don't barf. Let the
+ # callers process this like the head is unreadable.
+ return None
+ if b.startswith(R_HEADS):
+ return b[len(R_HEADS) :]
+ return None
+
+ def IsRebaseInProgress(self):
+ return (
+ os.path.exists(self.work_git.GetDotgitPath("rebase-apply"))
+ or os.path.exists(self.work_git.GetDotgitPath("rebase-merge"))
+ or os.path.exists(os.path.join(self.worktree, ".dotest"))
+ )
+
+ def IsDirty(self, consider_untracked=True):
+ """Is the working directory modified in some way?"""
+ self.work_git.update_index(
+ "-q", "--unmerged", "--ignore-missing", "--refresh"
+ )
+ if self.work_git.DiffZ("diff-index", "-M", "--cached", HEAD):
+ return True
+ if self.work_git.DiffZ("diff-files"):
+ return True
+ if consider_untracked and self.UntrackedFiles():
+ return True
+ return False
+
+ _userident_name = None
+ _userident_email = None
+
+ @property
+ def UserName(self):
+ """Obtain the user's personal name."""
+ if self._userident_name is None:
+ self._LoadUserIdentity()
+ return self._userident_name
+
+ @property
+ def UserEmail(self):
+ """Obtain the user's email address. This is very likely
+ to be their Gerrit login.
+ """
+ if self._userident_email is None:
+ self._LoadUserIdentity()
+ return self._userident_email
+
+ def _LoadUserIdentity(self):
+ u = self.bare_git.var("GIT_COMMITTER_IDENT")
+ m = re.compile("^(.*) <([^>]*)> ").match(u)
+ if m:
+ self._userident_name = m.group(1)
+ self._userident_email = m.group(2)
+ else:
+ self._userident_name = ""
+ self._userident_email = ""
+
+ def GetRemote(self, name=None):
+ """Get the configuration for a single remote.
+
+ Defaults to the current project's remote.
+ """
+ if name is None:
+ name = self.remote.name
+ return self.config.GetRemote(name)
+
+ def GetBranch(self, name):
+ """Get the configuration for a single branch."""
+ return self.config.GetBranch(name)
+
+ def GetBranches(self):
+ """Get all existing local branches."""
+ current = self.CurrentBranch
+ all_refs = self._allrefs
+ heads = {}
+
+ for name, ref_id in all_refs.items():
+ if name.startswith(R_HEADS):
+ name = name[len(R_HEADS) :]
+ b = self.GetBranch(name)
+ b.current = name == current
+ b.published = None
+ b.revision = ref_id
+ heads[name] = b
+
+ for name, ref_id in all_refs.items():
+ if name.startswith(R_PUB):
+ name = name[len(R_PUB) :]
+ b = heads.get(name)
+ if b:
+ b.published = ref_id
+
+ return heads
+
+ def MatchesGroups(self, manifest_groups):
+ """Returns true if the manifest groups specified at init should cause
+ this project to be synced.
+ Prefixing a manifest group with "-" inverts the meaning of a group.
+ All projects are implicitly labelled with "all".
+
+ labels are resolved in order. In the example case of
+ project_groups: "all,group1,group2"
+ manifest_groups: "-group1,group2"
+ the project will be matched.
+
+ The special manifest group "default" will match any project that
+ does not have the special project group "notdefault"
+ """
+ default_groups = self.manifest.default_groups or ["default"]
+ expanded_manifest_groups = manifest_groups or default_groups
+ expanded_project_groups = ["all"] + (self.groups or [])
+ if "notdefault" not in expanded_project_groups:
+ expanded_project_groups += ["default"]
+
+ matched = False
+ for group in expanded_manifest_groups:
+ if group.startswith("-") and group[1:] in expanded_project_groups:
+ matched = False
+ elif group in expanded_project_groups:
+ matched = True
+
+ return matched
+
+ def UncommitedFiles(self, get_all=True):
+ """Returns a list of strings, uncommitted files in the git tree.
+
+ Args:
+ get_all: a boolean, if True - get information about all different
+ uncommitted files. If False - return as soon as any kind of
+ uncommitted files is detected.
+ """
+ details = []
+ self.work_git.update_index(
+ "-q", "--unmerged", "--ignore-missing", "--refresh"
+ )
+ if self.IsRebaseInProgress():
+ details.append("rebase in progress")
+ if not get_all:
+ return details
+
+ changes = self.work_git.DiffZ("diff-index", "--cached", HEAD).keys()
+ if changes:
+ details.extend(changes)
+ if not get_all:
+ return details
+
+ changes = self.work_git.DiffZ("diff-files").keys()
+ if changes:
+ details.extend(changes)
+ if not get_all:
+ return details
+
+ changes = self.UntrackedFiles()
+ if changes:
+ details.extend(changes)
+
+ return details
+
+ def UntrackedFiles(self):
+ """Returns a list of strings, untracked files in the git tree."""
+ return self.work_git.LsOthers()
+
+ def HasChanges(self):
+ """Returns true if there are uncommitted changes."""
+ return bool(self.UncommitedFiles(get_all=False))
+
+ def PrintWorkTreeStatus(self, output_redir=None, quiet=False, local=False):
+ """Prints the status of the repository to stdout.
+
+ Args:
+ output_redir: If specified, redirect the output to this object.
+ quiet: If True then only print the project name. Do not print
+ the modified files, branch name, etc.
+ local: a boolean, if True, the path is relative to the local
+ (sub)manifest. If false, the path is relative to the outermost
+ manifest.
+ """
+ if not platform_utils.isdir(self.worktree):
+ if output_redir is None:
+ output_redir = sys.stdout
+ print(file=output_redir)
+ print("project %s/" % self.RelPath(local), file=output_redir)
+ print(' missing (run "repo sync")', file=output_redir)
+ return
+
+ self.work_git.update_index(
+ "-q", "--unmerged", "--ignore-missing", "--refresh"
+ )
+ rb = self.IsRebaseInProgress()
+ di = self.work_git.DiffZ("diff-index", "-M", "--cached", HEAD)
+ df = self.work_git.DiffZ("diff-files")
+ do = self.work_git.LsOthers()
+ if not rb and not di and not df and not do and not self.CurrentBranch:
+ return "CLEAN"
+
+ out = StatusColoring(self.config)
+ if output_redir is not None:
+ out.redirect(output_redir)
+ out.project("project %-40s", self.RelPath(local) + "/ ")
+
+ if quiet:
+ out.nl()
+ return "DIRTY"
+
+ branch = self.CurrentBranch
+ if branch is None:
+ out.nobranch("(*** NO BRANCH ***)")
+ else:
+ out.branch("branch %s", branch)
+ out.nl()
+
+ if rb:
+ out.important("prior sync failed; rebase still in progress")
+ out.nl()
+
+ paths = list()
+ paths.extend(di.keys())
+ paths.extend(df.keys())
+ paths.extend(do)
+
+ for p in sorted(set(paths)):
+ try:
+ i = di[p]
+ except KeyError:
+ i = None
+
+ try:
+ f = df[p]
+ except KeyError:
+ f = None
+
+ if i:
+ i_status = i.status.upper()
+ else:
+ i_status = "-"
+
+ if f:
+ f_status = f.status.lower()
+ else:
+ f_status = "-"
+
+ if i and i.src_path:
+ line = " %s%s\t%s => %s (%s%%)" % (
+ i_status,
+ f_status,
+ i.src_path,
+ p,
+ i.level,
+ )
+ else:
+ line = " %s%s\t%s" % (i_status, f_status, p)
+
+ if i and not f:
+ out.added("%s", line)
+ elif (i and f) or (not i and f):
+ out.changed("%s", line)
+ elif not i and not f:
+ out.untracked("%s", line)
+ else:
+ out.write("%s", line)
+ out.nl()
+
+ return "DIRTY"
+
+ def PrintWorkTreeDiff(
+ self, absolute_paths=False, output_redir=None, local=False
+ ):
+ """Prints the status of the repository to stdout."""
+ out = DiffColoring(self.config)
+ if output_redir:
+ out.redirect(output_redir)
+ cmd = ["diff"]
+ if out.is_on:
+ cmd.append("--color")
+ cmd.append(HEAD)
+ if absolute_paths:
+ cmd.append("--src-prefix=a/%s/" % self.RelPath(local))
+ cmd.append("--dst-prefix=b/%s/" % self.RelPath(local))
+ cmd.append("--")
+ try:
+ p = GitCommand(self, cmd, capture_stdout=True, capture_stderr=True)
+ p.Wait()
+ except GitError as e:
+ out.nl()
+ out.project("project %s/" % self.RelPath(local))
+ out.nl()
+ out.fail("%s", str(e))
+ out.nl()
+ return False
+ if p.stdout:
+ out.nl()
+ out.project("project %s/" % self.RelPath(local))
+ out.nl()
+ out.write("%s", p.stdout)
+ return p.Wait() == 0
+
+ def WasPublished(self, branch, all_refs=None):
+ """Was the branch published (uploaded) for code review?
+ If so, returns the SHA-1 hash of the last published
+ state for the branch.
+ """
+ key = R_PUB + branch
+ if all_refs is None:
+ try:
+ return self.bare_git.rev_parse(key)
+ except GitError:
+ return None
+ else:
+ try:
+ return all_refs[key]
+ except KeyError:
+ return None
+
+ def CleanPublishedCache(self, all_refs=None):
+ """Prunes any stale published refs."""
+ if all_refs is None:
+ all_refs = self._allrefs
+ heads = set()
+ canrm = {}
+ for name, ref_id in all_refs.items():
+ if name.startswith(R_HEADS):
+ heads.add(name)
+ elif name.startswith(R_PUB):
+ canrm[name] = ref_id
+
+ for name, ref_id in canrm.items():
+ n = name[len(R_PUB) :]
+ if R_HEADS + n not in heads:
+ self.bare_git.DeleteRef(name, ref_id)
+
+ def GetUploadableBranches(self, selected_branch=None):
+ """List any branches which can be uploaded for review."""
+ heads = {}
+ pubed = {}
+
+ for name, ref_id in self._allrefs.items():
+ if name.startswith(R_HEADS):
+ heads[name[len(R_HEADS) :]] = ref_id
+ elif name.startswith(R_PUB):
+ pubed[name[len(R_PUB) :]] = ref_id
+
+ ready = []
+ for branch, ref_id in heads.items():
+ if branch in pubed and pubed[branch] == ref_id:
+ continue
+ if selected_branch and branch != selected_branch:
+ continue
+
+ rb = self.GetUploadableBranch(branch)
+ if rb:
+ ready.append(rb)
+ return ready
+
+ def GetUploadableBranch(self, branch_name):
+ """Get a single uploadable branch, or None."""
+ branch = self.GetBranch(branch_name)
+ base = branch.LocalMerge
+ if branch.LocalMerge:
+ rb = ReviewableBranch(self, branch, base)
+ if rb.commits:
+ return rb
+ return None
+
+ def UploadForReview(
+ self,
+ branch=None,
+ people=([], []),
+ dryrun=False,
+ auto_topic=False,
+ hashtags=(),
+ labels=(),
+ private=False,
+ notify=None,
+ wip=False,
+ ready=False,
+ dest_branch=None,
+ validate_certs=True,
+ push_options=None,
+ ):
+ """Uploads the named branch for code review."""
+ if branch is None:
+ branch = self.CurrentBranch
+ if branch is None:
+ raise GitError("not currently on a branch")
+
+ branch = self.GetBranch(branch)
+ if not branch.LocalMerge:
+ raise GitError("branch %s does not track a remote" % branch.name)
+ if not branch.remote.review:
+ raise GitError("remote %s has no review url" % branch.remote.name)
+
+ # Basic validity check on label syntax.
+ for label in labels:
+ if not re.match(r"^.+[+-][0-9]+$", label):
+ raise UploadError(
+ f'invalid label syntax "{label}": labels use forms like '
+ "CodeReview+1 or Verified-1"
+ )
+
+ if dest_branch is None:
+ dest_branch = self.dest_branch
+ if dest_branch is None:
+ dest_branch = branch.merge
+ if not dest_branch.startswith(R_HEADS):
+ dest_branch = R_HEADS + dest_branch
+
+ if not branch.remote.projectname:
+ branch.remote.projectname = self.name
+ branch.remote.Save()
+
+ url = branch.remote.ReviewUrl(self.UserEmail, validate_certs)
+ if url is None:
+ raise UploadError("review not configured")
+ cmd = ["push"]
+ if dryrun:
+ cmd.append("-n")
+
+ if url.startswith("ssh://"):
+ cmd.append("--receive-pack=gerrit receive-pack")
+
+ # This stops git from pushing all reachable annotated tags when
+ # push.followTags is configured. Gerrit does not accept any tags
+ # pushed to a CL.
+ if git_require((1, 8, 3)):
+ cmd.append("--no-follow-tags")
+
+ for push_option in push_options or []:
+ cmd.append("-o")
+ cmd.append(push_option)
+
+ cmd.append(url)
+
+ if dest_branch.startswith(R_HEADS):
+ dest_branch = dest_branch[len(R_HEADS) :]
+
+ ref_spec = "%s:refs/for/%s" % (R_HEADS + branch.name, dest_branch)
+ opts = []
+ if auto_topic:
+ opts += ["topic=" + branch.name]
+ opts += ["t=%s" % p for p in hashtags]
+ # NB: No need to encode labels as they've been validated above.
+ opts += ["l=%s" % p for p in labels]
+
+ opts += ["r=%s" % p for p in people[0]]
+ opts += ["cc=%s" % p for p in people[1]]
+ if notify:
+ opts += ["notify=" + notify]
+ if private:
+ opts += ["private"]
+ if wip:
+ opts += ["wip"]
+ if ready:
+ opts += ["ready"]
+ if opts:
+ ref_spec = ref_spec + "%" + ",".join(opts)
+ cmd.append(ref_spec)
+
+ if GitCommand(self, cmd, bare=True).Wait() != 0:
+ raise UploadError("Upload failed")
+
+ if not dryrun:
+ msg = "posted to %s for %s" % (branch.remote.review, dest_branch)
+ self.bare_git.UpdateRef(
+ R_PUB + branch.name, R_HEADS + branch.name, message=msg
+ )
+
+ def _ExtractArchive(self, tarpath, path=None):
+ """Extract the given tar on its current location
+
+ Args:
+ tarpath: The path to the actual tar file
+
+ """
+ try:
+ with tarfile.open(tarpath, "r") as tar:
+ tar.extractall(path=path)
+ return True
+ except (IOError, tarfile.TarError) as e:
+ _error("Cannot extract archive %s: %s", tarpath, str(e))
+ return False
+
+ def CachePopulate(self, cache_dir, url):
+ """Populate cache in the cache_dir.
+
+ Args:
+ cache_dir: Directory to cache git files from Google Storage.
+ url: Git url of current repository.
+
+ Raises:
+ CacheApplyError if it fails to populate the git cache.
+ """
+ cmd = [
+ "cache",
+ "populate",
+ "--ignore_locks",
+ "-v",
+ "--cache-dir",
+ cache_dir,
+ url,
+ ]
+
+ if GitCommand(self, cmd, cwd=cache_dir).Wait() != 0:
+ raise CacheApplyError(
+ "Failed to populate cache. cache_dir: %s "
+ "url: %s" % (cache_dir, url)
+ )
+
+ def CacheExists(self, cache_dir, url):
+ """Check the existence of the cache files.
+
+ Args:
+ cache_dir: Directory to cache git files.
+ url: Git url of current repository.
+
+ Raises:
+ CacheApplyError if the cache files do not exist.
+ """
+ cmd = ["cache", "exists", "--quiet", "--cache-dir", cache_dir, url]
+
+ exist = GitCommand(self, cmd, cwd=self.gitdir, capture_stdout=True)
+ if exist.Wait() != 0:
+ raise CacheApplyError(
+ "Failed to execute git cache exists cmd. "
+ "cache_dir: %s url: %s" % (cache_dir, url)
+ )
+
+ if not exist.stdout or not exist.stdout.strip():
+ raise CacheApplyError(
+ "Failed to find cache. cache_dir: %s "
+ "url: %s" % (cache_dir, url)
+ )
+ return exist.stdout.strip()
+
+ def CacheApply(self, cache_dir):
+ """Apply git cache files populated from Google Storage buckets.
+
+ Args:
+ cache_dir: Directory to cache git files.
+
+ Raises:
+ CacheApplyError if it fails to apply git caches.
+ """
+ remote = self.GetRemote(self.remote.name)
+
+ self.CachePopulate(cache_dir, remote.url)
+
+ mirror_dir = self.CacheExists(cache_dir, remote.url)
+
+ refspec = RefSpec(
+ True, "refs/heads/*", "refs/remotes/%s/*" % remote.name
+ )
+
+ fetch_cache_cmd = ["fetch", mirror_dir, str(refspec)]
+ if GitCommand(self, fetch_cache_cmd, self.gitdir).Wait() != 0:
+ raise CacheApplyError(
+ "Failed to fetch refs %s from %s" % (mirror_dir, str(refspec))
+ )
+
+ def Sync_NetworkHalf(
+ self,
+ quiet=False,
+ verbose=False,
+ output_redir=None,
+ is_new=None,
+ current_branch_only=None,
+ force_sync=False,
+ clone_bundle=True,
+ tags=None,
+ archive=False,
+ optimized_fetch=False,
+ retry_fetches=0,
+ prune=False,
+ submodules=False,
+ cache_dir=None,
+ ssh_proxy=None,
+ clone_filter=None,
+ partial_clone_exclude=set(),
+ ):
+ """Perform only the network IO portion of the sync process.
+ Local working directory/branch state is not affected.
+ """
+ if archive and not isinstance(self, MetaProject):
+ if self.remote.url.startswith(("http://", "https://")):
+ _error(
+ "%s: Cannot fetch archives from http/https remotes.",
+ self.name,
+ )
+ return SyncNetworkHalfResult(False, False)
+
+ name = self.relpath.replace("\\", "/")
+ name = name.replace("/", "_")
+ tarpath = "%s.tar" % name
+ topdir = self.manifest.topdir
+
+ try:
+ self._FetchArchive(tarpath, cwd=topdir)
+ except GitError as e:
+ _error("%s", e)
+ return SyncNetworkHalfResult(False, False)
+
+ # From now on, we only need absolute tarpath.
+ tarpath = os.path.join(topdir, tarpath)
+
+ if not self._ExtractArchive(tarpath, path=topdir):
+ return SyncNetworkHalfResult(False, True)
+ try:
+ platform_utils.remove(tarpath)
+ except OSError as e:
+ _warn("Cannot remove archive %s: %s", tarpath, str(e))
+ self._CopyAndLinkFiles()
+ return SyncNetworkHalfResult(True, True)
+
+ # If the shared object dir already exists, don't try to rebootstrap with
+ # a clone bundle download. We should have the majority of objects
+ # already.
+ if clone_bundle and os.path.exists(self.objdir):
+ clone_bundle = False
+
+ if self.name in partial_clone_exclude:
+ clone_bundle = True
+ clone_filter = None
+
+ if is_new is None:
+ is_new = not self.Exists
+ if is_new:
+ self._InitGitDir(force_sync=force_sync, quiet=quiet)
+ else:
+ self._UpdateHooks(quiet=quiet)
+ self._InitRemote()
+
+ if self.UseAlternates:
+ # If gitdir/objects is a symlink, migrate it from the old layout.
+ gitdir_objects = os.path.join(self.gitdir, "objects")
+ if platform_utils.islink(gitdir_objects):
+ platform_utils.remove(gitdir_objects, missing_ok=True)
+ gitdir_alt = os.path.join(self.gitdir, "objects/info/alternates")
+ if not os.path.exists(gitdir_alt):
+ os.makedirs(os.path.dirname(gitdir_alt), exist_ok=True)
+ _lwrite(
+ gitdir_alt,
+ os.path.join(
+ os.path.relpath(self.objdir, gitdir_objects), "objects"
+ )
+ + "\n",
+ )
+
+ if is_new:
+ alt = os.path.join(self.objdir, "objects/info/alternates")
+ try:
+ with open(alt) as fd:
+ # This works for both absolute and relative alternate
+ # directories.
+ alt_dir = os.path.join(
+ self.objdir, "objects", fd.readline().rstrip()
+ )
+ except IOError:
+ alt_dir = None
+ else:
+ alt_dir = None
+
+ applied_cache = False
+ # If cache_dir is provided, and it's a new repository without
+ # alternative_dir, bootstrap this project repo with the git
+ # cache files.
+ if cache_dir is not None and is_new and alt_dir is None:
+ try:
+ self.CacheApply(cache_dir)
+ applied_cache = True
+ is_new = False
+ except CacheApplyError as e:
+ _error("Could not apply git cache: %s", e)
+ _error("Please check if you have the right GS credentials.")
+ _error("Please check if the cache files exist in GS.")
+
+ if (
+ clone_bundle
+ and not applied_cache
+ and alt_dir is None
+ and self._ApplyCloneBundle(
+ initial=is_new, quiet=quiet, verbose=verbose
+ )
+ ):
+ is_new = False
+
+ if current_branch_only is None:
+ if self.sync_c:
+ current_branch_only = True
+ elif not self.manifest._loaded:
+ # Manifest cannot check defaults until it syncs.
+ current_branch_only = False
+ elif self.manifest.default.sync_c:
+ current_branch_only = True
+
+ if tags is None:
+ tags = self.sync_tags
+
+ if self.clone_depth:
+ depth = self.clone_depth
+ else:
+ depth = self.manifest.manifestProject.depth
+
+ # See if we can skip the network fetch entirely.
+ remote_fetched = False
+ if not (
+ optimized_fetch
+ and (
+ ID_RE.match(self.revisionExpr)
+ and self._CheckForImmutableRevision()
+ )
+ ):
+ remote_fetched = True
+ if not self._RemoteFetch(
+ initial=is_new,
+ quiet=quiet,
+ verbose=verbose,
+ output_redir=output_redir,
+ alt_dir=alt_dir,
+ current_branch_only=current_branch_only,
+ tags=tags,
+ prune=prune,
+ depth=depth,
+ submodules=submodules,
+ force_sync=force_sync,
+ ssh_proxy=ssh_proxy,
+ clone_filter=clone_filter,
+ retry_fetches=retry_fetches,
+ ):
+ return SyncNetworkHalfResult(False, remote_fetched)
+
+ mp = self.manifest.manifestProject
+ dissociate = mp.dissociate
+ if dissociate:
+ alternates_file = os.path.join(
+ self.objdir, "objects/info/alternates"
+ )
+ if os.path.exists(alternates_file):
+ cmd = ["repack", "-a", "-d"]
+ p = GitCommand(
+ self,
+ cmd,
+ bare=True,
+ capture_stdout=bool(output_redir),
+ merge_output=bool(output_redir),
+ )
+ if p.stdout and output_redir:
+ output_redir.write(p.stdout)
+ if p.Wait() != 0:
+ return SyncNetworkHalfResult(False, remote_fetched)
+ platform_utils.remove(alternates_file)
+
+ if self.worktree:
+ self._InitMRef()
+ else:
+ self._InitMirrorHead()
+ platform_utils.remove(
+ os.path.join(self.gitdir, "FETCH_HEAD"), missing_ok=True
+ )
+ return SyncNetworkHalfResult(True, remote_fetched)
+
+ def PostRepoUpgrade(self):
+ self._InitHooks()
+
+ def _CopyAndLinkFiles(self):
+ if self.client.isGitcClient:
+ return
+ for copyfile in self.copyfiles:
+ copyfile._Copy()
+ for linkfile in self.linkfiles:
+ linkfile._Link()
+
+ def GetCommitRevisionId(self):
+ """Get revisionId of a commit.
+
+ Use this method instead of GetRevisionId to get the id of the commit
+ rather than the id of the current git object (for example, a tag)
+
+ """
+ if not self.revisionExpr.startswith(R_TAGS):
+ return self.GetRevisionId(self._allrefs)
+
+ try:
+ return self.bare_git.rev_list(self.revisionExpr, "-1")[0]
+ except GitError:
+ raise ManifestInvalidRevisionError(
+ "revision %s in %s not found" % (self.revisionExpr, self.name)
+ )
+
+ def GetRevisionId(self, all_refs=None):
+ if self.revisionId:
+ return self.revisionId
+
+ rem = self.GetRemote()
+ rev = rem.ToLocal(self.revisionExpr)
+
+ if all_refs is not None and rev in all_refs:
+ return all_refs[rev]
+
+ try:
+ return self.bare_git.rev_parse("--verify", "%s^0" % rev)
+ except GitError:
+ raise ManifestInvalidRevisionError(
+ "revision %s in %s not found" % (self.revisionExpr, self.name)
+ )
+
+ def SetRevisionId(self, revisionId):
+ if self.revisionExpr:
+ self.upstream = self.revisionExpr
+
+ self.revisionId = revisionId
+
+ def Sync_LocalHalf(self, syncbuf, force_sync=False, submodules=False):
+ """Perform only the local IO portion of the sync process.
+
+ Network access is not required.
+ """
+ if not os.path.exists(self.gitdir):
+ syncbuf.fail(
+ self,
+ "Cannot checkout %s due to missing network sync; Run "
+ "`repo sync -n %s` first." % (self.name, self.name),
+ )
+ return
+
+ self._InitWorkTree(force_sync=force_sync, submodules=submodules)
+ all_refs = self.bare_ref.all
+ self.CleanPublishedCache(all_refs)
+ revid = self.GetRevisionId(all_refs)
+
+ # Special case the root of the repo client checkout. Make sure it
+ # doesn't contain files being checked out to dirs we don't allow.
+ if self.relpath == ".":
+ PROTECTED_PATHS = {".repo"}
+ paths = set(
+ self.work_git.ls_tree("-z", "--name-only", "--", revid).split(
+ "\0"
+ )
+ )
+ bad_paths = paths & PROTECTED_PATHS
+ if bad_paths:
+ syncbuf.fail(
+ self,
+ "Refusing to checkout project that writes to protected "
+ "paths: %s" % (", ".join(bad_paths),),
+ )
+ return
+
+ def _doff():
+ self._FastForward(revid)
+ self._CopyAndLinkFiles()
+
+ def _dosubmodules():
+ self._SyncSubmodules(quiet=True)
+
+ head = self.work_git.GetHead()
+ if head.startswith(R_HEADS):
+ branch = head[len(R_HEADS) :]
+ try:
+ head = all_refs[head]
+ except KeyError:
+ head = None
+ else:
+ branch = None
+
+ if branch is None or syncbuf.detach_head:
+ # Currently on a detached HEAD. The user is assumed to
+ # not have any local modifications worth worrying about.
+ if self.IsRebaseInProgress():
+ syncbuf.fail(self, _PriorSyncFailedError())
+ return
+
+ if head == revid:
+ # No changes; don't do anything further.
+ # Except if the head needs to be detached.
+ if not syncbuf.detach_head:
+ # The copy/linkfile config may have changed.
+ self._CopyAndLinkFiles()
+ return
+ else:
+ lost = self._revlist(not_rev(revid), HEAD)
+ if lost:
+ syncbuf.info(self, "discarding %d commits", len(lost))
+
+ try:
+ self._Checkout(revid, quiet=True)
+ if submodules:
+ self._SyncSubmodules(quiet=True)
+ except GitError as e:
+ syncbuf.fail(self, e)
+ return
+ self._CopyAndLinkFiles()
+ return
+
+ if head == revid:
+ # No changes; don't do anything further.
+ #
+ # The copy/linkfile config may have changed.
+ self._CopyAndLinkFiles()
+ return
+
+ branch = self.GetBranch(branch)
+
+ if not branch.LocalMerge:
+ # The current branch has no tracking configuration.
+ # Jump off it to a detached HEAD.
+ syncbuf.info(
+ self, "leaving %s; does not track upstream", branch.name
+ )
+ try:
+ self._Checkout(revid, quiet=True)
+ if submodules:
+ self._SyncSubmodules(quiet=True)
+ except GitError as e:
+ syncbuf.fail(self, e)
+ return
+ self._CopyAndLinkFiles()
+ return
+
+ upstream_gain = self._revlist(not_rev(HEAD), revid)
+
+ # See if we can perform a fast forward merge. This can happen if our
+ # branch isn't in the exact same state as we last published.
+ try:
+ self.work_git.merge_base("--is-ancestor", HEAD, revid)
+ # Skip the published logic.
+ pub = False
+ except GitError:
+ pub = self.WasPublished(branch.name, all_refs)
+
+ if pub:
+ not_merged = self._revlist(not_rev(revid), pub)
+ if not_merged:
+ if upstream_gain:
+ # The user has published this branch and some of those
+ # commits are not yet merged upstream. We do not want
+ # to rewrite the published commits so we punt.
+ syncbuf.fail(
+ self,
+ "branch %s is published (but not merged) and is now "
+ "%d commits behind" % (branch.name, len(upstream_gain)),
+ )
+ return
+ elif pub == head:
+ # All published commits are merged, and thus we are a
+ # strict subset. We can fast-forward safely.
+ syncbuf.later1(self, _doff)
+ if submodules:
+ syncbuf.later1(self, _dosubmodules)
+ return
+
+ # Examine the local commits not in the remote. Find the
+ # last one attributed to this user, if any.
+ local_changes = self._revlist(not_rev(revid), HEAD, format="%H %ce")
+ last_mine = None
+ cnt_mine = 0
+ for commit in local_changes:
+ commit_id, committer_email = commit.split(" ", 1)
+ if committer_email == self.UserEmail:
+ last_mine = commit_id
+ cnt_mine += 1
+
+ if not upstream_gain and cnt_mine == len(local_changes):
+ # The copy/linkfile config may have changed.
+ self._CopyAndLinkFiles()
+ return
+
+ if self.IsDirty(consider_untracked=False):
+ syncbuf.fail(self, _DirtyError())
+ return
+
+ # If the upstream switched on us, warn the user.
+ if branch.merge != self.revisionExpr:
+ if branch.merge and self.revisionExpr:
+ syncbuf.info(
+ self,
+ "manifest switched %s...%s",
+ branch.merge,
+ self.revisionExpr,
+ )
+ elif branch.merge:
+ syncbuf.info(self, "manifest no longer tracks %s", branch.merge)
+
+ if cnt_mine < len(local_changes):
+ # Upstream rebased. Not everything in HEAD was created by this user.
+ syncbuf.info(
+ self,
+ "discarding %d commits removed from upstream",
+ len(local_changes) - cnt_mine,
+ )
+
+ branch.remote = self.GetRemote()
+ if not ID_RE.match(self.revisionExpr):
+ # In case of manifest sync the revisionExpr might be a SHA1.
+ branch.merge = self.revisionExpr
+ if not branch.merge.startswith("refs/"):
+ branch.merge = R_HEADS + branch.merge
+ branch.Save()
+
+ if cnt_mine > 0 and self.rebase:
+
+ def _docopyandlink():
+ self._CopyAndLinkFiles()
+
+ def _dorebase():
+ self._Rebase(upstream="%s^1" % last_mine, onto=revid)
+
+ syncbuf.later2(self, _dorebase)
+ if submodules:
+ syncbuf.later2(self, _dosubmodules)
+ syncbuf.later2(self, _docopyandlink)
+ elif local_changes:
+ try:
+ self._ResetHard(revid)
+ if submodules:
+ self._SyncSubmodules(quiet=True)
+ self._CopyAndLinkFiles()
+ except GitError as e:
+ syncbuf.fail(self, e)
+ return
+ else:
+ syncbuf.later1(self, _doff)
+ if submodules:
+ syncbuf.later1(self, _dosubmodules)
+
+ def AddCopyFile(self, src, dest, topdir):
+ """Mark |src| for copying to |dest| (relative to |topdir|).
+
+ No filesystem changes occur here. Actual copying happens later on.
+
+ Paths should have basic validation run on them before being queued.
+ Further checking will be handled when the actual copy happens.
+ """
+ self.copyfiles.append(_CopyFile(self.worktree, src, topdir, dest))
+
+ def AddLinkFile(self, src, dest, topdir):
+ """Mark |dest| to create a symlink (relative to |topdir|) pointing to
+ |src|.
+
+ No filesystem changes occur here. Actual linking happens later on.
+
+ Paths should have basic validation run on them before being queued.
+ Further checking will be handled when the actual link happens.
+ """
+ self.linkfiles.append(_LinkFile(self.worktree, src, topdir, dest))
+
+ def AddAnnotation(self, name, value, keep):
+ self.annotations.append(Annotation(name, value, keep))
+
+ def DownloadPatchSet(self, change_id, patch_id):
+ """Download a single patch set of a single change to FETCH_HEAD."""
+ remote = self.GetRemote()
+
+ cmd = ["fetch", remote.name]
+ cmd.append(
+ "refs/changes/%2.2d/%d/%d" % (change_id % 100, change_id, patch_id)
+ )
+ if GitCommand(self, cmd, bare=True).Wait() != 0:
+ return None
+ return DownloadedChange(
+ self,
+ self.GetRevisionId(),
+ change_id,
+ patch_id,
+ self.bare_git.rev_parse("FETCH_HEAD"),
+ )
+
+ def DeleteWorktree(self, quiet=False, force=False):
+ """Delete the source checkout and any other housekeeping tasks.
+
+ This currently leaves behind the internal .repo/ cache state. This
+ helps when switching branches or manifest changes get reverted as we
+ don't have to redownload all the git objects. But we should do some GC
+ at some point.
+
+ Args:
+ quiet: Whether to hide normal messages.
+ force: Always delete tree even if dirty.
Returns:
- True if the checkout succeeded; False if it didn't; None if the branch
- didn't exist.
- """
- rev = R_HEADS + name
- head = self.work_git.GetHead()
- if head == rev:
- # Already on the branch
- #
- return True
+ True if the worktree was completely cleaned out.
+ """
+ if self.IsDirty():
+ if force:
+ print(
+ "warning: %s: Removing dirty project: uncommitted changes "
+ "lost." % (self.RelPath(local=False),),
+ file=sys.stderr,
+ )
+ else:
+ print(
+ "error: %s: Cannot remove project: uncommitted changes are "
+ "present.\n" % (self.RelPath(local=False),),
+ file=sys.stderr,
+ )
+ return False
- all_refs = self.bare_ref.all
- try:
- revid = all_refs[rev]
- except KeyError:
- # Branch does not exist in this project
- #
- return None
+ if not quiet:
+ print(
+ "%s: Deleting obsolete checkout." % (self.RelPath(local=False),)
+ )
- if head.startswith(R_HEADS):
- try:
- head = all_refs[head]
- except KeyError:
- head = None
+ # Unlock and delink from the main worktree. We don't use git's worktree
+ # remove because it will recursively delete projects -- we handle that
+ # ourselves below. https://crbug.com/git/48
+ if self.use_git_worktrees:
+ needle = platform_utils.realpath(self.gitdir)
+ # Find the git worktree commondir under .repo/worktrees/.
+ output = self.bare_git.worktree("list", "--porcelain").splitlines()[
+ 0
+ ]
+ assert output.startswith("worktree "), output
+ commondir = output[9:]
+ # Walk each of the git worktrees to see where they point.
+ configs = os.path.join(commondir, "worktrees")
+ for name in os.listdir(configs):
+ gitdir = os.path.join(configs, name, "gitdir")
+ with open(gitdir) as fp:
+ relpath = fp.read().strip()
+ # Resolve the checkout path and see if it matches this project.
+ fullpath = platform_utils.realpath(
+ os.path.join(configs, name, relpath)
+ )
+ if fullpath == needle:
+ platform_utils.rmtree(os.path.join(configs, name))
- if head == revid:
- # Same revision; just update HEAD to point to the new
- # target branch, but otherwise take no other action.
- #
- _lwrite(self.work_git.GetDotgitPath(subpath=HEAD),
- 'ref: %s%s\n' % (R_HEADS, name))
- return True
+ # Delete the .git directory first, so we're less likely to have a
+ # partially working git repository around. There shouldn't be any git
+ # projects here, so rmtree works.
- return GitCommand(self,
- ['checkout', name, '--'],
- capture_stdout=True,
- capture_stderr=True).Wait() == 0
-
- def AbandonBranch(self, name):
- """Destroy a local topic branch.
-
- Args:
- name: The name of the branch to abandon.
-
- Returns:
- True if the abandon succeeded; False if it didn't; None if the branch
- didn't exist.
- """
- rev = R_HEADS + name
- all_refs = self.bare_ref.all
- if rev not in all_refs:
- # Doesn't exist
- return None
-
- head = self.work_git.GetHead()
- if head == rev:
- # We can't destroy the branch while we are sitting
- # on it. Switch to a detached HEAD.
- #
- head = all_refs[head]
-
- revid = self.GetRevisionId(all_refs)
- if head == revid:
- _lwrite(self.work_git.GetDotgitPath(subpath=HEAD), '%s\n' % revid)
- else:
- self._Checkout(revid, quiet=True)
-
- return GitCommand(self,
- ['branch', '-D', name],
- capture_stdout=True,
- capture_stderr=True).Wait() == 0
-
- def PruneHeads(self):
- """Prune any topic branches already merged into upstream.
- """
- cb = self.CurrentBranch
- kill = []
- left = self._allrefs
- for name in left.keys():
- if name.startswith(R_HEADS):
- name = name[len(R_HEADS):]
- if cb is None or name != cb:
- kill.append(name)
-
- # Minor optimization: If there's nothing to prune, then don't try to read
- # any project state.
- if not kill and not cb:
- return []
-
- rev = self.GetRevisionId(left)
- if cb is not None \
- and not self._revlist(HEAD + '...' + rev) \
- and not self.IsDirty(consider_untracked=False):
- self.work_git.DetachHead(HEAD)
- kill.append(cb)
-
- if kill:
- old = self.bare_git.GetHead()
-
- try:
- self.bare_git.DetachHead(rev)
-
- b = ['branch', '-d']
- b.extend(kill)
- b = GitCommand(self, b, bare=True,
- capture_stdout=True,
- capture_stderr=True)
- b.Wait()
- finally:
- if ID_RE.match(old):
- self.bare_git.DetachHead(old)
- else:
- self.bare_git.SetHead(old)
- left = self._allrefs
-
- for branch in kill:
- if (R_HEADS + branch) not in left:
- self.CleanPublishedCache()
- break
-
- if cb and cb not in kill:
- kill.append(cb)
- kill.sort()
-
- kept = []
- for branch in kill:
- if R_HEADS + branch in left:
- branch = self.GetBranch(branch)
- base = branch.LocalMerge
- if not base:
- base = rev
- kept.append(ReviewableBranch(self, branch, base))
- return kept
-
-# Submodule Management ##
- def GetRegisteredSubprojects(self):
- result = []
-
- def rec(subprojects):
- if not subprojects:
- return
- result.extend(subprojects)
- for p in subprojects:
- rec(p.subprojects)
- rec(self.subprojects)
- return result
-
- def _GetSubmodules(self):
- # Unfortunately we cannot call `git submodule status --recursive` here
- # because the working tree might not exist yet, and it cannot be used
- # without a working tree in its current implementation.
-
- def get_submodules(gitdir, rev):
- # Parse .gitmodules for submodule sub_paths and sub_urls
- sub_paths, sub_urls = parse_gitmodules(gitdir, rev)
- if not sub_paths:
- return []
- # Run `git ls-tree` to read SHAs of submodule object, which happen to be
- # revision of submodule repository
- sub_revs = git_ls_tree(gitdir, rev, sub_paths)
- submodules = []
- for sub_path, sub_url in zip(sub_paths, sub_urls):
+ # Try to remove plain files first in case of git worktrees. If this
+ # fails for any reason, we'll fall back to rmtree, and that'll display
+ # errors if it can't remove things either.
try:
- sub_rev = sub_revs[sub_path]
- except KeyError:
- # Ignore non-exist submodules
- continue
- submodules.append((sub_rev, sub_path, sub_url))
- return submodules
+ platform_utils.remove(self.gitdir)
+ except OSError:
+ pass
+ try:
+ platform_utils.rmtree(self.gitdir)
+ except OSError as e:
+ if e.errno != errno.ENOENT:
+ print("error: %s: %s" % (self.gitdir, e), file=sys.stderr)
+ print(
+ "error: %s: Failed to delete obsolete checkout; remove "
+ "manually, then run `repo sync -l`."
+ % (self.RelPath(local=False),),
+ file=sys.stderr,
+ )
+ return False
- re_path = re.compile(r'^submodule\.(.+)\.path=(.*)$')
- re_url = re.compile(r'^submodule\.(.+)\.url=(.*)$')
+ # Delete everything under the worktree, except for directories that
+ # contain another git project.
+ dirs_to_remove = []
+ failed = False
+ for root, dirs, files in platform_utils.walk(self.worktree):
+ for f in files:
+ path = os.path.join(root, f)
+ try:
+ platform_utils.remove(path)
+ except OSError as e:
+ if e.errno != errno.ENOENT:
+ print(
+ "error: %s: Failed to remove: %s" % (path, e),
+ file=sys.stderr,
+ )
+ failed = True
+ dirs[:] = [
+ d
+ for d in dirs
+ if not os.path.lexists(os.path.join(root, d, ".git"))
+ ]
+ dirs_to_remove += [
+ os.path.join(root, d)
+ for d in dirs
+ if os.path.join(root, d) not in dirs_to_remove
+ ]
+ for d in reversed(dirs_to_remove):
+ if platform_utils.islink(d):
+ try:
+ platform_utils.remove(d)
+ except OSError as e:
+ if e.errno != errno.ENOENT:
+ print(
+ "error: %s: Failed to remove: %s" % (d, e),
+ file=sys.stderr,
+ )
+ failed = True
+ elif not platform_utils.listdir(d):
+ try:
+ platform_utils.rmdir(d)
+ except OSError as e:
+ if e.errno != errno.ENOENT:
+ print(
+ "error: %s: Failed to remove: %s" % (d, e),
+ file=sys.stderr,
+ )
+ failed = True
+ if failed:
+ print(
+ "error: %s: Failed to delete obsolete checkout."
+ % (self.RelPath(local=False),),
+ file=sys.stderr,
+ )
+ print(
+ " Remove manually, then run `repo sync -l`.",
+ file=sys.stderr,
+ )
+ return False
- def parse_gitmodules(gitdir, rev):
- cmd = ['cat-file', 'blob', '%s:.gitmodules' % rev]
- try:
- p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True,
- bare=True, gitdir=gitdir)
- except GitError:
- return [], []
- if p.Wait() != 0:
- return [], []
+ # Try deleting parent dirs if they are empty.
+ path = self.worktree
+ while path != self.manifest.topdir:
+ try:
+ platform_utils.rmdir(path)
+ except OSError as e:
+ if e.errno != errno.ENOENT:
+ break
+ path = os.path.dirname(path)
- gitmodules_lines = []
- fd, temp_gitmodules_path = tempfile.mkstemp()
- try:
- os.write(fd, p.stdout.encode('utf-8'))
- os.close(fd)
- cmd = ['config', '--file', temp_gitmodules_path, '--list']
- p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True,
- bare=True, gitdir=gitdir)
- if p.Wait() != 0:
- return [], []
- gitmodules_lines = p.stdout.split('\n')
- except GitError:
- return [], []
- finally:
- platform_utils.remove(temp_gitmodules_path)
+ return True
- names = set()
- paths = {}
- urls = {}
- for line in gitmodules_lines:
- if not line:
- continue
- m = re_path.match(line)
- if m:
- names.add(m.group(1))
- paths[m.group(1)] = m.group(2)
- continue
- m = re_url.match(line)
- if m:
- names.add(m.group(1))
- urls[m.group(1)] = m.group(2)
- continue
- names = sorted(names)
- return ([paths.get(name, '') for name in names],
- [urls.get(name, '') for name in names])
-
- def git_ls_tree(gitdir, rev, paths):
- cmd = ['ls-tree', rev, '--']
- cmd.extend(paths)
- try:
- p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True,
- bare=True, gitdir=gitdir)
- except GitError:
- return []
- if p.Wait() != 0:
- return []
- objects = {}
- for line in p.stdout.split('\n'):
- if not line.strip():
- continue
- object_rev, object_path = line.split()[2:4]
- objects[object_path] = object_rev
- return objects
-
- try:
- rev = self.GetRevisionId()
- except GitError:
- return []
- return get_submodules(self.gitdir, rev)
-
- def GetDerivedSubprojects(self):
- result = []
- if not self.Exists:
- # If git repo does not exist yet, querying its submodules will
- # mess up its states; so return here.
- return result
- for rev, path, url in self._GetSubmodules():
- name = self.manifest.GetSubprojectName(self, path)
- relpath, worktree, gitdir, objdir = \
- self.manifest.GetSubprojectPaths(self, name, path)
- project = self.manifest.paths.get(relpath)
- if project:
- result.extend(project.GetDerivedSubprojects())
- continue
-
- if url.startswith('..'):
- url = urllib.parse.urljoin("%s/" % self.remote.url, url)
- remote = RemoteSpec(self.remote.name,
- url=url,
- pushUrl=self.remote.pushUrl,
- review=self.remote.review,
- revision=self.remote.revision)
- subproject = Project(manifest=self.manifest,
- name=name,
- remote=remote,
- gitdir=gitdir,
- objdir=objdir,
- worktree=worktree,
- relpath=relpath,
- revisionExpr=rev,
- revisionId=rev,
- rebase=self.rebase,
- groups=self.groups,
- sync_c=self.sync_c,
- sync_s=self.sync_s,
- sync_tags=self.sync_tags,
- parent=self,
- is_derived=True)
- result.append(subproject)
- result.extend(subproject.GetDerivedSubprojects())
- return result
-
-# Direct Git Commands ##
- def EnableRepositoryExtension(self, key, value='true', version=1):
- """Enable git repository extension |key| with |value|.
-
- Args:
- key: The extension to enabled. Omit the "extensions." prefix.
- value: The value to use for the extension.
- version: The minimum git repository version needed.
- """
- # Make sure the git repo version is new enough already.
- found_version = self.config.GetInt('core.repositoryFormatVersion')
- if found_version is None:
- found_version = 0
- if found_version < version:
- self.config.SetString('core.repositoryFormatVersion', str(version))
-
- # Enable the extension!
- self.config.SetString('extensions.%s' % (key,), value)
-
- def ResolveRemoteHead(self, name=None):
- """Find out what the default branch (HEAD) points to.
-
- Normally this points to refs/heads/master, but projects are moving to main.
- Support whatever the server uses rather than hardcoding "master" ourselves.
- """
- if name is None:
- name = self.remote.name
-
- # The output will look like (NB: tabs are separators):
- # ref: refs/heads/master HEAD
- # 5f6803b100bb3cd0f534e96e88c91373e8ed1c44 HEAD
- output = self.bare_git.ls_remote('-q', '--symref', '--exit-code', name, 'HEAD')
-
- for line in output.splitlines():
- lhs, rhs = line.split('\t', 1)
- if rhs == 'HEAD' and lhs.startswith('ref:'):
- return lhs[4:].strip()
-
- return None
-
- def _CheckForImmutableRevision(self):
- try:
- # if revision (sha or tag) is not present then following function
- # throws an error.
- self.bare_git.rev_list('-1', '--missing=allow-any',
- '%s^0' % self.revisionExpr, '--')
- if self.upstream:
- rev = self.GetRemote().ToLocal(self.upstream)
- self.bare_git.rev_list('-1', '--missing=allow-any',
- '%s^0' % rev, '--')
- self.bare_git.merge_base('--is-ancestor', self.revisionExpr, rev)
- return True
- except GitError:
- # There is no such persistent revision. We have to fetch it.
- return False
-
- def _FetchArchive(self, tarpath, cwd=None):
- cmd = ['archive', '-v', '-o', tarpath]
- cmd.append('--remote=%s' % self.remote.url)
- cmd.append('--prefix=%s/' % self.RelPath(local=False))
- cmd.append(self.revisionExpr)
-
- command = GitCommand(self, cmd, cwd=cwd,
- capture_stdout=True,
- capture_stderr=True)
-
- if command.Wait() != 0:
- raise GitError('git archive %s: %s' % (self.name, command.stderr))
-
- def _RemoteFetch(self, name=None,
- current_branch_only=False,
- initial=False,
- quiet=False,
- verbose=False,
- output_redir=None,
- alt_dir=None,
- tags=True,
- prune=False,
- depth=None,
- submodules=False,
- ssh_proxy=None,
- force_sync=False,
- clone_filter=None,
- retry_fetches=2,
- retry_sleep_initial_sec=4.0,
- retry_exp_factor=2.0):
- is_sha1 = False
- tag_name = None
- # The depth should not be used when fetching to a mirror because
- # it will result in a shallow repository that cannot be cloned or
- # fetched from.
- # The repo project should also never be synced with partial depth.
- if self.manifest.IsMirror or self.relpath == '.repo/repo':
- depth = None
-
- if depth:
- current_branch_only = True
-
- if ID_RE.match(self.revisionExpr) is not None:
- is_sha1 = True
-
- if current_branch_only:
- if self.revisionExpr.startswith(R_TAGS):
- # This is a tag and its commit id should never change.
- tag_name = self.revisionExpr[len(R_TAGS):]
- elif self.upstream and self.upstream.startswith(R_TAGS):
- # This is a tag and its commit id should never change.
- tag_name = self.upstream[len(R_TAGS):]
-
- if is_sha1 or tag_name is not None:
- if self._CheckForImmutableRevision():
- if verbose:
- print('Skipped fetching project %s (already have persistent ref)'
- % self.name)
- return True
- if is_sha1 and not depth:
- # When syncing a specific commit and --depth is not set:
- # * if upstream is explicitly specified and is not a sha1, fetch only
- # upstream as users expect only upstream to be fetch.
- # Note: The commit might not be in upstream in which case the sync
- # will fail.
- # * otherwise, fetch all branches to make sure we end up with the
- # specific commit.
- if self.upstream:
- current_branch_only = not ID_RE.match(self.upstream)
- else:
- current_branch_only = False
-
- if not name:
- name = self.remote.name
-
- remote = self.GetRemote(name)
- if not remote.PreConnectFetch(ssh_proxy):
- ssh_proxy = None
-
- if initial:
- if alt_dir and 'objects' == os.path.basename(alt_dir):
- ref_dir = os.path.dirname(alt_dir)
- packed_refs = os.path.join(self.gitdir, 'packed-refs')
+ def StartBranch(self, name, branch_merge="", revision=None):
+ """Create a new branch off the manifest's revision."""
+ if not branch_merge:
+ branch_merge = self.revisionExpr
+ head = self.work_git.GetHead()
+ if head == (R_HEADS + name):
+ return True
all_refs = self.bare_ref.all
- ids = set(all_refs.values())
- tmp = set()
+ if R_HEADS + name in all_refs:
+ return GitCommand(self, ["checkout", "-q", name, "--"]).Wait() == 0
- for r, ref_id in GitRefs(ref_dir).all.items():
- if r not in all_refs:
- if r.startswith(R_TAGS) or remote.WritesTo(r):
- all_refs[r] = ref_id
- ids.add(ref_id)
- continue
+ branch = self.GetBranch(name)
+ branch.remote = self.GetRemote()
+ branch.merge = branch_merge
+ if not branch.merge.startswith("refs/") and not ID_RE.match(
+ branch_merge
+ ):
+ branch.merge = R_HEADS + branch_merge
- if ref_id in ids:
- continue
-
- r = 'refs/_alt/%s' % ref_id
- all_refs[r] = ref_id
- ids.add(ref_id)
- tmp.add(r)
-
- tmp_packed_lines = []
- old_packed_lines = []
-
- for r in sorted(all_refs):
- line = '%s %s\n' % (all_refs[r], r)
- tmp_packed_lines.append(line)
- if r not in tmp:
- old_packed_lines.append(line)
-
- tmp_packed = ''.join(tmp_packed_lines)
- old_packed = ''.join(old_packed_lines)
- _lwrite(packed_refs, tmp_packed)
- else:
- alt_dir = None
-
- cmd = ['fetch']
-
- if clone_filter:
- git_require((2, 19, 0), fail=True, msg='partial clones')
- cmd.append('--filter=%s' % clone_filter)
- self.EnableRepositoryExtension('partialclone', self.remote.name)
-
- if depth:
- cmd.append('--depth=%s' % depth)
- else:
- # If this repo has shallow objects, then we don't know which refs have
- # shallow objects or not. Tell git to unshallow all fetched refs. Don't
- # do this with projects that don't have shallow objects, since it is less
- # efficient.
- if os.path.exists(os.path.join(self.gitdir, 'shallow')):
- cmd.append('--depth=2147483647')
-
- if not verbose:
- cmd.append('--quiet')
- if not quiet and sys.stdout.isatty():
- cmd.append('--progress')
- if not self.worktree:
- cmd.append('--update-head-ok')
- cmd.append(name)
-
- if force_sync:
- cmd.append('--force')
-
- if prune:
- cmd.append('--prune')
-
- # Always pass something for --recurse-submodules, git with GIT_DIR behaves
- # incorrectly when not given `--recurse-submodules=no`. (b/218891912)
- cmd.append(f'--recurse-submodules={"on-demand" if submodules else "no"}')
-
- spec = []
- if not current_branch_only:
- # Fetch whole repo
- spec.append(str((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*')))
- elif tag_name is not None:
- spec.append('tag')
- spec.append(tag_name)
-
- if self.manifest.IsMirror and not current_branch_only:
- branch = None
- else:
- branch = self.revisionExpr
- if (not self.manifest.IsMirror and is_sha1 and depth
- and git_require((1, 8, 3))):
- # Shallow checkout of a specific commit, fetch from that commit and not
- # the heads only as the commit might be deeper in the history.
- spec.append(branch)
- if self.upstream:
- spec.append(self.upstream)
- else:
- if is_sha1:
- branch = self.upstream
- if branch is not None and branch.strip():
- if not branch.startswith('refs/'):
- branch = R_HEADS + branch
- spec.append(str((u'+%s:' % branch) + remote.ToLocal(branch)))
-
- # If mirroring repo and we cannot deduce the tag or branch to fetch, fetch
- # whole repo.
- if self.manifest.IsMirror and not spec:
- spec.append(str((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*')))
-
- # If using depth then we should not get all the tags since they may
- # be outside of the depth.
- if not tags or depth:
- cmd.append('--no-tags')
- else:
- cmd.append('--tags')
- spec.append(str((u'+refs/tags/*:') + remote.ToLocal('refs/tags/*')))
-
- cmd.extend(spec)
-
- # At least one retry minimum due to git remote prune.
- retry_fetches = max(retry_fetches, 2)
- retry_cur_sleep = retry_sleep_initial_sec
- ok = prune_tried = False
- for try_n in range(retry_fetches):
- gitcmd = GitCommand(
- self, cmd, bare=True, objdir=os.path.join(self.objdir, 'objects'),
- ssh_proxy=ssh_proxy,
- merge_output=True, capture_stdout=quiet or bool(output_redir))
- if gitcmd.stdout and not quiet and output_redir:
- output_redir.write(gitcmd.stdout)
- ret = gitcmd.Wait()
- if ret == 0:
- ok = True
- break
-
- # Retry later due to HTTP 429 Too Many Requests.
- elif (gitcmd.stdout and
- 'error:' in gitcmd.stdout and
- 'HTTP 429' in gitcmd.stdout):
- # Fallthru to sleep+retry logic at the bottom.
- pass
-
- # Try to prune remote branches once in case there are conflicts.
- # For example, if the remote had refs/heads/upstream, but deleted that and
- # now has refs/heads/upstream/foo.
- elif (gitcmd.stdout and
- 'error:' in gitcmd.stdout and
- 'git remote prune' in gitcmd.stdout and
- not prune_tried):
- prune_tried = True
- prunecmd = GitCommand(self, ['remote', 'prune', name], bare=True,
- ssh_proxy=ssh_proxy)
- ret = prunecmd.Wait()
- if ret:
- break
- print('retrying fetch after pruning remote branches', file=output_redir)
- # Continue right away so we don't sleep as we shouldn't need to.
- continue
- elif current_branch_only and is_sha1 and ret == 128:
- # Exit code 128 means "couldn't find the ref you asked for"; if we're
- # in sha1 mode, we just tried sync'ing from the upstream field; it
- # doesn't exist, thus abort the optimization attempt and do a full sync.
- break
- elif ret < 0:
- # Git died with a signal, exit immediately
- break
-
- # Figure out how long to sleep before the next attempt, if there is one.
- if not verbose and gitcmd.stdout:
- print('\n%s:\n%s' % (self.name, gitcmd.stdout), end='', file=output_redir)
- if try_n < retry_fetches - 1:
- print('%s: sleeping %s seconds before retrying' % (self.name, retry_cur_sleep),
- file=output_redir)
- time.sleep(retry_cur_sleep)
- retry_cur_sleep = min(retry_exp_factor * retry_cur_sleep,
- MAXIMUM_RETRY_SLEEP_SEC)
- retry_cur_sleep *= (1 - random.uniform(-RETRY_JITTER_PERCENT,
- RETRY_JITTER_PERCENT))
-
- if initial:
- if alt_dir:
- if old_packed != '':
- _lwrite(packed_refs, old_packed)
+ if revision is None:
+ revid = self.GetRevisionId(all_refs)
else:
- platform_utils.remove(packed_refs)
- self.bare_git.pack_refs('--all', '--prune')
+ revid = self.work_git.rev_parse(revision)
- if is_sha1 and current_branch_only:
- # We just synced the upstream given branch; verify we
- # got what we wanted, else trigger a second run of all
- # refs.
- if not self._CheckForImmutableRevision():
- # Sync the current branch only with depth set to None.
- # We always pass depth=None down to avoid infinite recursion.
- return self._RemoteFetch(
- name=name, quiet=quiet, verbose=verbose, output_redir=output_redir,
- current_branch_only=current_branch_only and depth,
- initial=False, alt_dir=alt_dir,
- depth=None, ssh_proxy=ssh_proxy, clone_filter=clone_filter)
-
- return ok
-
- def _ApplyCloneBundle(self, initial=False, quiet=False, verbose=False):
- if initial and (self.manifest.manifestProject.depth or self.clone_depth):
- return False
-
- remote = self.GetRemote()
- bundle_url = remote.url + '/clone.bundle'
- bundle_url = GitConfig.ForUser().UrlInsteadOf(bundle_url)
- if GetSchemeFromUrl(bundle_url) not in ('http', 'https',
- 'persistent-http',
- 'persistent-https'):
- return False
-
- bundle_dst = os.path.join(self.gitdir, 'clone.bundle')
- bundle_tmp = os.path.join(self.gitdir, 'clone.bundle.tmp')
-
- exist_dst = os.path.exists(bundle_dst)
- exist_tmp = os.path.exists(bundle_tmp)
-
- if not initial and not exist_dst and not exist_tmp:
- return False
-
- if not exist_dst:
- exist_dst = self._FetchBundle(bundle_url, bundle_tmp, bundle_dst, quiet,
- verbose)
- if not exist_dst:
- return False
-
- cmd = ['fetch']
- if not verbose:
- cmd.append('--quiet')
- if not quiet and sys.stdout.isatty():
- cmd.append('--progress')
- if not self.worktree:
- cmd.append('--update-head-ok')
- cmd.append(bundle_dst)
- for f in remote.fetch:
- cmd.append(str(f))
- cmd.append('+refs/tags/*:refs/tags/*')
-
- ok = GitCommand(
- self, cmd, bare=True, objdir=os.path.join(self.objdir, 'objects')).Wait() == 0
- platform_utils.remove(bundle_dst, missing_ok=True)
- platform_utils.remove(bundle_tmp, missing_ok=True)
- return ok
-
- def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet, verbose):
- platform_utils.remove(dstPath, missing_ok=True)
-
- cmd = ['curl', '--fail', '--output', tmpPath, '--netrc', '--location']
- if quiet:
- cmd += ['--silent', '--show-error']
- if os.path.exists(tmpPath):
- size = os.stat(tmpPath).st_size
- if size >= 1024:
- cmd += ['--continue-at', '%d' % (size,)]
- else:
- platform_utils.remove(tmpPath)
- with GetUrlCookieFile(srcUrl, quiet) as (cookiefile, proxy):
- if cookiefile:
- cmd += ['--cookie', cookiefile]
- if proxy:
- cmd += ['--proxy', proxy]
- elif 'http_proxy' in os.environ and 'darwin' == sys.platform:
- cmd += ['--proxy', os.environ['http_proxy']]
- if srcUrl.startswith('persistent-https'):
- srcUrl = 'http' + srcUrl[len('persistent-https'):]
- elif srcUrl.startswith('persistent-http'):
- srcUrl = 'http' + srcUrl[len('persistent-http'):]
- cmd += [srcUrl]
-
- proc = None
- with Trace('Fetching bundle: %s', ' '.join(cmd)):
- if verbose:
- print('%s: Downloading bundle: %s' % (self.name, srcUrl))
- stdout = None if verbose else subprocess.PIPE
- stderr = None if verbose else subprocess.STDOUT
- try:
- proc = subprocess.Popen(cmd, stdout=stdout, stderr=stderr)
- except OSError:
- return False
-
- (output, _) = proc.communicate()
- curlret = proc.returncode
-
- if curlret == 22:
- # From curl man page:
- # 22: HTTP page not retrieved. The requested url was not found or
- # returned another error with the HTTP error code being 400 or above.
- # This return code only appears if -f, --fail is used.
- if verbose:
- print('%s: Unable to retrieve clone.bundle; ignoring.' % self.name)
- if output:
- print('Curl output:\n%s' % output)
- return False
- elif curlret and not verbose and output:
- print('%s' % output, file=sys.stderr)
-
- if os.path.exists(tmpPath):
- if curlret == 0 and self._IsValidBundle(tmpPath, quiet):
- platform_utils.rename(tmpPath, dstPath)
- return True
- else:
- platform_utils.remove(tmpPath)
- return False
- else:
- return False
-
- def _IsValidBundle(self, path, quiet):
- try:
- with open(path, 'rb') as f:
- if f.read(16) == b'# v2 git bundle\n':
- return True
- else:
- if not quiet:
- print("Invalid clone.bundle file; ignoring.", file=sys.stderr)
- return False
- except OSError:
- return False
-
- def _Checkout(self, rev, quiet=False):
- cmd = ['checkout']
- if quiet:
- cmd.append('-q')
- cmd.append(rev)
- cmd.append('--')
- if GitCommand(self, cmd).Wait() != 0:
- if self._allrefs:
- raise GitError('%s checkout %s ' % (self.name, rev))
-
- def _CherryPick(self, rev, ffonly=False, record_origin=False):
- cmd = ['cherry-pick']
- if ffonly:
- cmd.append('--ff')
- if record_origin:
- cmd.append('-x')
- cmd.append(rev)
- cmd.append('--')
- if GitCommand(self, cmd).Wait() != 0:
- if self._allrefs:
- raise GitError('%s cherry-pick %s ' % (self.name, rev))
-
- def _LsRemote(self, refs):
- cmd = ['ls-remote', self.remote.name, refs]
- p = GitCommand(self, cmd, capture_stdout=True)
- if p.Wait() == 0:
- return p.stdout
- return None
-
- def _Revert(self, rev):
- cmd = ['revert']
- cmd.append('--no-edit')
- cmd.append(rev)
- cmd.append('--')
- if GitCommand(self, cmd).Wait() != 0:
- if self._allrefs:
- raise GitError('%s revert %s ' % (self.name, rev))
-
- def _ResetHard(self, rev, quiet=True):
- cmd = ['reset', '--hard']
- if quiet:
- cmd.append('-q')
- cmd.append(rev)
- if GitCommand(self, cmd).Wait() != 0:
- raise GitError('%s reset --hard %s ' % (self.name, rev))
-
- def _SyncSubmodules(self, quiet=True):
- cmd = ['submodule', 'update', '--init', '--recursive']
- if quiet:
- cmd.append('-q')
- if GitCommand(self, cmd).Wait() != 0:
- raise GitError('%s submodule update --init --recursive ' % self.name)
-
- def _Rebase(self, upstream, onto=None):
- cmd = ['rebase']
- if onto is not None:
- cmd.extend(['--onto', onto])
- cmd.append(upstream)
- if GitCommand(self, cmd).Wait() != 0:
- raise GitError('%s rebase %s ' % (self.name, upstream))
-
- def _FastForward(self, head, ffonly=False):
- cmd = ['merge', '--no-stat', head]
- if ffonly:
- cmd.append("--ff-only")
- if GitCommand(self, cmd).Wait() != 0:
- raise GitError('%s merge %s ' % (self.name, head))
-
- def _InitGitDir(self, mirror_git=None, force_sync=False, quiet=False):
- init_git_dir = not os.path.exists(self.gitdir)
- init_obj_dir = not os.path.exists(self.objdir)
- try:
- # Initialize the bare repository, which contains all of the objects.
- if init_obj_dir:
- os.makedirs(self.objdir)
- self.bare_objdir.init()
-
- self._UpdateHooks(quiet=quiet)
-
- if self.use_git_worktrees:
- # Enable per-worktree config file support if possible. This is more a
- # nice-to-have feature for users rather than a hard requirement.
- if git_require((2, 20, 0)):
- self.EnableRepositoryExtension('worktreeConfig')
-
- # If we have a separate directory to hold refs, initialize it as well.
- if self.objdir != self.gitdir:
- if init_git_dir:
- os.makedirs(self.gitdir)
-
- if init_obj_dir or init_git_dir:
- self._ReferenceGitDir(self.objdir, self.gitdir, copy_all=True)
- try:
- self._CheckDirReference(self.objdir, self.gitdir)
- except GitError as e:
- if force_sync:
- print("Retrying clone after deleting %s" %
- self.gitdir, file=sys.stderr)
+ if head.startswith(R_HEADS):
try:
- platform_utils.rmtree(platform_utils.realpath(self.gitdir))
- if self.worktree and os.path.exists(platform_utils.realpath
- (self.worktree)):
- platform_utils.rmtree(platform_utils.realpath(self.worktree))
- return self._InitGitDir(mirror_git=mirror_git, force_sync=False,
- quiet=quiet)
- except Exception:
- raise e
- raise e
+ head = all_refs[head]
+ except KeyError:
+ head = None
+ if revid and head and revid == head:
+ ref = R_HEADS + name
+ self.work_git.update_ref(ref, revid)
+ self.work_git.symbolic_ref(HEAD, ref)
+ branch.Save()
+ return True
- if init_git_dir:
- mp = self.manifest.manifestProject
- ref_dir = mp.reference or ''
+ if (
+ GitCommand(
+ self, ["checkout", "-q", "-b", branch.name, revid]
+ ).Wait()
+ == 0
+ ):
+ branch.Save()
+ return True
+ return False
- def _expanded_ref_dirs():
- """Iterate through the possible git reference directory paths."""
- name = self.name + '.git'
- yield mirror_git or os.path.join(ref_dir, name)
- for prefix in '', self.remote.name:
- yield os.path.join(ref_dir, '.repo', 'project-objects', prefix, name)
- yield os.path.join(ref_dir, '.repo', 'worktrees', prefix, name)
+ def CheckoutBranch(self, name):
+ """Checkout a local topic branch.
- if ref_dir or mirror_git:
- found_ref_dir = None
- for path in _expanded_ref_dirs():
- if os.path.exists(path):
- found_ref_dir = path
- break
- ref_dir = found_ref_dir
+ Args:
+ name: The name of the branch to checkout.
- if ref_dir:
- if not os.path.isabs(ref_dir):
- # The alternate directory is relative to the object database.
- ref_dir = os.path.relpath(ref_dir,
- os.path.join(self.objdir, 'objects'))
- _lwrite(os.path.join(self.objdir, 'objects/info/alternates'),
- os.path.join(ref_dir, 'objects') + '\n')
+ Returns:
+ True if the checkout succeeded; False if it didn't; None if the
+ branch didn't exist.
+ """
+ rev = R_HEADS + name
+ head = self.work_git.GetHead()
+ if head == rev:
+ # Already on the branch.
+ return True
- m = self.manifest.manifestProject.config
- for key in ['user.name', 'user.email']:
- if m.Has(key, include_defaults=False):
- self.config.SetString(key, m.GetString(key))
- if not self.manifest.EnableGitLfs:
- self.config.SetString('filter.lfs.smudge', 'git-lfs smudge --skip -- %f')
- self.config.SetString('filter.lfs.process', 'git-lfs filter-process --skip')
- self.config.SetBoolean('core.bare', True if self.manifest.IsMirror else None)
- except Exception:
- if init_obj_dir and os.path.exists(self.objdir):
- platform_utils.rmtree(self.objdir)
- if init_git_dir and os.path.exists(self.gitdir):
- platform_utils.rmtree(self.gitdir)
- raise
+ all_refs = self.bare_ref.all
+ try:
+ revid = all_refs[rev]
+ except KeyError:
+ # Branch does not exist in this project.
+ return None
- def _UpdateHooks(self, quiet=False):
- if os.path.exists(self.objdir):
- self._InitHooks(quiet=quiet)
+ if head.startswith(R_HEADS):
+ try:
+ head = all_refs[head]
+ except KeyError:
+ head = None
- def _InitHooks(self, quiet=False):
- hooks = platform_utils.realpath(os.path.join(self.objdir, 'hooks'))
- if not os.path.exists(hooks):
- os.makedirs(hooks)
+ if head == revid:
+ # Same revision; just update HEAD to point to the new
+ # target branch, but otherwise take no other action.
+ _lwrite(
+ self.work_git.GetDotgitPath(subpath=HEAD),
+ "ref: %s%s\n" % (R_HEADS, name),
+ )
+ return True
- # Delete sample hooks. They're noise.
- for hook in glob.glob(os.path.join(hooks, '*.sample')):
- try:
- platform_utils.remove(hook, missing_ok=True)
- except PermissionError:
- pass
+ return (
+ GitCommand(
+ self,
+ ["checkout", name, "--"],
+ capture_stdout=True,
+ capture_stderr=True,
+ ).Wait()
+ == 0
+ )
- for stock_hook in _ProjectHooks():
- name = os.path.basename(stock_hook)
+ def AbandonBranch(self, name):
+ """Destroy a local topic branch.
- if name in ('commit-msg',) and not self.remote.review \
- and self is not self.manifest.manifestProject:
- # Don't install a Gerrit Code Review hook if this
- # project does not appear to use it for reviews.
- #
- # Since the manifest project is one of those, but also
- # managed through gerrit, it's excluded
- continue
+ Args:
+ name: The name of the branch to abandon.
- dst = os.path.join(hooks, name)
- if platform_utils.islink(dst):
- continue
- if os.path.exists(dst):
- # If the files are the same, we'll leave it alone. We create symlinks
- # below by default but fallback to hardlinks if the OS blocks them.
- # So if we're here, it's probably because we made a hardlink below.
- if not filecmp.cmp(stock_hook, dst, shallow=False):
- if not quiet:
- _warn("%s: Not replacing locally modified %s hook",
- self.RelPath(local=False), name)
- continue
- try:
- platform_utils.symlink(
- os.path.relpath(stock_hook, os.path.dirname(dst)), dst)
- except OSError as e:
- if e.errno == errno.EPERM:
- try:
- os.link(stock_hook, dst)
- except OSError:
- raise GitError(self._get_symlink_error_message())
+ Returns:
+ True if the abandon succeeded; False if it didn't; None if the
+ branch didn't exist.
+ """
+ rev = R_HEADS + name
+ all_refs = self.bare_ref.all
+ if rev not in all_refs:
+ # Doesn't exist
+ return None
+
+ head = self.work_git.GetHead()
+ if head == rev:
+ # We can't destroy the branch while we are sitting
+ # on it. Switch to a detached HEAD.
+ head = all_refs[head]
+
+ revid = self.GetRevisionId(all_refs)
+ if head == revid:
+ _lwrite(
+ self.work_git.GetDotgitPath(subpath=HEAD), "%s\n" % revid
+ )
+ else:
+ self._Checkout(revid, quiet=True)
+
+ return (
+ GitCommand(
+ self,
+ ["branch", "-D", name],
+ capture_stdout=True,
+ capture_stderr=True,
+ ).Wait()
+ == 0
+ )
+
+ def PruneHeads(self):
+ """Prune any topic branches already merged into upstream."""
+ cb = self.CurrentBranch
+ kill = []
+ left = self._allrefs
+ for name in left.keys():
+ if name.startswith(R_HEADS):
+ name = name[len(R_HEADS) :]
+ if cb is None or name != cb:
+ kill.append(name)
+
+ # Minor optimization: If there's nothing to prune, then don't try to
+ # read any project state.
+ if not kill and not cb:
+ return []
+
+ rev = self.GetRevisionId(left)
+ if (
+ cb is not None
+ and not self._revlist(HEAD + "..." + rev)
+ and not self.IsDirty(consider_untracked=False)
+ ):
+ self.work_git.DetachHead(HEAD)
+ kill.append(cb)
+
+ if kill:
+ old = self.bare_git.GetHead()
+
+ try:
+ self.bare_git.DetachHead(rev)
+
+ b = ["branch", "-d"]
+ b.extend(kill)
+ b = GitCommand(
+ self, b, bare=True, capture_stdout=True, capture_stderr=True
+ )
+ b.Wait()
+ finally:
+ if ID_RE.match(old):
+ self.bare_git.DetachHead(old)
+ else:
+ self.bare_git.SetHead(old)
+ left = self._allrefs
+
+ for branch in kill:
+ if (R_HEADS + branch) not in left:
+ self.CleanPublishedCache()
+ break
+
+ if cb and cb not in kill:
+ kill.append(cb)
+ kill.sort()
+
+ kept = []
+ for branch in kill:
+ if R_HEADS + branch in left:
+ branch = self.GetBranch(branch)
+ base = branch.LocalMerge
+ if not base:
+ base = rev
+ kept.append(ReviewableBranch(self, branch, base))
+ return kept
+
+ def GetRegisteredSubprojects(self):
+ result = []
+
+ def rec(subprojects):
+ if not subprojects:
+ return
+ result.extend(subprojects)
+ for p in subprojects:
+ rec(p.subprojects)
+
+ rec(self.subprojects)
+ return result
+
+ def _GetSubmodules(self):
+ # Unfortunately we cannot call `git submodule status --recursive` here
+ # because the working tree might not exist yet, and it cannot be used
+ # without a working tree in its current implementation.
+
+ def get_submodules(gitdir, rev):
+ # Parse .gitmodules for submodule sub_paths and sub_urls.
+ sub_paths, sub_urls = parse_gitmodules(gitdir, rev)
+ if not sub_paths:
+ return []
+ # Run `git ls-tree` to read SHAs of submodule object, which happen
+ # to be revision of submodule repository.
+ sub_revs = git_ls_tree(gitdir, rev, sub_paths)
+ submodules = []
+ for sub_path, sub_url in zip(sub_paths, sub_urls):
+ try:
+ sub_rev = sub_revs[sub_path]
+ except KeyError:
+ # Ignore non-exist submodules.
+ continue
+ submodules.append((sub_rev, sub_path, sub_url))
+ return submodules
+
+ re_path = re.compile(r"^submodule\.(.+)\.path=(.*)$")
+ re_url = re.compile(r"^submodule\.(.+)\.url=(.*)$")
+
+ def parse_gitmodules(gitdir, rev):
+ cmd = ["cat-file", "blob", "%s:.gitmodules" % rev]
+ try:
+ p = GitCommand(
+ None,
+ cmd,
+ capture_stdout=True,
+ capture_stderr=True,
+ bare=True,
+ gitdir=gitdir,
+ )
+ except GitError:
+ return [], []
+ if p.Wait() != 0:
+ return [], []
+
+ gitmodules_lines = []
+ fd, temp_gitmodules_path = tempfile.mkstemp()
+ try:
+ os.write(fd, p.stdout.encode("utf-8"))
+ os.close(fd)
+ cmd = ["config", "--file", temp_gitmodules_path, "--list"]
+ p = GitCommand(
+ None,
+ cmd,
+ capture_stdout=True,
+ capture_stderr=True,
+ bare=True,
+ gitdir=gitdir,
+ )
+ if p.Wait() != 0:
+ return [], []
+ gitmodules_lines = p.stdout.split("\n")
+ except GitError:
+ return [], []
+ finally:
+ platform_utils.remove(temp_gitmodules_path)
+
+ names = set()
+ paths = {}
+ urls = {}
+ for line in gitmodules_lines:
+ if not line:
+ continue
+ m = re_path.match(line)
+ if m:
+ names.add(m.group(1))
+ paths[m.group(1)] = m.group(2)
+ continue
+ m = re_url.match(line)
+ if m:
+ names.add(m.group(1))
+ urls[m.group(1)] = m.group(2)
+ continue
+ names = sorted(names)
+ return (
+ [paths.get(name, "") for name in names],
+ [urls.get(name, "") for name in names],
+ )
+
+ def git_ls_tree(gitdir, rev, paths):
+ cmd = ["ls-tree", rev, "--"]
+ cmd.extend(paths)
+ try:
+ p = GitCommand(
+ None,
+ cmd,
+ capture_stdout=True,
+ capture_stderr=True,
+ bare=True,
+ gitdir=gitdir,
+ )
+ except GitError:
+ return []
+ if p.Wait() != 0:
+ return []
+ objects = {}
+ for line in p.stdout.split("\n"):
+ if not line.strip():
+ continue
+ object_rev, object_path = line.split()[2:4]
+ objects[object_path] = object_rev
+ return objects
+
+ try:
+ rev = self.GetRevisionId()
+ except GitError:
+ return []
+ return get_submodules(self.gitdir, rev)
+
+ def GetDerivedSubprojects(self):
+ result = []
+ if not self.Exists:
+ # If git repo does not exist yet, querying its submodules will
+ # mess up its states; so return here.
+ return result
+ for rev, path, url in self._GetSubmodules():
+ name = self.manifest.GetSubprojectName(self, path)
+ (
+ relpath,
+ worktree,
+ gitdir,
+ objdir,
+ ) = self.manifest.GetSubprojectPaths(self, name, path)
+ project = self.manifest.paths.get(relpath)
+ if project:
+ result.extend(project.GetDerivedSubprojects())
+ continue
+
+ if url.startswith(".."):
+ url = urllib.parse.urljoin("%s/" % self.remote.url, url)
+ remote = RemoteSpec(
+ self.remote.name,
+ url=url,
+ pushUrl=self.remote.pushUrl,
+ review=self.remote.review,
+ revision=self.remote.revision,
+ )
+ subproject = Project(
+ manifest=self.manifest,
+ name=name,
+ remote=remote,
+ gitdir=gitdir,
+ objdir=objdir,
+ worktree=worktree,
+ relpath=relpath,
+ revisionExpr=rev,
+ revisionId=rev,
+ rebase=self.rebase,
+ groups=self.groups,
+ sync_c=self.sync_c,
+ sync_s=self.sync_s,
+ sync_tags=self.sync_tags,
+ parent=self,
+ is_derived=True,
+ )
+ result.append(subproject)
+ result.extend(subproject.GetDerivedSubprojects())
+ return result
+
+ def EnableRepositoryExtension(self, key, value="true", version=1):
+ """Enable git repository extension |key| with |value|.
+
+ Args:
+ key: The extension to enabled. Omit the "extensions." prefix.
+ value: The value to use for the extension.
+ version: The minimum git repository version needed.
+ """
+ # Make sure the git repo version is new enough already.
+ found_version = self.config.GetInt("core.repositoryFormatVersion")
+ if found_version is None:
+ found_version = 0
+ if found_version < version:
+ self.config.SetString("core.repositoryFormatVersion", str(version))
+
+ # Enable the extension!
+ self.config.SetString("extensions.%s" % (key,), value)
+
+ def ResolveRemoteHead(self, name=None):
+ """Find out what the default branch (HEAD) points to.
+
+ Normally this points to refs/heads/master, but projects are moving to
+ main. Support whatever the server uses rather than hardcoding "master"
+ ourselves.
+ """
+ if name is None:
+ name = self.remote.name
+
+ # The output will look like (NB: tabs are separators):
+ # ref: refs/heads/master HEAD
+ # 5f6803b100bb3cd0f534e96e88c91373e8ed1c44 HEAD
+ output = self.bare_git.ls_remote(
+ "-q", "--symref", "--exit-code", name, "HEAD"
+ )
+
+ for line in output.splitlines():
+ lhs, rhs = line.split("\t", 1)
+ if rhs == "HEAD" and lhs.startswith("ref:"):
+ return lhs[4:].strip()
+
+ return None
+
+ def _CheckForImmutableRevision(self):
+ try:
+ # if revision (sha or tag) is not present then following function
+ # throws an error.
+ self.bare_git.rev_list(
+ "-1", "--missing=allow-any", "%s^0" % self.revisionExpr, "--"
+ )
+ if self.upstream:
+ rev = self.GetRemote().ToLocal(self.upstream)
+ self.bare_git.rev_list(
+ "-1", "--missing=allow-any", "%s^0" % rev, "--"
+ )
+ self.bare_git.merge_base(
+ "--is-ancestor", self.revisionExpr, rev
+ )
+ return True
+ except GitError:
+ # There is no such persistent revision. We have to fetch it.
+ return False
+
+ def _FetchArchive(self, tarpath, cwd=None):
+ cmd = ["archive", "-v", "-o", tarpath]
+ cmd.append("--remote=%s" % self.remote.url)
+ cmd.append("--prefix=%s/" % self.RelPath(local=False))
+ cmd.append(self.revisionExpr)
+
+ command = GitCommand(
+ self, cmd, cwd=cwd, capture_stdout=True, capture_stderr=True
+ )
+
+ if command.Wait() != 0:
+ raise GitError("git archive %s: %s" % (self.name, command.stderr))
+
+ def _RemoteFetch(
+ self,
+ name=None,
+ current_branch_only=False,
+ initial=False,
+ quiet=False,
+ verbose=False,
+ output_redir=None,
+ alt_dir=None,
+ tags=True,
+ prune=False,
+ depth=None,
+ submodules=False,
+ ssh_proxy=None,
+ force_sync=False,
+ clone_filter=None,
+ retry_fetches=2,
+ retry_sleep_initial_sec=4.0,
+ retry_exp_factor=2.0,
+ ):
+ is_sha1 = False
+ tag_name = None
+ # The depth should not be used when fetching to a mirror because
+ # it will result in a shallow repository that cannot be cloned or
+ # fetched from.
+ # The repo project should also never be synced with partial depth.
+ if self.manifest.IsMirror or self.relpath == ".repo/repo":
+ depth = None
+
+ if depth:
+ current_branch_only = True
+
+ if ID_RE.match(self.revisionExpr) is not None:
+ is_sha1 = True
+
+ if current_branch_only:
+ if self.revisionExpr.startswith(R_TAGS):
+ # This is a tag and its commit id should never change.
+ tag_name = self.revisionExpr[len(R_TAGS) :]
+ elif self.upstream and self.upstream.startswith(R_TAGS):
+ # This is a tag and its commit id should never change.
+ tag_name = self.upstream[len(R_TAGS) :]
+
+ if is_sha1 or tag_name is not None:
+ if self._CheckForImmutableRevision():
+ if verbose:
+ print(
+ "Skipped fetching project %s (already have "
+ "persistent ref)" % self.name
+ )
+ return True
+ if is_sha1 and not depth:
+ # When syncing a specific commit and --depth is not set:
+ # * if upstream is explicitly specified and is not a sha1, fetch
+ # only upstream as users expect only upstream to be fetch.
+ # Note: The commit might not be in upstream in which case the
+ # sync will fail.
+ # * otherwise, fetch all branches to make sure we end up with
+ # the specific commit.
+ if self.upstream:
+ current_branch_only = not ID_RE.match(self.upstream)
+ else:
+ current_branch_only = False
+
+ if not name:
+ name = self.remote.name
+
+ remote = self.GetRemote(name)
+ if not remote.PreConnectFetch(ssh_proxy):
+ ssh_proxy = None
+
+ if initial:
+ if alt_dir and "objects" == os.path.basename(alt_dir):
+ ref_dir = os.path.dirname(alt_dir)
+ packed_refs = os.path.join(self.gitdir, "packed-refs")
+
+ all_refs = self.bare_ref.all
+ ids = set(all_refs.values())
+ tmp = set()
+
+ for r, ref_id in GitRefs(ref_dir).all.items():
+ if r not in all_refs:
+ if r.startswith(R_TAGS) or remote.WritesTo(r):
+ all_refs[r] = ref_id
+ ids.add(ref_id)
+ continue
+
+ if ref_id in ids:
+ continue
+
+ r = "refs/_alt/%s" % ref_id
+ all_refs[r] = ref_id
+ ids.add(ref_id)
+ tmp.add(r)
+
+ tmp_packed_lines = []
+ old_packed_lines = []
+
+ for r in sorted(all_refs):
+ line = "%s %s\n" % (all_refs[r], r)
+ tmp_packed_lines.append(line)
+ if r not in tmp:
+ old_packed_lines.append(line)
+
+ tmp_packed = "".join(tmp_packed_lines)
+ old_packed = "".join(old_packed_lines)
+ _lwrite(packed_refs, tmp_packed)
+ else:
+ alt_dir = None
+
+ cmd = ["fetch"]
+
+ if clone_filter:
+ git_require((2, 19, 0), fail=True, msg="partial clones")
+ cmd.append("--filter=%s" % clone_filter)
+ self.EnableRepositoryExtension("partialclone", self.remote.name)
+
+ if depth:
+ cmd.append("--depth=%s" % depth)
else:
- raise
+ # If this repo has shallow objects, then we don't know which refs
+ # have shallow objects or not. Tell git to unshallow all fetched
+ # refs. Don't do this with projects that don't have shallow
+ # objects, since it is less efficient.
+ if os.path.exists(os.path.join(self.gitdir, "shallow")):
+ cmd.append("--depth=2147483647")
- def _InitRemote(self):
- if self.remote.url:
- remote = self.GetRemote()
- remote.url = self.remote.url
- remote.pushUrl = self.remote.pushUrl
- remote.review = self.remote.review
- remote.projectname = self.name
+ if not verbose:
+ cmd.append("--quiet")
+ if not quiet and sys.stdout.isatty():
+ cmd.append("--progress")
+ if not self.worktree:
+ cmd.append("--update-head-ok")
+ cmd.append(name)
- if self.worktree:
- remote.ResetFetch(mirror=False)
- else:
- remote.ResetFetch(mirror=True)
- remote.Save()
+ if force_sync:
+ cmd.append("--force")
- def _InitMRef(self):
- """Initialize the pseudo m/<manifest branch> ref."""
- if self.manifest.branch:
- if self.use_git_worktrees:
- # Set up the m/ space to point to the worktree-specific ref space.
- # We'll update the worktree-specific ref space on each checkout.
- ref = R_M + self.manifest.branch
- if not self.bare_ref.symref(ref):
- self.bare_git.symbolic_ref(
- '-m', 'redirecting to worktree scope',
- ref, R_WORKTREE_M + self.manifest.branch)
+ if prune:
+ cmd.append("--prune")
- # We can't update this ref with git worktrees until it exists.
- # We'll wait until the initial checkout to set it.
- if not os.path.exists(self.worktree):
- return
+ # Always pass something for --recurse-submodules, git with GIT_DIR
+ # behaves incorrectly when not given `--recurse-submodules=no`.
+ # (b/218891912)
+ cmd.append(
+ f'--recurse-submodules={"on-demand" if submodules else "no"}'
+ )
- base = R_WORKTREE_M
- active_git = self.work_git
+ spec = []
+ if not current_branch_only:
+ # Fetch whole repo.
+ spec.append(
+ str(("+refs/heads/*:") + remote.ToLocal("refs/heads/*"))
+ )
+ elif tag_name is not None:
+ spec.append("tag")
+ spec.append(tag_name)
- self._InitAnyMRef(HEAD, self.bare_git, detach=True)
- else:
- base = R_M
- active_git = self.bare_git
-
- self._InitAnyMRef(base + self.manifest.branch, active_git)
-
- def _InitMirrorHead(self):
- self._InitAnyMRef(HEAD, self.bare_git)
-
- def _InitAnyMRef(self, ref, active_git, detach=False):
- """Initialize |ref| in |active_git| to the value in the manifest.
-
- This points |ref| to the <project> setting in the manifest.
-
- Args:
- ref: The branch to update.
- active_git: The git repository to make updates in.
- detach: Whether to update target of symbolic refs, or overwrite the ref
- directly (and thus make it non-symbolic).
- """
- cur = self.bare_ref.symref(ref)
-
- if self.revisionId:
- if cur != '' or self.bare_ref.get(ref) != self.revisionId:
- msg = 'manifest set to %s' % self.revisionId
- dst = self.revisionId + '^0'
- active_git.UpdateRef(ref, dst, message=msg, detach=True)
- else:
- remote = self.GetRemote()
- dst = remote.ToLocal(self.revisionExpr)
- if cur != dst:
- msg = 'manifest set to %s' % self.revisionExpr
- if detach:
- active_git.UpdateRef(ref, dst, message=msg, detach=True)
+ if self.manifest.IsMirror and not current_branch_only:
+ branch = None
else:
- active_git.symbolic_ref('-m', msg, ref, dst)
-
- def _CheckDirReference(self, srcdir, destdir):
- # Git worktrees don't use symlinks to share at all.
- if self.use_git_worktrees:
- return
-
- for name in self.shareable_dirs:
- # Try to self-heal a bit in simple cases.
- dst_path = os.path.join(destdir, name)
- src_path = os.path.join(srcdir, name)
-
- dst = platform_utils.realpath(dst_path)
- if os.path.lexists(dst):
- src = platform_utils.realpath(src_path)
- # Fail if the links are pointing to the wrong place
- if src != dst:
- _error('%s is different in %s vs %s', name, destdir, srcdir)
- raise GitError('--force-sync not enabled; cannot overwrite a local '
- 'work tree. If you\'re comfortable with the '
- 'possibility of losing the work tree\'s git metadata,'
- ' use `repo sync --force-sync {0}` to '
- 'proceed.'.format(self.RelPath(local=False)))
-
- def _ReferenceGitDir(self, gitdir, dotgit, copy_all):
- """Update |dotgit| to reference |gitdir|, using symlinks where possible.
-
- Args:
- gitdir: The bare git repository. Must already be initialized.
- dotgit: The repository you would like to initialize.
- copy_all: If true, copy all remaining files from |gitdir| -> |dotgit|.
- This saves you the effort of initializing |dotgit| yourself.
- """
- symlink_dirs = self.shareable_dirs[:]
- to_symlink = symlink_dirs
-
- to_copy = []
- if copy_all:
- to_copy = platform_utils.listdir(gitdir)
-
- dotgit = platform_utils.realpath(dotgit)
- for name in set(to_copy).union(to_symlink):
- try:
- src = platform_utils.realpath(os.path.join(gitdir, name))
- dst = os.path.join(dotgit, name)
-
- if os.path.lexists(dst):
- continue
-
- # If the source dir doesn't exist, create an empty dir.
- if name in symlink_dirs and not os.path.lexists(src):
- os.makedirs(src)
-
- if name in to_symlink:
- platform_utils.symlink(
- os.path.relpath(src, os.path.dirname(dst)), dst)
- elif copy_all and not platform_utils.islink(dst):
- if platform_utils.isdir(src):
- shutil.copytree(src, dst)
- elif os.path.isfile(src):
- shutil.copy(src, dst)
-
- except OSError as e:
- if e.errno == errno.EPERM:
- raise DownloadError(self._get_symlink_error_message())
+ branch = self.revisionExpr
+ if (
+ not self.manifest.IsMirror
+ and is_sha1
+ and depth
+ and git_require((1, 8, 3))
+ ):
+ # Shallow checkout of a specific commit, fetch from that commit and
+ # not the heads only as the commit might be deeper in the history.
+ spec.append(branch)
+ if self.upstream:
+ spec.append(self.upstream)
else:
- raise
+ if is_sha1:
+ branch = self.upstream
+ if branch is not None and branch.strip():
+ if not branch.startswith("refs/"):
+ branch = R_HEADS + branch
+ spec.append(str(("+%s:" % branch) + remote.ToLocal(branch)))
- def _InitGitWorktree(self):
- """Init the project using git worktrees."""
- self.bare_git.worktree('prune')
- self.bare_git.worktree('add', '-ff', '--checkout', '--detach', '--lock',
- self.worktree, self.GetRevisionId())
+ # If mirroring repo and we cannot deduce the tag or branch to fetch,
+ # fetch whole repo.
+ if self.manifest.IsMirror and not spec:
+ spec.append(
+ str(("+refs/heads/*:") + remote.ToLocal("refs/heads/*"))
+ )
- # Rewrite the internal state files to use relative paths between the
- # checkouts & worktrees.
- dotgit = os.path.join(self.worktree, '.git')
- with open(dotgit, 'r') as fp:
- # Figure out the checkout->worktree path.
- setting = fp.read()
- assert setting.startswith('gitdir:')
- git_worktree_path = setting.split(':', 1)[1].strip()
- # Some platforms (e.g. Windows) won't let us update dotgit in situ because
- # of file permissions. Delete it and recreate it from scratch to avoid.
- platform_utils.remove(dotgit)
- # Use relative path from checkout->worktree & maintain Unix line endings
- # on all OS's to match git behavior.
- with open(dotgit, 'w', newline='\n') as fp:
- print('gitdir:', os.path.relpath(git_worktree_path, self.worktree),
- file=fp)
- # Use relative path from worktree->checkout & maintain Unix line endings
- # on all OS's to match git behavior.
- with open(os.path.join(git_worktree_path, 'gitdir'), 'w', newline='\n') as fp:
- print(os.path.relpath(dotgit, git_worktree_path), file=fp)
+ # If using depth then we should not get all the tags since they may
+ # be outside of the depth.
+ if not tags or depth:
+ cmd.append("--no-tags")
+ else:
+ cmd.append("--tags")
+ spec.append(str(("+refs/tags/*:") + remote.ToLocal("refs/tags/*")))
- self._InitMRef()
+ cmd.extend(spec)
- def _InitWorkTree(self, force_sync=False, submodules=False):
- """Setup the worktree .git path.
+ # At least one retry minimum due to git remote prune.
+ retry_fetches = max(retry_fetches, 2)
+ retry_cur_sleep = retry_sleep_initial_sec
+ ok = prune_tried = False
+ for try_n in range(retry_fetches):
+ gitcmd = GitCommand(
+ self,
+ cmd,
+ bare=True,
+ objdir=os.path.join(self.objdir, "objects"),
+ ssh_proxy=ssh_proxy,
+ merge_output=True,
+ capture_stdout=quiet or bool(output_redir),
+ )
+ if gitcmd.stdout and not quiet and output_redir:
+ output_redir.write(gitcmd.stdout)
+ ret = gitcmd.Wait()
+ if ret == 0:
+ ok = True
+ break
- This is the user-visible path like src/foo/.git/.
+ # Retry later due to HTTP 429 Too Many Requests.
+ elif (
+ gitcmd.stdout
+ and "error:" in gitcmd.stdout
+ and "HTTP 429" in gitcmd.stdout
+ ):
+ # Fallthru to sleep+retry logic at the bottom.
+ pass
- With non-git-worktrees, this will be a symlink to the .repo/projects/ path.
- With git-worktrees, this will be a .git file using "gitdir: ..." syntax.
+ # Try to prune remote branches once in case there are conflicts.
+ # For example, if the remote had refs/heads/upstream, but deleted
+ # that and now has refs/heads/upstream/foo.
+ elif (
+ gitcmd.stdout
+ and "error:" in gitcmd.stdout
+ and "git remote prune" in gitcmd.stdout
+ and not prune_tried
+ ):
+ prune_tried = True
+ prunecmd = GitCommand(
+ self,
+ ["remote", "prune", name],
+ bare=True,
+ ssh_proxy=ssh_proxy,
+ )
+ ret = prunecmd.Wait()
+ if ret:
+ break
+ print(
+ "retrying fetch after pruning remote branches",
+ file=output_redir,
+ )
+ # Continue right away so we don't sleep as we shouldn't need to.
+ continue
+ elif current_branch_only and is_sha1 and ret == 128:
+ # Exit code 128 means "couldn't find the ref you asked for"; if
+ # we're in sha1 mode, we just tried sync'ing from the upstream
+ # field; it doesn't exist, thus abort the optimization attempt
+ # and do a full sync.
+ break
+ elif ret < 0:
+ # Git died with a signal, exit immediately.
+ break
- Older checkouts had .git/ directories. If we see that, migrate it.
+ # Figure out how long to sleep before the next attempt, if there is
+ # one.
+ if not verbose and gitcmd.stdout:
+ print(
+ "\n%s:\n%s" % (self.name, gitcmd.stdout),
+ end="",
+ file=output_redir,
+ )
+ if try_n < retry_fetches - 1:
+ print(
+ "%s: sleeping %s seconds before retrying"
+ % (self.name, retry_cur_sleep),
+ file=output_redir,
+ )
+ time.sleep(retry_cur_sleep)
+ retry_cur_sleep = min(
+ retry_exp_factor * retry_cur_sleep, MAXIMUM_RETRY_SLEEP_SEC
+ )
+ retry_cur_sleep *= 1 - random.uniform(
+ -RETRY_JITTER_PERCENT, RETRY_JITTER_PERCENT
+ )
- This also handles changes in the manifest. Maybe this project was backed
- by "foo/bar" on the server, but now it's "new/foo/bar". We have to update
- the path we point to under .repo/projects/ to match.
- """
- dotgit = os.path.join(self.worktree, '.git')
+ if initial:
+ if alt_dir:
+ if old_packed != "":
+ _lwrite(packed_refs, old_packed)
+ else:
+ platform_utils.remove(packed_refs)
+ self.bare_git.pack_refs("--all", "--prune")
- # If using an old layout style (a directory), migrate it.
- if not platform_utils.islink(dotgit) and platform_utils.isdir(dotgit):
- self._MigrateOldWorkTreeGitDir(dotgit)
+ if is_sha1 and current_branch_only:
+ # We just synced the upstream given branch; verify we
+ # got what we wanted, else trigger a second run of all
+ # refs.
+ if not self._CheckForImmutableRevision():
+ # Sync the current branch only with depth set to None.
+ # We always pass depth=None down to avoid infinite recursion.
+ return self._RemoteFetch(
+ name=name,
+ quiet=quiet,
+ verbose=verbose,
+ output_redir=output_redir,
+ current_branch_only=current_branch_only and depth,
+ initial=False,
+ alt_dir=alt_dir,
+ depth=None,
+ ssh_proxy=ssh_proxy,
+ clone_filter=clone_filter,
+ )
- init_dotgit = not os.path.exists(dotgit)
- if self.use_git_worktrees:
- if init_dotgit:
- self._InitGitWorktree()
- self._CopyAndLinkFiles()
- else:
- if not init_dotgit:
- # See if the project has changed.
- if platform_utils.realpath(self.gitdir) != platform_utils.realpath(dotgit):
- platform_utils.remove(dotgit)
+ return ok
- if init_dotgit or not os.path.exists(dotgit):
- os.makedirs(self.worktree, exist_ok=True)
- platform_utils.symlink(os.path.relpath(self.gitdir, self.worktree), dotgit)
+ def _ApplyCloneBundle(self, initial=False, quiet=False, verbose=False):
+ if initial and (
+ self.manifest.manifestProject.depth or self.clone_depth
+ ):
+ return False
- if init_dotgit:
- _lwrite(os.path.join(dotgit, HEAD), '%s\n' % self.GetRevisionId())
+ remote = self.GetRemote()
+ bundle_url = remote.url + "/clone.bundle"
+ bundle_url = GitConfig.ForUser().UrlInsteadOf(bundle_url)
+ if GetSchemeFromUrl(bundle_url) not in (
+ "http",
+ "https",
+ "persistent-http",
+ "persistent-https",
+ ):
+ return False
- # Finish checking out the worktree.
- cmd = ['read-tree', '--reset', '-u', '-v', HEAD]
+ bundle_dst = os.path.join(self.gitdir, "clone.bundle")
+ bundle_tmp = os.path.join(self.gitdir, "clone.bundle.tmp")
+
+ exist_dst = os.path.exists(bundle_dst)
+ exist_tmp = os.path.exists(bundle_tmp)
+
+ if not initial and not exist_dst and not exist_tmp:
+ return False
+
+ if not exist_dst:
+ exist_dst = self._FetchBundle(
+ bundle_url, bundle_tmp, bundle_dst, quiet, verbose
+ )
+ if not exist_dst:
+ return False
+
+ cmd = ["fetch"]
+ if not verbose:
+ cmd.append("--quiet")
+ if not quiet and sys.stdout.isatty():
+ cmd.append("--progress")
+ if not self.worktree:
+ cmd.append("--update-head-ok")
+ cmd.append(bundle_dst)
+ for f in remote.fetch:
+ cmd.append(str(f))
+ cmd.append("+refs/tags/*:refs/tags/*")
+
+ ok = (
+ GitCommand(
+ self,
+ cmd,
+ bare=True,
+ objdir=os.path.join(self.objdir, "objects"),
+ ).Wait()
+ == 0
+ )
+ platform_utils.remove(bundle_dst, missing_ok=True)
+ platform_utils.remove(bundle_tmp, missing_ok=True)
+ return ok
+
+ def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet, verbose):
+ platform_utils.remove(dstPath, missing_ok=True)
+
+ cmd = ["curl", "--fail", "--output", tmpPath, "--netrc", "--location"]
+ if quiet:
+ cmd += ["--silent", "--show-error"]
+ if os.path.exists(tmpPath):
+ size = os.stat(tmpPath).st_size
+ if size >= 1024:
+ cmd += ["--continue-at", "%d" % (size,)]
+ else:
+ platform_utils.remove(tmpPath)
+ with GetUrlCookieFile(srcUrl, quiet) as (cookiefile, proxy):
+ if cookiefile:
+ cmd += ["--cookie", cookiefile]
+ if proxy:
+ cmd += ["--proxy", proxy]
+ elif "http_proxy" in os.environ and "darwin" == sys.platform:
+ cmd += ["--proxy", os.environ["http_proxy"]]
+ if srcUrl.startswith("persistent-https"):
+ srcUrl = "http" + srcUrl[len("persistent-https") :]
+ elif srcUrl.startswith("persistent-http"):
+ srcUrl = "http" + srcUrl[len("persistent-http") :]
+ cmd += [srcUrl]
+
+ proc = None
+ with Trace("Fetching bundle: %s", " ".join(cmd)):
+ if verbose:
+ print("%s: Downloading bundle: %s" % (self.name, srcUrl))
+ stdout = None if verbose else subprocess.PIPE
+ stderr = None if verbose else subprocess.STDOUT
+ try:
+ proc = subprocess.Popen(cmd, stdout=stdout, stderr=stderr)
+ except OSError:
+ return False
+
+ (output, _) = proc.communicate()
+ curlret = proc.returncode
+
+ if curlret == 22:
+ # From curl man page:
+ # 22: HTTP page not retrieved. The requested url was not found
+ # or returned another error with the HTTP error code being 400
+ # or above. This return code only appears if -f, --fail is used.
+ if verbose:
+ print(
+ "%s: Unable to retrieve clone.bundle; ignoring."
+ % self.name
+ )
+ if output:
+ print("Curl output:\n%s" % output)
+ return False
+ elif curlret and not verbose and output:
+ print("%s" % output, file=sys.stderr)
+
+ if os.path.exists(tmpPath):
+ if curlret == 0 and self._IsValidBundle(tmpPath, quiet):
+ platform_utils.rename(tmpPath, dstPath)
+ return True
+ else:
+ platform_utils.remove(tmpPath)
+ return False
+ else:
+ return False
+
+ def _IsValidBundle(self, path, quiet):
+ try:
+ with open(path, "rb") as f:
+ if f.read(16) == b"# v2 git bundle\n":
+ return True
+ else:
+ if not quiet:
+ print(
+ "Invalid clone.bundle file; ignoring.",
+ file=sys.stderr,
+ )
+ return False
+ except OSError:
+ return False
+
+ def _Checkout(self, rev, quiet=False):
+ cmd = ["checkout"]
+ if quiet:
+ cmd.append("-q")
+ cmd.append(rev)
+ cmd.append("--")
if GitCommand(self, cmd).Wait() != 0:
- raise GitError('Cannot initialize work tree for ' + self.name)
+ if self._allrefs:
+ raise GitError("%s checkout %s " % (self.name, rev))
- if submodules:
- self._SyncSubmodules(quiet=True)
- self._CopyAndLinkFiles()
+ def _CherryPick(self, rev, ffonly=False, record_origin=False):
+ cmd = ["cherry-pick"]
+ if ffonly:
+ cmd.append("--ff")
+ if record_origin:
+ cmd.append("-x")
+ cmd.append(rev)
+ cmd.append("--")
+ if GitCommand(self, cmd).Wait() != 0:
+ if self._allrefs:
+ raise GitError("%s cherry-pick %s " % (self.name, rev))
- @classmethod
- def _MigrateOldWorkTreeGitDir(cls, dotgit):
- """Migrate the old worktree .git/ dir style to a symlink.
+ def _LsRemote(self, refs):
+ cmd = ["ls-remote", self.remote.name, refs]
+ p = GitCommand(self, cmd, capture_stdout=True)
+ if p.Wait() == 0:
+ return p.stdout
+ return None
- This logic specifically only uses state from |dotgit| to figure out where to
- move content and not |self|. This way if the backing project also changed
- places, we only do the .git/ dir to .git symlink migration here. The path
- updates will happen independently.
- """
- # Figure out where in .repo/projects/ it's pointing to.
- if not os.path.islink(os.path.join(dotgit, 'refs')):
- raise GitError(f'{dotgit}: unsupported checkout state')
- gitdir = os.path.dirname(os.path.realpath(os.path.join(dotgit, 'refs')))
+ def _Revert(self, rev):
+ cmd = ["revert"]
+ cmd.append("--no-edit")
+ cmd.append(rev)
+ cmd.append("--")
+ if GitCommand(self, cmd).Wait() != 0:
+ if self._allrefs:
+ raise GitError("%s revert %s " % (self.name, rev))
- # Remove known symlink paths that exist in .repo/projects/.
- KNOWN_LINKS = {
- 'config', 'description', 'hooks', 'info', 'logs', 'objects',
- 'packed-refs', 'refs', 'rr-cache', 'shallow', 'svn',
- }
- # Paths that we know will be in both, but are safe to clobber in .repo/projects/.
- SAFE_TO_CLOBBER = {
- 'COMMIT_EDITMSG', 'FETCH_HEAD', 'HEAD', 'gc.log', 'gitk.cache', 'index',
- 'ORIG_HEAD',
- }
+ def _ResetHard(self, rev, quiet=True):
+ cmd = ["reset", "--hard"]
+ if quiet:
+ cmd.append("-q")
+ cmd.append(rev)
+ if GitCommand(self, cmd).Wait() != 0:
+ raise GitError("%s reset --hard %s " % (self.name, rev))
- # First see if we'd succeed before starting the migration.
- unknown_paths = []
- for name in platform_utils.listdir(dotgit):
- # Ignore all temporary/backup names. These are common with vim & emacs.
- if name.endswith('~') or (name[0] == '#' and name[-1] == '#'):
- continue
+ def _SyncSubmodules(self, quiet=True):
+ cmd = ["submodule", "update", "--init", "--recursive"]
+ if quiet:
+ cmd.append("-q")
+ if GitCommand(self, cmd).Wait() != 0:
+ raise GitError(
+ "%s submodule update --init --recursive " % self.name
+ )
- dotgit_path = os.path.join(dotgit, name)
- if name in KNOWN_LINKS:
- if not platform_utils.islink(dotgit_path):
- unknown_paths.append(f'{dotgit_path}: should be a symlink')
- else:
- gitdir_path = os.path.join(gitdir, name)
- if name not in SAFE_TO_CLOBBER and os.path.exists(gitdir_path):
- unknown_paths.append(f'{dotgit_path}: unknown file; please file a bug')
- if unknown_paths:
- raise GitError('Aborting migration: ' + '\n'.join(unknown_paths))
+ def _Rebase(self, upstream, onto=None):
+ cmd = ["rebase"]
+ if onto is not None:
+ cmd.extend(["--onto", onto])
+ cmd.append(upstream)
+ if GitCommand(self, cmd).Wait() != 0:
+ raise GitError("%s rebase %s " % (self.name, upstream))
- # Now walk the paths and sync the .git/ to .repo/projects/.
- for name in platform_utils.listdir(dotgit):
- dotgit_path = os.path.join(dotgit, name)
+ def _FastForward(self, head, ffonly=False):
+ cmd = ["merge", "--no-stat", head]
+ if ffonly:
+ cmd.append("--ff-only")
+ if GitCommand(self, cmd).Wait() != 0:
+ raise GitError("%s merge %s " % (self.name, head))
- # Ignore all temporary/backup names. These are common with vim & emacs.
- if name.endswith('~') or (name[0] == '#' and name[-1] == '#'):
- platform_utils.remove(dotgit_path)
- elif name in KNOWN_LINKS:
- platform_utils.remove(dotgit_path)
- else:
- gitdir_path = os.path.join(gitdir, name)
- platform_utils.remove(gitdir_path, missing_ok=True)
- platform_utils.rename(dotgit_path, gitdir_path)
+ def _InitGitDir(self, mirror_git=None, force_sync=False, quiet=False):
+ init_git_dir = not os.path.exists(self.gitdir)
+ init_obj_dir = not os.path.exists(self.objdir)
+ try:
+ # Initialize the bare repository, which contains all of the objects.
+ if init_obj_dir:
+ os.makedirs(self.objdir)
+ self.bare_objdir.init()
- # Now that the dir should be empty, clear it out, and symlink it over.
- platform_utils.rmdir(dotgit)
- platform_utils.symlink(os.path.relpath(gitdir, os.path.dirname(dotgit)), dotgit)
+ self._UpdateHooks(quiet=quiet)
- def _get_symlink_error_message(self):
- if platform_utils.isWindows():
- return ('Unable to create symbolic link. Please re-run the command as '
- 'Administrator, or see '
- 'https://github.com/git-for-windows/git/wiki/Symbolic-Links '
- 'for other options.')
- return 'filesystem must support symlinks'
+ if self.use_git_worktrees:
+ # Enable per-worktree config file support if possible. This
+ # is more a nice-to-have feature for users rather than a
+ # hard requirement.
+ if git_require((2, 20, 0)):
+ self.EnableRepositoryExtension("worktreeConfig")
- def _revlist(self, *args, **kw):
- a = []
- a.extend(args)
- a.append('--')
- return self.work_git.rev_list(*a, **kw)
+ # If we have a separate directory to hold refs, initialize it as
+ # well.
+ if self.objdir != self.gitdir:
+ if init_git_dir:
+ os.makedirs(self.gitdir)
- @property
- def _allrefs(self):
- return self.bare_ref.all
+ if init_obj_dir or init_git_dir:
+ self._ReferenceGitDir(
+ self.objdir, self.gitdir, copy_all=True
+ )
+ try:
+ self._CheckDirReference(self.objdir, self.gitdir)
+ except GitError as e:
+ if force_sync:
+ print(
+ "Retrying clone after deleting %s" % self.gitdir,
+ file=sys.stderr,
+ )
+ try:
+ platform_utils.rmtree(
+ platform_utils.realpath(self.gitdir)
+ )
+ if self.worktree and os.path.exists(
+ platform_utils.realpath(self.worktree)
+ ):
+ platform_utils.rmtree(
+ platform_utils.realpath(self.worktree)
+ )
+ return self._InitGitDir(
+ mirror_git=mirror_git,
+ force_sync=False,
+ quiet=quiet,
+ )
+ except Exception:
+ raise e
+ raise e
- def _getLogs(self, rev1, rev2, oneline=False, color=True, pretty_format=None):
- """Get logs between two revisions of this project."""
- comp = '..'
- if rev1:
- revs = [rev1]
- if rev2:
- revs.extend([comp, rev2])
- cmd = ['log', ''.join(revs)]
- out = DiffColoring(self.config)
- if out.is_on and color:
- cmd.append('--color')
- if pretty_format is not None:
- cmd.append('--pretty=format:%s' % pretty_format)
- if oneline:
- cmd.append('--oneline')
+ if init_git_dir:
+ mp = self.manifest.manifestProject
+ ref_dir = mp.reference or ""
- try:
- log = GitCommand(self, cmd, capture_stdout=True, capture_stderr=True)
- if log.Wait() == 0:
- return log.stdout
- except GitError:
- # worktree may not exist if groups changed for example. In that case,
- # try in gitdir instead.
- if not os.path.exists(self.worktree):
- return self.bare_git.log(*cmd[1:])
+ def _expanded_ref_dirs():
+ """Iterate through possible git reference dir paths."""
+ name = self.name + ".git"
+ yield mirror_git or os.path.join(ref_dir, name)
+ for prefix in "", self.remote.name:
+ yield os.path.join(
+ ref_dir, ".repo", "project-objects", prefix, name
+ )
+ yield os.path.join(
+ ref_dir, ".repo", "worktrees", prefix, name
+ )
+
+ if ref_dir or mirror_git:
+ found_ref_dir = None
+ for path in _expanded_ref_dirs():
+ if os.path.exists(path):
+ found_ref_dir = path
+ break
+ ref_dir = found_ref_dir
+
+ if ref_dir:
+ if not os.path.isabs(ref_dir):
+ # The alternate directory is relative to the object
+ # database.
+ ref_dir = os.path.relpath(
+ ref_dir, os.path.join(self.objdir, "objects")
+ )
+ _lwrite(
+ os.path.join(
+ self.objdir, "objects/info/alternates"
+ ),
+ os.path.join(ref_dir, "objects") + "\n",
+ )
+
+ m = self.manifest.manifestProject.config
+ for key in ["user.name", "user.email"]:
+ if m.Has(key, include_defaults=False):
+ self.config.SetString(key, m.GetString(key))
+ if not self.manifest.EnableGitLfs:
+ self.config.SetString(
+ "filter.lfs.smudge", "git-lfs smudge --skip -- %f"
+ )
+ self.config.SetString(
+ "filter.lfs.process", "git-lfs filter-process --skip"
+ )
+ self.config.SetBoolean(
+ "core.bare", True if self.manifest.IsMirror else None
+ )
+ except Exception:
+ if init_obj_dir and os.path.exists(self.objdir):
+ platform_utils.rmtree(self.objdir)
+ if init_git_dir and os.path.exists(self.gitdir):
+ platform_utils.rmtree(self.gitdir)
+ raise
+
+ def _UpdateHooks(self, quiet=False):
+ if os.path.exists(self.objdir):
+ self._InitHooks(quiet=quiet)
+
+ def _InitHooks(self, quiet=False):
+ hooks = platform_utils.realpath(os.path.join(self.objdir, "hooks"))
+ if not os.path.exists(hooks):
+ os.makedirs(hooks)
+
+ # Delete sample hooks. They're noise.
+ for hook in glob.glob(os.path.join(hooks, "*.sample")):
+ try:
+ platform_utils.remove(hook, missing_ok=True)
+ except PermissionError:
+ pass
+
+ for stock_hook in _ProjectHooks():
+ name = os.path.basename(stock_hook)
+
+ if (
+ name in ("commit-msg",)
+ and not self.remote.review
+ and self is not self.manifest.manifestProject
+ ):
+ # Don't install a Gerrit Code Review hook if this
+ # project does not appear to use it for reviews.
+ #
+ # Since the manifest project is one of those, but also
+ # managed through gerrit, it's excluded.
+ continue
+
+ dst = os.path.join(hooks, name)
+ if platform_utils.islink(dst):
+ continue
+ if os.path.exists(dst):
+ # If the files are the same, we'll leave it alone. We create
+ # symlinks below by default but fallback to hardlinks if the OS
+ # blocks them. So if we're here, it's probably because we made a
+ # hardlink below.
+ if not filecmp.cmp(stock_hook, dst, shallow=False):
+ if not quiet:
+ _warn(
+ "%s: Not replacing locally modified %s hook",
+ self.RelPath(local=False),
+ name,
+ )
+ continue
+ try:
+ platform_utils.symlink(
+ os.path.relpath(stock_hook, os.path.dirname(dst)), dst
+ )
+ except OSError as e:
+ if e.errno == errno.EPERM:
+ try:
+ os.link(stock_hook, dst)
+ except OSError:
+ raise GitError(self._get_symlink_error_message())
+ else:
+ raise
+
+ def _InitRemote(self):
+ if self.remote.url:
+ remote = self.GetRemote()
+ remote.url = self.remote.url
+ remote.pushUrl = self.remote.pushUrl
+ remote.review = self.remote.review
+ remote.projectname = self.name
+
+ if self.worktree:
+ remote.ResetFetch(mirror=False)
+ else:
+ remote.ResetFetch(mirror=True)
+ remote.Save()
+
+ def _InitMRef(self):
+ """Initialize the pseudo m/<manifest branch> ref."""
+ if self.manifest.branch:
+ if self.use_git_worktrees:
+ # Set up the m/ space to point to the worktree-specific ref
+ # space. We'll update the worktree-specific ref space on each
+ # checkout.
+ ref = R_M + self.manifest.branch
+ if not self.bare_ref.symref(ref):
+ self.bare_git.symbolic_ref(
+ "-m",
+ "redirecting to worktree scope",
+ ref,
+ R_WORKTREE_M + self.manifest.branch,
+ )
+
+ # We can't update this ref with git worktrees until it exists.
+ # We'll wait until the initial checkout to set it.
+ if not os.path.exists(self.worktree):
+ return
+
+ base = R_WORKTREE_M
+ active_git = self.work_git
+
+ self._InitAnyMRef(HEAD, self.bare_git, detach=True)
+ else:
+ base = R_M
+ active_git = self.bare_git
+
+ self._InitAnyMRef(base + self.manifest.branch, active_git)
+
+ def _InitMirrorHead(self):
+ self._InitAnyMRef(HEAD, self.bare_git)
+
+ def _InitAnyMRef(self, ref, active_git, detach=False):
+ """Initialize |ref| in |active_git| to the value in the manifest.
+
+ This points |ref| to the <project> setting in the manifest.
+
+ Args:
+ ref: The branch to update.
+ active_git: The git repository to make updates in.
+ detach: Whether to update target of symbolic refs, or overwrite the
+ ref directly (and thus make it non-symbolic).
+ """
+ cur = self.bare_ref.symref(ref)
+
+ if self.revisionId:
+ if cur != "" or self.bare_ref.get(ref) != self.revisionId:
+ msg = "manifest set to %s" % self.revisionId
+ dst = self.revisionId + "^0"
+ active_git.UpdateRef(ref, dst, message=msg, detach=True)
else:
- raise
- return None
+ remote = self.GetRemote()
+ dst = remote.ToLocal(self.revisionExpr)
+ if cur != dst:
+ msg = "manifest set to %s" % self.revisionExpr
+ if detach:
+ active_git.UpdateRef(ref, dst, message=msg, detach=True)
+ else:
+ active_git.symbolic_ref("-m", msg, ref, dst)
- def getAddedAndRemovedLogs(self, toProject, oneline=False, color=True,
- pretty_format=None):
- """Get the list of logs from this revision to given revisionId"""
- logs = {}
- selfId = self.GetRevisionId(self._allrefs)
- toId = toProject.GetRevisionId(toProject._allrefs)
+ def _CheckDirReference(self, srcdir, destdir):
+ # Git worktrees don't use symlinks to share at all.
+ if self.use_git_worktrees:
+ return
- logs['added'] = self._getLogs(selfId, toId, oneline=oneline, color=color,
- pretty_format=pretty_format)
- logs['removed'] = self._getLogs(toId, selfId, oneline=oneline, color=color,
- pretty_format=pretty_format)
- return logs
+ for name in self.shareable_dirs:
+ # Try to self-heal a bit in simple cases.
+ dst_path = os.path.join(destdir, name)
+ src_path = os.path.join(srcdir, name)
- class _GitGetByExec(object):
+ dst = platform_utils.realpath(dst_path)
+ if os.path.lexists(dst):
+ src = platform_utils.realpath(src_path)
+ # Fail if the links are pointing to the wrong place.
+ if src != dst:
+ _error("%s is different in %s vs %s", name, destdir, srcdir)
+ raise GitError(
+ "--force-sync not enabled; cannot overwrite a local "
+ "work tree. If you're comfortable with the "
+ "possibility of losing the work tree's git metadata,"
+ " use `repo sync --force-sync {0}` to "
+ "proceed.".format(self.RelPath(local=False))
+ )
- def __init__(self, project, bare, gitdir):
- self._project = project
- self._bare = bare
- self._gitdir = gitdir
+ def _ReferenceGitDir(self, gitdir, dotgit, copy_all):
+ """Update |dotgit| to reference |gitdir|, using symlinks where possible.
- # __getstate__ and __setstate__ are required for pickling because __getattr__ exists.
- def __getstate__(self):
- return (self._project, self._bare, self._gitdir)
+ Args:
+ gitdir: The bare git repository. Must already be initialized.
+ dotgit: The repository you would like to initialize.
+ copy_all: If true, copy all remaining files from |gitdir| ->
+ |dotgit|. This saves you the effort of initializing |dotgit|
+ yourself.
+ """
+ symlink_dirs = self.shareable_dirs[:]
+ to_symlink = symlink_dirs
- def __setstate__(self, state):
- self._project, self._bare, self._gitdir = state
+ to_copy = []
+ if copy_all:
+ to_copy = platform_utils.listdir(gitdir)
- def LsOthers(self):
- p = GitCommand(self._project,
- ['ls-files',
- '-z',
- '--others',
- '--exclude-standard'],
- bare=False,
- gitdir=self._gitdir,
- capture_stdout=True,
- capture_stderr=True)
- if p.Wait() == 0:
- out = p.stdout
- if out:
- # Backslash is not anomalous
- return out[:-1].split('\0')
- return []
+ dotgit = platform_utils.realpath(dotgit)
+ for name in set(to_copy).union(to_symlink):
+ try:
+ src = platform_utils.realpath(os.path.join(gitdir, name))
+ dst = os.path.join(dotgit, name)
- def DiffZ(self, name, *args):
- cmd = [name]
- cmd.append('-z')
- cmd.append('--ignore-submodules')
- cmd.extend(args)
- p = GitCommand(self._project,
- cmd,
- gitdir=self._gitdir,
- bare=False,
- capture_stdout=True,
- capture_stderr=True)
- p.Wait()
- r = {}
- out = p.stdout
- if out:
- out = iter(out[:-1].split('\0'))
- while out:
- try:
- info = next(out)
- path = next(out)
- except StopIteration:
- break
+ if os.path.lexists(dst):
+ continue
- class _Info(object):
+ # If the source dir doesn't exist, create an empty dir.
+ if name in symlink_dirs and not os.path.lexists(src):
+ os.makedirs(src)
- def __init__(self, path, omode, nmode, oid, nid, state):
- self.path = path
- self.src_path = None
- self.old_mode = omode
- self.new_mode = nmode
- self.old_id = oid
- self.new_id = nid
+ if name in to_symlink:
+ platform_utils.symlink(
+ os.path.relpath(src, os.path.dirname(dst)), dst
+ )
+ elif copy_all and not platform_utils.islink(dst):
+ if platform_utils.isdir(src):
+ shutil.copytree(src, dst)
+ elif os.path.isfile(src):
+ shutil.copy(src, dst)
- if len(state) == 1:
- self.status = state
- self.level = None
- else:
- self.status = state[:1]
- self.level = state[1:]
- while self.level.startswith('0'):
- self.level = self.level[1:]
+ except OSError as e:
+ if e.errno == errno.EPERM:
+ raise DownloadError(self._get_symlink_error_message())
+ else:
+ raise
- info = info[1:].split(' ')
- info = _Info(path, *info)
- if info.status in ('R', 'C'):
- info.src_path = info.path
- info.path = next(out)
- r[info.path] = info
- return r
+ def _InitGitWorktree(self):
+ """Init the project using git worktrees."""
+ self.bare_git.worktree("prune")
+ self.bare_git.worktree(
+ "add",
+ "-ff",
+ "--checkout",
+ "--detach",
+ "--lock",
+ self.worktree,
+ self.GetRevisionId(),
+ )
- def GetDotgitPath(self, subpath=None):
- """Return the full path to the .git dir.
-
- As a convenience, append |subpath| if provided.
- """
- if self._bare:
- dotgit = self._gitdir
- else:
- dotgit = os.path.join(self._project.worktree, '.git')
- if os.path.isfile(dotgit):
- # Git worktrees use a "gitdir:" syntax to point to the scratch space.
- with open(dotgit) as fp:
+ # Rewrite the internal state files to use relative paths between the
+ # checkouts & worktrees.
+ dotgit = os.path.join(self.worktree, ".git")
+ with open(dotgit, "r") as fp:
+ # Figure out the checkout->worktree path.
setting = fp.read()
- assert setting.startswith('gitdir:')
- gitdir = setting.split(':', 1)[1].strip()
- dotgit = os.path.normpath(os.path.join(self._project.worktree, gitdir))
+ assert setting.startswith("gitdir:")
+ git_worktree_path = setting.split(":", 1)[1].strip()
+ # Some platforms (e.g. Windows) won't let us update dotgit in situ
+ # because of file permissions. Delete it and recreate it from scratch
+ # to avoid.
+ platform_utils.remove(dotgit)
+ # Use relative path from checkout->worktree & maintain Unix line endings
+ # on all OS's to match git behavior.
+ with open(dotgit, "w", newline="\n") as fp:
+ print(
+ "gitdir:",
+ os.path.relpath(git_worktree_path, self.worktree),
+ file=fp,
+ )
+ # Use relative path from worktree->checkout & maintain Unix line endings
+ # on all OS's to match git behavior.
+ with open(
+ os.path.join(git_worktree_path, "gitdir"), "w", newline="\n"
+ ) as fp:
+ print(os.path.relpath(dotgit, git_worktree_path), file=fp)
- return dotgit if subpath is None else os.path.join(dotgit, subpath)
+ self._InitMRef()
- def GetHead(self):
- """Return the ref that HEAD points to."""
- path = self.GetDotgitPath(subpath=HEAD)
- try:
- with open(path) as fd:
- line = fd.readline()
- except IOError as e:
- raise NoManifestException(path, str(e))
- try:
- line = line.decode()
- except AttributeError:
- pass
- if line.startswith('ref: '):
- return line[5:-1]
- return line[:-1]
+ def _InitWorkTree(self, force_sync=False, submodules=False):
+ """Setup the worktree .git path.
- def SetHead(self, ref, message=None):
- cmdv = []
- if message is not None:
- cmdv.extend(['-m', message])
- cmdv.append(HEAD)
- cmdv.append(ref)
- self.symbolic_ref(*cmdv)
+ This is the user-visible path like src/foo/.git/.
- def DetachHead(self, new, message=None):
- cmdv = ['--no-deref']
- if message is not None:
- cmdv.extend(['-m', message])
- cmdv.append(HEAD)
- cmdv.append(new)
- self.update_ref(*cmdv)
+ With non-git-worktrees, this will be a symlink to the .repo/projects/
+ path. With git-worktrees, this will be a .git file using "gitdir: ..."
+ syntax.
- def UpdateRef(self, name, new, old=None,
- message=None,
- detach=False):
- cmdv = []
- if message is not None:
- cmdv.extend(['-m', message])
- if detach:
- cmdv.append('--no-deref')
- cmdv.append(name)
- cmdv.append(new)
- if old is not None:
- cmdv.append(old)
- self.update_ref(*cmdv)
+ Older checkouts had .git/ directories. If we see that, migrate it.
- def DeleteRef(self, name, old=None):
- if not old:
- old = self.rev_parse(name)
- self.update_ref('-d', name, old)
- self._project.bare_ref.deleted(name)
+ This also handles changes in the manifest. Maybe this project was
+ backed by "foo/bar" on the server, but now it's "new/foo/bar". We have
+ to update the path we point to under .repo/projects/ to match.
+ """
+ dotgit = os.path.join(self.worktree, ".git")
- def rev_list(self, *args, **kw):
- if 'format' in kw:
- cmdv = ['log', '--pretty=format:%s' % kw['format']]
- else:
- cmdv = ['rev-list']
- cmdv.extend(args)
- p = GitCommand(self._project,
- cmdv,
- bare=self._bare,
- gitdir=self._gitdir,
- capture_stdout=True,
- capture_stderr=True)
- if p.Wait() != 0:
- raise GitError('%s rev-list %s: %s' %
- (self._project.name, str(args), p.stderr))
- return p.stdout.splitlines()
+ # If using an old layout style (a directory), migrate it.
+ if not platform_utils.islink(dotgit) and platform_utils.isdir(dotgit):
+ self._MigrateOldWorkTreeGitDir(dotgit)
- def __getattr__(self, name):
- """Allow arbitrary git commands using pythonic syntax.
+ init_dotgit = not os.path.exists(dotgit)
+ if self.use_git_worktrees:
+ if init_dotgit:
+ self._InitGitWorktree()
+ self._CopyAndLinkFiles()
+ else:
+ if not init_dotgit:
+ # See if the project has changed.
+ if platform_utils.realpath(
+ self.gitdir
+ ) != platform_utils.realpath(dotgit):
+ platform_utils.remove(dotgit)
- This allows you to do things like:
- git_obj.rev_parse('HEAD')
+ if init_dotgit or not os.path.exists(dotgit):
+ os.makedirs(self.worktree, exist_ok=True)
+ platform_utils.symlink(
+ os.path.relpath(self.gitdir, self.worktree), dotgit
+ )
- Since we don't have a 'rev_parse' method defined, the __getattr__ will
- run. We'll replace the '_' with a '-' and try to run a git command.
- Any other positional arguments will be passed to the git command, and the
- following keyword arguments are supported:
- config: An optional dict of git config options to be passed with '-c'.
+ if init_dotgit:
+ _lwrite(
+ os.path.join(dotgit, HEAD), "%s\n" % self.GetRevisionId()
+ )
- Args:
- name: The name of the git command to call. Any '_' characters will
- be replaced with '-'.
+ # Finish checking out the worktree.
+ cmd = ["read-tree", "--reset", "-u", "-v", HEAD]
+ if GitCommand(self, cmd).Wait() != 0:
+ raise GitError(
+ "Cannot initialize work tree for " + self.name
+ )
- Returns:
- A callable object that will try to call git with the named command.
- """
- name = name.replace('_', '-')
+ if submodules:
+ self._SyncSubmodules(quiet=True)
+ self._CopyAndLinkFiles()
- def runner(*args, **kwargs):
- cmdv = []
- config = kwargs.pop('config', None)
- for k in kwargs:
- raise TypeError('%s() got an unexpected keyword argument %r'
- % (name, k))
- if config is not None:
- for k, v in config.items():
- cmdv.append('-c')
- cmdv.append('%s=%s' % (k, v))
- cmdv.append(name)
- cmdv.extend(args)
- p = GitCommand(self._project,
- cmdv,
- bare=self._bare,
- gitdir=self._gitdir,
- capture_stdout=True,
- capture_stderr=True)
- if p.Wait() != 0:
- raise GitError('%s %s: %s' %
- (self._project.name, name, p.stderr))
- r = p.stdout
- if r.endswith('\n') and r.index('\n') == len(r) - 1:
- return r[:-1]
- return r
- return runner
+ @classmethod
+ def _MigrateOldWorkTreeGitDir(cls, dotgit):
+ """Migrate the old worktree .git/ dir style to a symlink.
+
+ This logic specifically only uses state from |dotgit| to figure out
+ where to move content and not |self|. This way if the backing project
+ also changed places, we only do the .git/ dir to .git symlink migration
+ here. The path updates will happen independently.
+ """
+ # Figure out where in .repo/projects/ it's pointing to.
+ if not os.path.islink(os.path.join(dotgit, "refs")):
+ raise GitError(f"{dotgit}: unsupported checkout state")
+ gitdir = os.path.dirname(os.path.realpath(os.path.join(dotgit, "refs")))
+
+ # Remove known symlink paths that exist in .repo/projects/.
+ KNOWN_LINKS = {
+ "config",
+ "description",
+ "hooks",
+ "info",
+ "logs",
+ "objects",
+ "packed-refs",
+ "refs",
+ "rr-cache",
+ "shallow",
+ "svn",
+ }
+ # Paths that we know will be in both, but are safe to clobber in
+ # .repo/projects/.
+ SAFE_TO_CLOBBER = {
+ "COMMIT_EDITMSG",
+ "FETCH_HEAD",
+ "HEAD",
+ "gc.log",
+ "gitk.cache",
+ "index",
+ "ORIG_HEAD",
+ }
+
+ # First see if we'd succeed before starting the migration.
+ unknown_paths = []
+ for name in platform_utils.listdir(dotgit):
+ # Ignore all temporary/backup names. These are common with vim &
+ # emacs.
+ if name.endswith("~") or (name[0] == "#" and name[-1] == "#"):
+ continue
+
+ dotgit_path = os.path.join(dotgit, name)
+ if name in KNOWN_LINKS:
+ if not platform_utils.islink(dotgit_path):
+ unknown_paths.append(f"{dotgit_path}: should be a symlink")
+ else:
+ gitdir_path = os.path.join(gitdir, name)
+ if name not in SAFE_TO_CLOBBER and os.path.exists(gitdir_path):
+ unknown_paths.append(
+ f"{dotgit_path}: unknown file; please file a bug"
+ )
+ if unknown_paths:
+ raise GitError("Aborting migration: " + "\n".join(unknown_paths))
+
+ # Now walk the paths and sync the .git/ to .repo/projects/.
+ for name in platform_utils.listdir(dotgit):
+ dotgit_path = os.path.join(dotgit, name)
+
+ # Ignore all temporary/backup names. These are common with vim &
+ # emacs.
+ if name.endswith("~") or (name[0] == "#" and name[-1] == "#"):
+ platform_utils.remove(dotgit_path)
+ elif name in KNOWN_LINKS:
+ platform_utils.remove(dotgit_path)
+ else:
+ gitdir_path = os.path.join(gitdir, name)
+ platform_utils.remove(gitdir_path, missing_ok=True)
+ platform_utils.rename(dotgit_path, gitdir_path)
+
+ # Now that the dir should be empty, clear it out, and symlink it over.
+ platform_utils.rmdir(dotgit)
+ platform_utils.symlink(
+ os.path.relpath(gitdir, os.path.dirname(dotgit)), dotgit
+ )
+
+ def _get_symlink_error_message(self):
+ if platform_utils.isWindows():
+ return (
+ "Unable to create symbolic link. Please re-run the command as "
+ "Administrator, or see "
+ "https://github.com/git-for-windows/git/wiki/Symbolic-Links "
+ "for other options."
+ )
+ return "filesystem must support symlinks"
+
+ def _revlist(self, *args, **kw):
+ a = []
+ a.extend(args)
+ a.append("--")
+ return self.work_git.rev_list(*a, **kw)
+
+ @property
+ def _allrefs(self):
+ return self.bare_ref.all
+
+ def _getLogs(
+ self, rev1, rev2, oneline=False, color=True, pretty_format=None
+ ):
+ """Get logs between two revisions of this project."""
+ comp = ".."
+ if rev1:
+ revs = [rev1]
+ if rev2:
+ revs.extend([comp, rev2])
+ cmd = ["log", "".join(revs)]
+ out = DiffColoring(self.config)
+ if out.is_on and color:
+ cmd.append("--color")
+ if pretty_format is not None:
+ cmd.append("--pretty=format:%s" % pretty_format)
+ if oneline:
+ cmd.append("--oneline")
+
+ try:
+ log = GitCommand(
+ self, cmd, capture_stdout=True, capture_stderr=True
+ )
+ if log.Wait() == 0:
+ return log.stdout
+ except GitError:
+ # worktree may not exist if groups changed for example. In that
+ # case, try in gitdir instead.
+ if not os.path.exists(self.worktree):
+ return self.bare_git.log(*cmd[1:])
+ else:
+ raise
+ return None
+
+ def getAddedAndRemovedLogs(
+ self, toProject, oneline=False, color=True, pretty_format=None
+ ):
+ """Get the list of logs from this revision to given revisionId"""
+ logs = {}
+ selfId = self.GetRevisionId(self._allrefs)
+ toId = toProject.GetRevisionId(toProject._allrefs)
+
+ logs["added"] = self._getLogs(
+ selfId,
+ toId,
+ oneline=oneline,
+ color=color,
+ pretty_format=pretty_format,
+ )
+ logs["removed"] = self._getLogs(
+ toId,
+ selfId,
+ oneline=oneline,
+ color=color,
+ pretty_format=pretty_format,
+ )
+ return logs
+
+ class _GitGetByExec(object):
+ def __init__(self, project, bare, gitdir):
+ self._project = project
+ self._bare = bare
+ self._gitdir = gitdir
+
+ # __getstate__ and __setstate__ are required for pickling because
+ # __getattr__ exists.
+ def __getstate__(self):
+ return (self._project, self._bare, self._gitdir)
+
+ def __setstate__(self, state):
+ self._project, self._bare, self._gitdir = state
+
+ def LsOthers(self):
+ p = GitCommand(
+ self._project,
+ ["ls-files", "-z", "--others", "--exclude-standard"],
+ bare=False,
+ gitdir=self._gitdir,
+ capture_stdout=True,
+ capture_stderr=True,
+ )
+ if p.Wait() == 0:
+ out = p.stdout
+ if out:
+ # Backslash is not anomalous.
+ return out[:-1].split("\0")
+ return []
+
+ def DiffZ(self, name, *args):
+ cmd = [name]
+ cmd.append("-z")
+ cmd.append("--ignore-submodules")
+ cmd.extend(args)
+ p = GitCommand(
+ self._project,
+ cmd,
+ gitdir=self._gitdir,
+ bare=False,
+ capture_stdout=True,
+ capture_stderr=True,
+ )
+ p.Wait()
+ r = {}
+ out = p.stdout
+ if out:
+ out = iter(out[:-1].split("\0"))
+ while out:
+ try:
+ info = next(out)
+ path = next(out)
+ except StopIteration:
+ break
+
+ class _Info(object):
+ def __init__(self, path, omode, nmode, oid, nid, state):
+ self.path = path
+ self.src_path = None
+ self.old_mode = omode
+ self.new_mode = nmode
+ self.old_id = oid
+ self.new_id = nid
+
+ if len(state) == 1:
+ self.status = state
+ self.level = None
+ else:
+ self.status = state[:1]
+ self.level = state[1:]
+ while self.level.startswith("0"):
+ self.level = self.level[1:]
+
+ info = info[1:].split(" ")
+ info = _Info(path, *info)
+ if info.status in ("R", "C"):
+ info.src_path = info.path
+ info.path = next(out)
+ r[info.path] = info
+ return r
+
+ def GetDotgitPath(self, subpath=None):
+ """Return the full path to the .git dir.
+
+ As a convenience, append |subpath| if provided.
+ """
+ if self._bare:
+ dotgit = self._gitdir
+ else:
+ dotgit = os.path.join(self._project.worktree, ".git")
+ if os.path.isfile(dotgit):
+ # Git worktrees use a "gitdir:" syntax to point to the
+ # scratch space.
+ with open(dotgit) as fp:
+ setting = fp.read()
+ assert setting.startswith("gitdir:")
+ gitdir = setting.split(":", 1)[1].strip()
+ dotgit = os.path.normpath(
+ os.path.join(self._project.worktree, gitdir)
+ )
+
+ return dotgit if subpath is None else os.path.join(dotgit, subpath)
+
+ def GetHead(self):
+ """Return the ref that HEAD points to."""
+ path = self.GetDotgitPath(subpath=HEAD)
+ try:
+ with open(path) as fd:
+ line = fd.readline()
+ except IOError as e:
+ raise NoManifestException(path, str(e))
+ try:
+ line = line.decode()
+ except AttributeError:
+ pass
+ if line.startswith("ref: "):
+ return line[5:-1]
+ return line[:-1]
+
+ def SetHead(self, ref, message=None):
+ cmdv = []
+ if message is not None:
+ cmdv.extend(["-m", message])
+ cmdv.append(HEAD)
+ cmdv.append(ref)
+ self.symbolic_ref(*cmdv)
+
+ def DetachHead(self, new, message=None):
+ cmdv = ["--no-deref"]
+ if message is not None:
+ cmdv.extend(["-m", message])
+ cmdv.append(HEAD)
+ cmdv.append(new)
+ self.update_ref(*cmdv)
+
+ def UpdateRef(self, name, new, old=None, message=None, detach=False):
+ cmdv = []
+ if message is not None:
+ cmdv.extend(["-m", message])
+ if detach:
+ cmdv.append("--no-deref")
+ cmdv.append(name)
+ cmdv.append(new)
+ if old is not None:
+ cmdv.append(old)
+ self.update_ref(*cmdv)
+
+ def DeleteRef(self, name, old=None):
+ if not old:
+ old = self.rev_parse(name)
+ self.update_ref("-d", name, old)
+ self._project.bare_ref.deleted(name)
+
+ def rev_list(self, *args, **kw):
+ if "format" in kw:
+ cmdv = ["log", "--pretty=format:%s" % kw["format"]]
+ else:
+ cmdv = ["rev-list"]
+ cmdv.extend(args)
+ p = GitCommand(
+ self._project,
+ cmdv,
+ bare=self._bare,
+ gitdir=self._gitdir,
+ capture_stdout=True,
+ capture_stderr=True,
+ )
+ if p.Wait() != 0:
+ raise GitError(
+ "%s rev-list %s: %s"
+ % (self._project.name, str(args), p.stderr)
+ )
+ return p.stdout.splitlines()
+
+ def __getattr__(self, name):
+ """Allow arbitrary git commands using pythonic syntax.
+
+ This allows you to do things like:
+ git_obj.rev_parse('HEAD')
+
+ Since we don't have a 'rev_parse' method defined, the __getattr__
+ will run. We'll replace the '_' with a '-' and try to run a git
+ command. Any other positional arguments will be passed to the git
+ command, and the following keyword arguments are supported:
+ config: An optional dict of git config options to be passed with
+ '-c'.
+
+ Args:
+ name: The name of the git command to call. Any '_' characters
+ will be replaced with '-'.
+
+ Returns:
+ A callable object that will try to call git with the named
+ command.
+ """
+ name = name.replace("_", "-")
+
+ def runner(*args, **kwargs):
+ cmdv = []
+ config = kwargs.pop("config", None)
+ for k in kwargs:
+ raise TypeError(
+ "%s() got an unexpected keyword argument %r" % (name, k)
+ )
+ if config is not None:
+ for k, v in config.items():
+ cmdv.append("-c")
+ cmdv.append("%s=%s" % (k, v))
+ cmdv.append(name)
+ cmdv.extend(args)
+ p = GitCommand(
+ self._project,
+ cmdv,
+ bare=self._bare,
+ gitdir=self._gitdir,
+ capture_stdout=True,
+ capture_stderr=True,
+ )
+ if p.Wait() != 0:
+ raise GitError(
+ "%s %s: %s" % (self._project.name, name, p.stderr)
+ )
+ r = p.stdout
+ if r.endswith("\n") and r.index("\n") == len(r) - 1:
+ return r[:-1]
+ return r
+
+ return runner
class _PriorSyncFailedError(Exception):
-
- def __str__(self):
- return 'prior sync failed; rebase still in progress'
+ def __str__(self):
+ return "prior sync failed; rebase still in progress"
class _DirtyError(Exception):
-
- def __str__(self):
- return 'contains uncommitted changes'
+ def __str__(self):
+ return "contains uncommitted changes"
class _InfoMessage(object):
+ def __init__(self, project, text):
+ self.project = project
+ self.text = text
- def __init__(self, project, text):
- self.project = project
- self.text = text
-
- def Print(self, syncbuf):
- syncbuf.out.info('%s/: %s', self.project.RelPath(local=False), self.text)
- syncbuf.out.nl()
+ def Print(self, syncbuf):
+ syncbuf.out.info(
+ "%s/: %s", self.project.RelPath(local=False), self.text
+ )
+ syncbuf.out.nl()
class _Failure(object):
+ def __init__(self, project, why):
+ self.project = project
+ self.why = why
- def __init__(self, project, why):
- self.project = project
- self.why = why
-
- def Print(self, syncbuf):
- syncbuf.out.fail('error: %s/: %s',
- self.project.RelPath(local=False),
- str(self.why))
- syncbuf.out.nl()
+ def Print(self, syncbuf):
+ syncbuf.out.fail(
+ "error: %s/: %s", self.project.RelPath(local=False), str(self.why)
+ )
+ syncbuf.out.nl()
class _Later(object):
+ def __init__(self, project, action):
+ self.project = project
+ self.action = action
- def __init__(self, project, action):
- self.project = project
- self.action = action
-
- def Run(self, syncbuf):
- out = syncbuf.out
- out.project('project %s/', self.project.RelPath(local=False))
- out.nl()
- try:
- self.action()
- out.nl()
- return True
- except GitError:
- out.nl()
- return False
+ def Run(self, syncbuf):
+ out = syncbuf.out
+ out.project("project %s/", self.project.RelPath(local=False))
+ out.nl()
+ try:
+ self.action()
+ out.nl()
+ return True
+ except GitError:
+ out.nl()
+ return False
class _SyncColoring(Coloring):
-
- def __init__(self, config):
- super().__init__(config, 'reposync')
- self.project = self.printer('header', attr='bold')
- self.info = self.printer('info')
- self.fail = self.printer('fail', fg='red')
+ def __init__(self, config):
+ super().__init__(config, "reposync")
+ self.project = self.printer("header", attr="bold")
+ self.info = self.printer("info")
+ self.fail = self.printer("fail", fg="red")
class SyncBuffer(object):
+ def __init__(self, config, detach_head=False):
+ self._messages = []
+ self._failures = []
+ self._later_queue1 = []
+ self._later_queue2 = []
- def __init__(self, config, detach_head=False):
- self._messages = []
- self._failures = []
- self._later_queue1 = []
- self._later_queue2 = []
+ self.out = _SyncColoring(config)
+ self.out.redirect(sys.stderr)
- self.out = _SyncColoring(config)
- self.out.redirect(sys.stderr)
+ self.detach_head = detach_head
+ self.clean = True
+ self.recent_clean = True
- self.detach_head = detach_head
- self.clean = True
- self.recent_clean = True
+ def info(self, project, fmt, *args):
+ self._messages.append(_InfoMessage(project, fmt % args))
- def info(self, project, fmt, *args):
- self._messages.append(_InfoMessage(project, fmt % args))
-
- def fail(self, project, err=None):
- self._failures.append(_Failure(project, err))
- self._MarkUnclean()
-
- def later1(self, project, what):
- self._later_queue1.append(_Later(project, what))
-
- def later2(self, project, what):
- self._later_queue2.append(_Later(project, what))
-
- def Finish(self):
- self._PrintMessages()
- self._RunLater()
- self._PrintMessages()
- return self.clean
-
- def Recently(self):
- recent_clean = self.recent_clean
- self.recent_clean = True
- return recent_clean
-
- def _MarkUnclean(self):
- self.clean = False
- self.recent_clean = False
-
- def _RunLater(self):
- for q in ['_later_queue1', '_later_queue2']:
- if not self._RunQueue(q):
- return
-
- def _RunQueue(self, queue):
- for m in getattr(self, queue):
- if not m.Run(self):
+ def fail(self, project, err=None):
+ self._failures.append(_Failure(project, err))
self._MarkUnclean()
- return False
- setattr(self, queue, [])
- return True
- def _PrintMessages(self):
- if self._messages or self._failures:
- if os.isatty(2):
- self.out.write(progress.CSI_ERASE_LINE)
- self.out.write('\r')
+ def later1(self, project, what):
+ self._later_queue1.append(_Later(project, what))
- for m in self._messages:
- m.Print(self)
- for m in self._failures:
- m.Print(self)
+ def later2(self, project, what):
+ self._later_queue2.append(_Later(project, what))
- self._messages = []
- self._failures = []
+ def Finish(self):
+ self._PrintMessages()
+ self._RunLater()
+ self._PrintMessages()
+ return self.clean
+
+ def Recently(self):
+ recent_clean = self.recent_clean
+ self.recent_clean = True
+ return recent_clean
+
+ def _MarkUnclean(self):
+ self.clean = False
+ self.recent_clean = False
+
+ def _RunLater(self):
+ for q in ["_later_queue1", "_later_queue2"]:
+ if not self._RunQueue(q):
+ return
+
+ def _RunQueue(self, queue):
+ for m in getattr(self, queue):
+ if not m.Run(self):
+ self._MarkUnclean()
+ return False
+ setattr(self, queue, [])
+ return True
+
+ def _PrintMessages(self):
+ if self._messages or self._failures:
+ if os.isatty(2):
+ self.out.write(progress.CSI_ERASE_LINE)
+ self.out.write("\r")
+
+ for m in self._messages:
+ m.Print(self)
+ for m in self._failures:
+ m.Print(self)
+
+ self._messages = []
+ self._failures = []
class MetaProject(Project):
- """A special project housed under .repo."""
+ """A special project housed under .repo."""
- def __init__(self, manifest, name, gitdir, worktree):
- Project.__init__(self,
- manifest=manifest,
- name=name,
- gitdir=gitdir,
- objdir=gitdir,
- worktree=worktree,
- remote=RemoteSpec('origin'),
- relpath='.repo/%s' % name,
- revisionExpr='refs/heads/master',
- revisionId=None,
- groups=None)
+ def __init__(self, manifest, name, gitdir, worktree):
+ Project.__init__(
+ self,
+ manifest=manifest,
+ name=name,
+ gitdir=gitdir,
+ objdir=gitdir,
+ worktree=worktree,
+ remote=RemoteSpec("origin"),
+ relpath=".repo/%s" % name,
+ revisionExpr="refs/heads/master",
+ revisionId=None,
+ groups=None,
+ )
- def PreSync(self):
- if self.Exists:
- cb = self.CurrentBranch
- if cb:
- base = self.GetBranch(cb).merge
- if base:
- self.revisionExpr = base
- self.revisionId = None
+ def PreSync(self):
+ if self.Exists:
+ cb = self.CurrentBranch
+ if cb:
+ base = self.GetBranch(cb).merge
+ if base:
+ self.revisionExpr = base
+ self.revisionId = None
- @property
- def HasChanges(self):
- """Has the remote received new commits not yet checked out?"""
- if not self.remote or not self.revisionExpr:
- return False
+ @property
+ def HasChanges(self):
+ """Has the remote received new commits not yet checked out?"""
+ if not self.remote or not self.revisionExpr:
+ return False
- all_refs = self.bare_ref.all
- revid = self.GetRevisionId(all_refs)
- head = self.work_git.GetHead()
- if head.startswith(R_HEADS):
- try:
- head = all_refs[head]
- except KeyError:
- head = None
+ all_refs = self.bare_ref.all
+ revid = self.GetRevisionId(all_refs)
+ head = self.work_git.GetHead()
+ if head.startswith(R_HEADS):
+ try:
+ head = all_refs[head]
+ except KeyError:
+ head = None
- if revid == head:
- return False
- elif self._revlist(not_rev(HEAD), revid):
- return True
- return False
+ if revid == head:
+ return False
+ elif self._revlist(not_rev(HEAD), revid):
+ return True
+ return False
class RepoProject(MetaProject):
- """The MetaProject for repo itself."""
+ """The MetaProject for repo itself."""
- @property
- def LastFetch(self):
- try:
- fh = os.path.join(self.gitdir, 'FETCH_HEAD')
- return os.path.getmtime(fh)
- except OSError:
- return 0
+ @property
+ def LastFetch(self):
+ try:
+ fh = os.path.join(self.gitdir, "FETCH_HEAD")
+ return os.path.getmtime(fh)
+ except OSError:
+ return 0
class ManifestProject(MetaProject):
- """The MetaProject for manifests."""
+ """The MetaProject for manifests."""
- def MetaBranchSwitch(self, submodules=False):
- """Prepare for manifest branch switch."""
+ def MetaBranchSwitch(self, submodules=False):
+ """Prepare for manifest branch switch."""
- # detach and delete manifest branch, allowing a new
- # branch to take over
- syncbuf = SyncBuffer(self.config, detach_head=True)
- self.Sync_LocalHalf(syncbuf, submodules=submodules)
- syncbuf.Finish()
+ # detach and delete manifest branch, allowing a new
+ # branch to take over
+ syncbuf = SyncBuffer(self.config, detach_head=True)
+ self.Sync_LocalHalf(syncbuf, submodules=submodules)
+ syncbuf.Finish()
- return GitCommand(self,
- ['update-ref', '-d', 'refs/heads/default'],
- capture_stdout=True,
- capture_stderr=True).Wait() == 0
+ return (
+ GitCommand(
+ self,
+ ["update-ref", "-d", "refs/heads/default"],
+ capture_stdout=True,
+ capture_stderr=True,
+ ).Wait()
+ == 0
+ )
- @property
- def standalone_manifest_url(self):
- """The URL of the standalone manifest, or None."""
- return self.config.GetString('manifest.standalone')
+ @property
+ def standalone_manifest_url(self):
+ """The URL of the standalone manifest, or None."""
+ return self.config.GetString("manifest.standalone")
- @property
- def manifest_groups(self):
- """The manifest groups string."""
- return self.config.GetString('manifest.groups')
+ @property
+ def manifest_groups(self):
+ """The manifest groups string."""
+ return self.config.GetString("manifest.groups")
- @property
- def reference(self):
- """The --reference for this manifest."""
- return self.config.GetString('repo.reference')
+ @property
+ def reference(self):
+ """The --reference for this manifest."""
+ return self.config.GetString("repo.reference")
- @property
- def dissociate(self):
- """Whether to dissociate."""
- return self.config.GetBoolean('repo.dissociate')
+ @property
+ def dissociate(self):
+ """Whether to dissociate."""
+ return self.config.GetBoolean("repo.dissociate")
- @property
- def archive(self):
- """Whether we use archive."""
- return self.config.GetBoolean('repo.archive')
+ @property
+ def archive(self):
+ """Whether we use archive."""
+ return self.config.GetBoolean("repo.archive")
- @property
- def mirror(self):
- """Whether we use mirror."""
- return self.config.GetBoolean('repo.mirror')
+ @property
+ def mirror(self):
+ """Whether we use mirror."""
+ return self.config.GetBoolean("repo.mirror")
- @property
- def use_worktree(self):
- """Whether we use worktree."""
- return self.config.GetBoolean('repo.worktree')
+ @property
+ def use_worktree(self):
+ """Whether we use worktree."""
+ return self.config.GetBoolean("repo.worktree")
- @property
- def clone_bundle(self):
- """Whether we use clone_bundle."""
- return self.config.GetBoolean('repo.clonebundle')
+ @property
+ def clone_bundle(self):
+ """Whether we use clone_bundle."""
+ return self.config.GetBoolean("repo.clonebundle")
- @property
- def submodules(self):
- """Whether we use submodules."""
- return self.config.GetBoolean('repo.submodules')
+ @property
+ def submodules(self):
+ """Whether we use submodules."""
+ return self.config.GetBoolean("repo.submodules")
- @property
- def git_lfs(self):
- """Whether we use git_lfs."""
- return self.config.GetBoolean('repo.git-lfs')
+ @property
+ def git_lfs(self):
+ """Whether we use git_lfs."""
+ return self.config.GetBoolean("repo.git-lfs")
- @property
- def use_superproject(self):
- """Whether we use superproject."""
- return self.config.GetBoolean('repo.superproject')
+ @property
+ def use_superproject(self):
+ """Whether we use superproject."""
+ return self.config.GetBoolean("repo.superproject")
- @property
- def partial_clone(self):
- """Whether this is a partial clone."""
- return self.config.GetBoolean('repo.partialclone')
+ @property
+ def partial_clone(self):
+ """Whether this is a partial clone."""
+ return self.config.GetBoolean("repo.partialclone")
- @property
- def depth(self):
- """Partial clone depth."""
- return self.config.GetString('repo.depth')
+ @property
+ def depth(self):
+ """Partial clone depth."""
+ return self.config.GetString("repo.depth")
- @property
- def clone_filter(self):
- """The clone filter."""
- return self.config.GetString('repo.clonefilter')
+ @property
+ def clone_filter(self):
+ """The clone filter."""
+ return self.config.GetString("repo.clonefilter")
- @property
- def partial_clone_exclude(self):
- """Partial clone exclude string"""
- return self.config.GetString('repo.partialcloneexclude')
+ @property
+ def partial_clone_exclude(self):
+ """Partial clone exclude string"""
+ return self.config.GetString("repo.partialcloneexclude")
- @property
- def manifest_platform(self):
- """The --platform argument from `repo init`."""
- return self.config.GetString('manifest.platform')
+ @property
+ def manifest_platform(self):
+ """The --platform argument from `repo init`."""
+ return self.config.GetString("manifest.platform")
- @property
- def _platform_name(self):
- """Return the name of the platform."""
- return platform.system().lower()
+ @property
+ def _platform_name(self):
+ """Return the name of the platform."""
+ return platform.system().lower()
- def SyncWithPossibleInit(self, submanifest, verbose=False,
- current_branch_only=False, tags='', git_event_log=None):
- """Sync a manifestProject, possibly for the first time.
+ def SyncWithPossibleInit(
+ self,
+ submanifest,
+ verbose=False,
+ current_branch_only=False,
+ tags="",
+ git_event_log=None,
+ ):
+ """Sync a manifestProject, possibly for the first time.
- Call Sync() with arguments from the most recent `repo init`. If this is a
- new sub manifest, then inherit options from the parent's manifestProject.
+ Call Sync() with arguments from the most recent `repo init`. If this is
+ a new sub manifest, then inherit options from the parent's
+ manifestProject.
- This is used by subcmds.Sync() to do an initial download of new sub
- manifests.
+ This is used by subcmds.Sync() to do an initial download of new sub
+ manifests.
- Args:
- submanifest: an XmlSubmanifest, the submanifest to re-sync.
- verbose: a boolean, whether to show all output, rather than only errors.
- current_branch_only: a boolean, whether to only fetch the current manifest
- branch from the server.
- tags: a boolean, whether to fetch tags.
- git_event_log: an EventLog, for git tracing.
- """
- # TODO(lamontjones): when refactoring sync (and init?) consider how to
- # better get the init options that we should use for new submanifests that
- # are added when syncing an existing workspace.
- git_event_log = git_event_log or EventLog()
- spec = submanifest.ToSubmanifestSpec()
- # Use the init options from the existing manifestProject, or the parent if
- # it doesn't exist.
- #
- # Today, we only support changing manifest_groups on the sub-manifest, with
- # no supported-for-the-user way to change the other arguments from those
- # specified by the outermost manifest.
- #
- # TODO(lamontjones): determine which of these should come from the outermost
- # manifest and which should come from the parent manifest.
- mp = self if self.Exists else submanifest.parent.manifestProject
- return self.Sync(
- manifest_url=spec.manifestUrl,
- manifest_branch=spec.revision,
- standalone_manifest=mp.standalone_manifest_url,
- groups=mp.manifest_groups,
- platform=mp.manifest_platform,
- mirror=mp.mirror,
- dissociate=mp.dissociate,
- reference=mp.reference,
- worktree=mp.use_worktree,
- submodules=mp.submodules,
- archive=mp.archive,
- partial_clone=mp.partial_clone,
- clone_filter=mp.clone_filter,
- partial_clone_exclude=mp.partial_clone_exclude,
- clone_bundle=mp.clone_bundle,
- git_lfs=mp.git_lfs,
- use_superproject=mp.use_superproject,
- verbose=verbose,
- current_branch_only=current_branch_only,
- tags=tags,
- depth=mp.depth,
- git_event_log=git_event_log,
- manifest_name=spec.manifestName,
- this_manifest_only=True,
- outer_manifest=False,
- )
-
- def Sync(self, _kwargs_only=(), manifest_url='', manifest_branch=None,
- standalone_manifest=False, groups='', mirror=False, reference='',
- dissociate=False, worktree=False, submodules=False, archive=False,
- partial_clone=None, depth=None, clone_filter='blob:none',
- partial_clone_exclude=None, clone_bundle=None, git_lfs=None,
- use_superproject=None, verbose=False, current_branch_only=False,
- git_event_log=None, platform='', manifest_name='default.xml',
- tags='', this_manifest_only=False, outer_manifest=True):
- """Sync the manifest and all submanifests.
-
- Args:
- manifest_url: a string, the URL of the manifest project.
- manifest_branch: a string, the manifest branch to use.
- standalone_manifest: a boolean, whether to store the manifest as a static
- file.
- groups: a string, restricts the checkout to projects with the specified
- groups.
- mirror: a boolean, whether to create a mirror of the remote repository.
- reference: a string, location of a repo instance to use as a reference.
- dissociate: a boolean, whether to dissociate from reference mirrors after
- clone.
- worktree: a boolean, whether to use git-worktree to manage projects.
- submodules: a boolean, whether sync submodules associated with the
- manifest project.
- archive: a boolean, whether to checkout each project as an archive. See
- git-archive.
- partial_clone: a boolean, whether to perform a partial clone.
- depth: an int, how deep of a shallow clone to create.
- clone_filter: a string, filter to use with partial_clone.
- partial_clone_exclude : a string, comma-delimeted list of project namess
- to exclude from partial clone.
- clone_bundle: a boolean, whether to enable /clone.bundle on HTTP/HTTPS.
- git_lfs: a boolean, whether to enable git LFS support.
- use_superproject: a boolean, whether to use the manifest superproject to
- sync projects.
- verbose: a boolean, whether to show all output, rather than only errors.
- current_branch_only: a boolean, whether to only fetch the current manifest
- branch from the server.
- platform: a string, restrict the checkout to projects with the specified
- platform group.
- git_event_log: an EventLog, for git tracing.
- tags: a boolean, whether to fetch tags.
- manifest_name: a string, the name of the manifest file to use.
- this_manifest_only: a boolean, whether to only operate on the current sub
- manifest.
- outer_manifest: a boolean, whether to start at the outermost manifest.
-
- Returns:
- a boolean, whether the sync was successful.
- """
- assert _kwargs_only == (), 'Sync only accepts keyword arguments.'
-
- groups = groups or self.manifest.GetDefaultGroupsStr(with_platform=False)
- platform = platform or 'auto'
- git_event_log = git_event_log or EventLog()
- if outer_manifest and self.manifest.is_submanifest:
- # In a multi-manifest checkout, use the outer manifest unless we are told
- # not to.
- return self.client.outer_manifest.manifestProject.Sync(
- manifest_url=manifest_url,
- manifest_branch=manifest_branch,
- standalone_manifest=standalone_manifest,
- groups=groups,
- platform=platform,
- mirror=mirror,
- dissociate=dissociate,
- reference=reference,
- worktree=worktree,
- submodules=submodules,
- archive=archive,
- partial_clone=partial_clone,
- clone_filter=clone_filter,
- partial_clone_exclude=partial_clone_exclude,
- clone_bundle=clone_bundle,
- git_lfs=git_lfs,
- use_superproject=use_superproject,
- verbose=verbose,
- current_branch_only=current_branch_only,
- tags=tags,
- depth=depth,
- git_event_log=git_event_log,
- manifest_name=manifest_name,
- this_manifest_only=this_manifest_only,
- outer_manifest=False)
-
- # If repo has already been initialized, we take -u with the absence of
- # --standalone-manifest to mean "transition to a standard repo set up",
- # which necessitates starting fresh.
- # If --standalone-manifest is set, we always tear everything down and start
- # anew.
- if self.Exists:
- was_standalone_manifest = self.config.GetString('manifest.standalone')
- if was_standalone_manifest and not manifest_url:
- print('fatal: repo was initialized with a standlone manifest, '
- 'cannot be re-initialized without --manifest-url/-u')
- return False
-
- if standalone_manifest or (was_standalone_manifest and manifest_url):
- self.config.ClearCache()
- if self.gitdir and os.path.exists(self.gitdir):
- platform_utils.rmtree(self.gitdir)
- if self.worktree and os.path.exists(self.worktree):
- platform_utils.rmtree(self.worktree)
-
- is_new = not self.Exists
- if is_new:
- if not manifest_url:
- print('fatal: manifest url is required.', file=sys.stderr)
- return False
-
- if verbose:
- print('Downloading manifest from %s' %
- (GitConfig.ForUser().UrlInsteadOf(manifest_url),),
- file=sys.stderr)
-
- # The manifest project object doesn't keep track of the path on the
- # server where this git is located, so let's save that here.
- mirrored_manifest_git = None
- if reference:
- manifest_git_path = urllib.parse.urlparse(manifest_url).path[1:]
- mirrored_manifest_git = os.path.join(reference, manifest_git_path)
- if not mirrored_manifest_git.endswith(".git"):
- mirrored_manifest_git += ".git"
- if not os.path.exists(mirrored_manifest_git):
- mirrored_manifest_git = os.path.join(reference,
- '.repo/manifests.git')
-
- self._InitGitDir(mirror_git=mirrored_manifest_git)
-
- # If standalone_manifest is set, mark the project as "standalone" -- we'll
- # still do much of the manifests.git set up, but will avoid actual syncs to
- # a remote.
- if standalone_manifest:
- self.config.SetString('manifest.standalone', manifest_url)
- elif not manifest_url and not manifest_branch:
- # If -u is set and --standalone-manifest is not, then we're not in
- # standalone mode. Otherwise, use config to infer what we were in the last
- # init.
- standalone_manifest = bool(self.config.GetString('manifest.standalone'))
- if not standalone_manifest:
- self.config.SetString('manifest.standalone', None)
-
- self._ConfigureDepth(depth)
-
- # Set the remote URL before the remote branch as we might need it below.
- if manifest_url:
- r = self.GetRemote()
- r.url = manifest_url
- r.ResetFetch()
- r.Save()
-
- if not standalone_manifest:
- if manifest_branch:
- if manifest_branch == 'HEAD':
- manifest_branch = self.ResolveRemoteHead()
- if manifest_branch is None:
- print('fatal: unable to resolve HEAD', file=sys.stderr)
- return False
- self.revisionExpr = manifest_branch
- else:
- if is_new:
- default_branch = self.ResolveRemoteHead()
- if default_branch is None:
- # If the remote doesn't have HEAD configured, default to master.
- default_branch = 'refs/heads/master'
- self.revisionExpr = default_branch
- else:
- self.PreSync()
-
- groups = re.split(r'[,\s]+', groups or '')
- all_platforms = ['linux', 'darwin', 'windows']
- platformize = lambda x: 'platform-' + x
- if platform == 'auto':
- if not mirror and not self.mirror:
- groups.append(platformize(self._platform_name))
- elif platform == 'all':
- groups.extend(map(platformize, all_platforms))
- elif platform in all_platforms:
- groups.append(platformize(platform))
- elif platform != 'none':
- print('fatal: invalid platform flag', file=sys.stderr)
- return False
- self.config.SetString('manifest.platform', platform)
-
- groups = [x for x in groups if x]
- groupstr = ','.join(groups)
- if platform == 'auto' and groupstr == self.manifest.GetDefaultGroupsStr():
- groupstr = None
- self.config.SetString('manifest.groups', groupstr)
-
- if reference:
- self.config.SetString('repo.reference', reference)
-
- if dissociate:
- self.config.SetBoolean('repo.dissociate', dissociate)
-
- if worktree:
- if mirror:
- print('fatal: --mirror and --worktree are incompatible',
- file=sys.stderr)
- return False
- if submodules:
- print('fatal: --submodules and --worktree are incompatible',
- file=sys.stderr)
- return False
- self.config.SetBoolean('repo.worktree', worktree)
- if is_new:
- self.use_git_worktrees = True
- print('warning: --worktree is experimental!', file=sys.stderr)
-
- if archive:
- if is_new:
- self.config.SetBoolean('repo.archive', archive)
- else:
- print('fatal: --archive is only supported when initializing a new '
- 'workspace.', file=sys.stderr)
- print('Either delete the .repo folder in this workspace, or initialize '
- 'in another location.', file=sys.stderr)
- return False
-
- if mirror:
- if is_new:
- self.config.SetBoolean('repo.mirror', mirror)
- else:
- print('fatal: --mirror is only supported when initializing a new '
- 'workspace.', file=sys.stderr)
- print('Either delete the .repo folder in this workspace, or initialize '
- 'in another location.', file=sys.stderr)
- return False
-
- if partial_clone is not None:
- if mirror:
- print('fatal: --mirror and --partial-clone are mutually exclusive',
- file=sys.stderr)
- return False
- self.config.SetBoolean('repo.partialclone', partial_clone)
- if clone_filter:
- self.config.SetString('repo.clonefilter', clone_filter)
- elif self.partial_clone:
- clone_filter = self.clone_filter
- else:
- clone_filter = None
-
- if partial_clone_exclude is not None:
- self.config.SetString('repo.partialcloneexclude', partial_clone_exclude)
-
- if clone_bundle is None:
- clone_bundle = False if partial_clone else True
- else:
- self.config.SetBoolean('repo.clonebundle', clone_bundle)
-
- if submodules:
- self.config.SetBoolean('repo.submodules', submodules)
-
- if git_lfs is not None:
- if git_lfs:
- git_require((2, 17, 0), fail=True, msg='Git LFS support')
-
- self.config.SetBoolean('repo.git-lfs', git_lfs)
- if not is_new:
- print('warning: Changing --git-lfs settings will only affect new project checkouts.\n'
- ' Existing projects will require manual updates.\n', file=sys.stderr)
-
- if use_superproject is not None:
- self.config.SetBoolean('repo.superproject', use_superproject)
-
- if not standalone_manifest:
- success = self.Sync_NetworkHalf(
- is_new=is_new, quiet=not verbose, verbose=verbose,
- clone_bundle=clone_bundle, current_branch_only=current_branch_only,
- tags=tags, submodules=submodules, clone_filter=clone_filter,
- partial_clone_exclude=self.manifest.PartialCloneExclude).success
- if not success:
- r = self.GetRemote()
- print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr)
-
- # Better delete the manifest git dir if we created it; otherwise next
- # time (when user fixes problems) we won't go through the "is_new" logic.
- if is_new:
- platform_utils.rmtree(self.gitdir)
- return False
-
- if manifest_branch:
- self.MetaBranchSwitch(submodules=submodules)
-
- syncbuf = SyncBuffer(self.config)
- self.Sync_LocalHalf(syncbuf, submodules=submodules)
- syncbuf.Finish()
-
- if is_new or self.CurrentBranch is None:
- if not self.StartBranch('default'):
- print('fatal: cannot create default in manifest', file=sys.stderr)
- return False
-
- if not manifest_name:
- print('fatal: manifest name (-m) is required.', file=sys.stderr)
- return False
-
- elif is_new:
- # This is a new standalone manifest.
- manifest_name = 'default.xml'
- manifest_data = fetch.fetch_file(manifest_url, verbose=verbose)
- dest = os.path.join(self.worktree, manifest_name)
- os.makedirs(os.path.dirname(dest), exist_ok=True)
- with open(dest, 'wb') as f:
- f.write(manifest_data)
-
- try:
- self.manifest.Link(manifest_name)
- except ManifestParseError as e:
- print("fatal: manifest '%s' not available" % manifest_name,
- file=sys.stderr)
- print('fatal: %s' % str(e), file=sys.stderr)
- return False
-
- if not this_manifest_only:
- for submanifest in self.manifest.submanifests.values():
+ Args:
+ submanifest: an XmlSubmanifest, the submanifest to re-sync.
+ verbose: a boolean, whether to show all output, rather than only
+ errors.
+ current_branch_only: a boolean, whether to only fetch the current
+ manifest branch from the server.
+ tags: a boolean, whether to fetch tags.
+ git_event_log: an EventLog, for git tracing.
+ """
+ # TODO(lamontjones): when refactoring sync (and init?) consider how to
+ # better get the init options that we should use for new submanifests
+ # that are added when syncing an existing workspace.
+ git_event_log = git_event_log or EventLog()
spec = submanifest.ToSubmanifestSpec()
- submanifest.repo_client.manifestProject.Sync(
+ # Use the init options from the existing manifestProject, or the parent
+ # if it doesn't exist.
+ #
+ # Today, we only support changing manifest_groups on the sub-manifest,
+ # with no supported-for-the-user way to change the other arguments from
+ # those specified by the outermost manifest.
+ #
+ # TODO(lamontjones): determine which of these should come from the
+ # outermost manifest and which should come from the parent manifest.
+ mp = self if self.Exists else submanifest.parent.manifestProject
+ return self.Sync(
manifest_url=spec.manifestUrl,
manifest_branch=spec.revision,
- standalone_manifest=standalone_manifest,
- groups=self.manifest_groups,
- platform=platform,
- mirror=mirror,
- dissociate=dissociate,
- reference=reference,
- worktree=worktree,
- submodules=submodules,
- archive=archive,
- partial_clone=partial_clone,
- clone_filter=clone_filter,
- partial_clone_exclude=partial_clone_exclude,
- clone_bundle=clone_bundle,
- git_lfs=git_lfs,
- use_superproject=use_superproject,
+ standalone_manifest=mp.standalone_manifest_url,
+ groups=mp.manifest_groups,
+ platform=mp.manifest_platform,
+ mirror=mp.mirror,
+ dissociate=mp.dissociate,
+ reference=mp.reference,
+ worktree=mp.use_worktree,
+ submodules=mp.submodules,
+ archive=mp.archive,
+ partial_clone=mp.partial_clone,
+ clone_filter=mp.clone_filter,
+ partial_clone_exclude=mp.partial_clone_exclude,
+ clone_bundle=mp.clone_bundle,
+ git_lfs=mp.git_lfs,
+ use_superproject=mp.use_superproject,
verbose=verbose,
current_branch_only=current_branch_only,
tags=tags,
- depth=depth,
+ depth=mp.depth,
git_event_log=git_event_log,
manifest_name=spec.manifestName,
- this_manifest_only=False,
+ this_manifest_only=True,
outer_manifest=False,
)
- # Lastly, if the manifest has a <superproject> then have the superproject
- # sync it (if it will be used).
- if git_superproject.UseSuperproject(use_superproject, self.manifest):
- sync_result = self.manifest.superproject.Sync(git_event_log)
- if not sync_result.success:
- submanifest = ''
- if self.manifest.path_prefix:
- submanifest = f'for {self.manifest.path_prefix} '
- print(f'warning: git update of superproject {submanifest}failed, repo '
- 'sync will not use superproject to fetch source; while this '
- 'error is not fatal, and you can continue to run repo sync, '
- 'please run repo init with the --no-use-superproject option to '
- 'stop seeing this warning', file=sys.stderr)
- if sync_result.fatal and use_superproject is not None:
- return False
+ def Sync(
+ self,
+ _kwargs_only=(),
+ manifest_url="",
+ manifest_branch=None,
+ standalone_manifest=False,
+ groups="",
+ mirror=False,
+ reference="",
+ dissociate=False,
+ worktree=False,
+ submodules=False,
+ archive=False,
+ partial_clone=None,
+ depth=None,
+ clone_filter="blob:none",
+ partial_clone_exclude=None,
+ clone_bundle=None,
+ git_lfs=None,
+ use_superproject=None,
+ verbose=False,
+ current_branch_only=False,
+ git_event_log=None,
+ platform="",
+ manifest_name="default.xml",
+ tags="",
+ this_manifest_only=False,
+ outer_manifest=True,
+ ):
+ """Sync the manifest and all submanifests.
- return True
+ Args:
+ manifest_url: a string, the URL of the manifest project.
+ manifest_branch: a string, the manifest branch to use.
+ standalone_manifest: a boolean, whether to store the manifest as a
+ static file.
+ groups: a string, restricts the checkout to projects with the
+ specified groups.
+ mirror: a boolean, whether to create a mirror of the remote
+ repository.
+ reference: a string, location of a repo instance to use as a
+ reference.
+ dissociate: a boolean, whether to dissociate from reference mirrors
+ after clone.
+ worktree: a boolean, whether to use git-worktree to manage projects.
+ submodules: a boolean, whether sync submodules associated with the
+ manifest project.
+ archive: a boolean, whether to checkout each project as an archive.
+ See git-archive.
+ partial_clone: a boolean, whether to perform a partial clone.
+ depth: an int, how deep of a shallow clone to create.
+ clone_filter: a string, filter to use with partial_clone.
+ partial_clone_exclude : a string, comma-delimeted list of project
+ names to exclude from partial clone.
+ clone_bundle: a boolean, whether to enable /clone.bundle on
+ HTTP/HTTPS.
+ git_lfs: a boolean, whether to enable git LFS support.
+ use_superproject: a boolean, whether to use the manifest
+ superproject to sync projects.
+ verbose: a boolean, whether to show all output, rather than only
+ errors.
+ current_branch_only: a boolean, whether to only fetch the current
+ manifest branch from the server.
+ platform: a string, restrict the checkout to projects with the
+ specified platform group.
+ git_event_log: an EventLog, for git tracing.
+ tags: a boolean, whether to fetch tags.
+ manifest_name: a string, the name of the manifest file to use.
+ this_manifest_only: a boolean, whether to only operate on the
+ current sub manifest.
+ outer_manifest: a boolean, whether to start at the outermost
+ manifest.
- def _ConfigureDepth(self, depth):
- """Configure the depth we'll sync down.
+ Returns:
+ a boolean, whether the sync was successful.
+ """
+ assert _kwargs_only == (), "Sync only accepts keyword arguments."
- Args:
- depth: an int, how deep of a partial clone to create.
- """
- # Opt.depth will be non-None if user actually passed --depth to repo init.
- if depth is not None:
- if depth > 0:
- # Positive values will set the depth.
- depth = str(depth)
- else:
- # Negative numbers will clear the depth; passing None to SetString
- # will do that.
- depth = None
+ groups = groups or self.manifest.GetDefaultGroupsStr(
+ with_platform=False
+ )
+ platform = platform or "auto"
+ git_event_log = git_event_log or EventLog()
+ if outer_manifest and self.manifest.is_submanifest:
+ # In a multi-manifest checkout, use the outer manifest unless we are
+ # told not to.
+ return self.client.outer_manifest.manifestProject.Sync(
+ manifest_url=manifest_url,
+ manifest_branch=manifest_branch,
+ standalone_manifest=standalone_manifest,
+ groups=groups,
+ platform=platform,
+ mirror=mirror,
+ dissociate=dissociate,
+ reference=reference,
+ worktree=worktree,
+ submodules=submodules,
+ archive=archive,
+ partial_clone=partial_clone,
+ clone_filter=clone_filter,
+ partial_clone_exclude=partial_clone_exclude,
+ clone_bundle=clone_bundle,
+ git_lfs=git_lfs,
+ use_superproject=use_superproject,
+ verbose=verbose,
+ current_branch_only=current_branch_only,
+ tags=tags,
+ depth=depth,
+ git_event_log=git_event_log,
+ manifest_name=manifest_name,
+ this_manifest_only=this_manifest_only,
+ outer_manifest=False,
+ )
- # We store the depth in the main manifest project.
- self.config.SetString('repo.depth', depth)
+ # If repo has already been initialized, we take -u with the absence of
+ # --standalone-manifest to mean "transition to a standard repo set up",
+ # which necessitates starting fresh.
+ # If --standalone-manifest is set, we always tear everything down and
+ # start anew.
+ if self.Exists:
+ was_standalone_manifest = self.config.GetString(
+ "manifest.standalone"
+ )
+ if was_standalone_manifest and not manifest_url:
+ print(
+ "fatal: repo was initialized with a standlone manifest, "
+ "cannot be re-initialized without --manifest-url/-u"
+ )
+ return False
+
+ if standalone_manifest or (
+ was_standalone_manifest and manifest_url
+ ):
+ self.config.ClearCache()
+ if self.gitdir and os.path.exists(self.gitdir):
+ platform_utils.rmtree(self.gitdir)
+ if self.worktree and os.path.exists(self.worktree):
+ platform_utils.rmtree(self.worktree)
+
+ is_new = not self.Exists
+ if is_new:
+ if not manifest_url:
+ print("fatal: manifest url is required.", file=sys.stderr)
+ return False
+
+ if verbose:
+ print(
+ "Downloading manifest from %s"
+ % (GitConfig.ForUser().UrlInsteadOf(manifest_url),),
+ file=sys.stderr,
+ )
+
+ # The manifest project object doesn't keep track of the path on the
+ # server where this git is located, so let's save that here.
+ mirrored_manifest_git = None
+ if reference:
+ manifest_git_path = urllib.parse.urlparse(manifest_url).path[1:]
+ mirrored_manifest_git = os.path.join(
+ reference, manifest_git_path
+ )
+ if not mirrored_manifest_git.endswith(".git"):
+ mirrored_manifest_git += ".git"
+ if not os.path.exists(mirrored_manifest_git):
+ mirrored_manifest_git = os.path.join(
+ reference, ".repo/manifests.git"
+ )
+
+ self._InitGitDir(mirror_git=mirrored_manifest_git)
+
+ # If standalone_manifest is set, mark the project as "standalone" --
+ # we'll still do much of the manifests.git set up, but will avoid actual
+ # syncs to a remote.
+ if standalone_manifest:
+ self.config.SetString("manifest.standalone", manifest_url)
+ elif not manifest_url and not manifest_branch:
+ # If -u is set and --standalone-manifest is not, then we're not in
+ # standalone mode. Otherwise, use config to infer what we were in
+ # the last init.
+ standalone_manifest = bool(
+ self.config.GetString("manifest.standalone")
+ )
+ if not standalone_manifest:
+ self.config.SetString("manifest.standalone", None)
+
+ self._ConfigureDepth(depth)
+
+ # Set the remote URL before the remote branch as we might need it below.
+ if manifest_url:
+ r = self.GetRemote()
+ r.url = manifest_url
+ r.ResetFetch()
+ r.Save()
+
+ if not standalone_manifest:
+ if manifest_branch:
+ if manifest_branch == "HEAD":
+ manifest_branch = self.ResolveRemoteHead()
+ if manifest_branch is None:
+ print("fatal: unable to resolve HEAD", file=sys.stderr)
+ return False
+ self.revisionExpr = manifest_branch
+ else:
+ if is_new:
+ default_branch = self.ResolveRemoteHead()
+ if default_branch is None:
+ # If the remote doesn't have HEAD configured, default to
+ # master.
+ default_branch = "refs/heads/master"
+ self.revisionExpr = default_branch
+ else:
+ self.PreSync()
+
+ groups = re.split(r"[,\s]+", groups or "")
+ all_platforms = ["linux", "darwin", "windows"]
+ platformize = lambda x: "platform-" + x
+ if platform == "auto":
+ if not mirror and not self.mirror:
+ groups.append(platformize(self._platform_name))
+ elif platform == "all":
+ groups.extend(map(platformize, all_platforms))
+ elif platform in all_platforms:
+ groups.append(platformize(platform))
+ elif platform != "none":
+ print("fatal: invalid platform flag", file=sys.stderr)
+ return False
+ self.config.SetString("manifest.platform", platform)
+
+ groups = [x for x in groups if x]
+ groupstr = ",".join(groups)
+ if (
+ platform == "auto"
+ and groupstr == self.manifest.GetDefaultGroupsStr()
+ ):
+ groupstr = None
+ self.config.SetString("manifest.groups", groupstr)
+
+ if reference:
+ self.config.SetString("repo.reference", reference)
+
+ if dissociate:
+ self.config.SetBoolean("repo.dissociate", dissociate)
+
+ if worktree:
+ if mirror:
+ print(
+ "fatal: --mirror and --worktree are incompatible",
+ file=sys.stderr,
+ )
+ return False
+ if submodules:
+ print(
+ "fatal: --submodules and --worktree are incompatible",
+ file=sys.stderr,
+ )
+ return False
+ self.config.SetBoolean("repo.worktree", worktree)
+ if is_new:
+ self.use_git_worktrees = True
+ print("warning: --worktree is experimental!", file=sys.stderr)
+
+ if archive:
+ if is_new:
+ self.config.SetBoolean("repo.archive", archive)
+ else:
+ print(
+ "fatal: --archive is only supported when initializing a "
+ "new workspace.",
+ file=sys.stderr,
+ )
+ print(
+ "Either delete the .repo folder in this workspace, or "
+ "initialize in another location.",
+ file=sys.stderr,
+ )
+ return False
+
+ if mirror:
+ if is_new:
+ self.config.SetBoolean("repo.mirror", mirror)
+ else:
+ print(
+ "fatal: --mirror is only supported when initializing a new "
+ "workspace.",
+ file=sys.stderr,
+ )
+ print(
+ "Either delete the .repo folder in this workspace, or "
+ "initialize in another location.",
+ file=sys.stderr,
+ )
+ return False
+
+ if partial_clone is not None:
+ if mirror:
+ print(
+ "fatal: --mirror and --partial-clone are mutually "
+ "exclusive",
+ file=sys.stderr,
+ )
+ return False
+ self.config.SetBoolean("repo.partialclone", partial_clone)
+ if clone_filter:
+ self.config.SetString("repo.clonefilter", clone_filter)
+ elif self.partial_clone:
+ clone_filter = self.clone_filter
+ else:
+ clone_filter = None
+
+ if partial_clone_exclude is not None:
+ self.config.SetString(
+ "repo.partialcloneexclude", partial_clone_exclude
+ )
+
+ if clone_bundle is None:
+ clone_bundle = False if partial_clone else True
+ else:
+ self.config.SetBoolean("repo.clonebundle", clone_bundle)
+
+ if submodules:
+ self.config.SetBoolean("repo.submodules", submodules)
+
+ if git_lfs is not None:
+ if git_lfs:
+ git_require((2, 17, 0), fail=True, msg="Git LFS support")
+
+ self.config.SetBoolean("repo.git-lfs", git_lfs)
+ if not is_new:
+ print(
+ "warning: Changing --git-lfs settings will only affect new "
+ "project checkouts.\n"
+ " Existing projects will require manual updates.\n",
+ file=sys.stderr,
+ )
+
+ if use_superproject is not None:
+ self.config.SetBoolean("repo.superproject", use_superproject)
+
+ if not standalone_manifest:
+ success = self.Sync_NetworkHalf(
+ is_new=is_new,
+ quiet=not verbose,
+ verbose=verbose,
+ clone_bundle=clone_bundle,
+ current_branch_only=current_branch_only,
+ tags=tags,
+ submodules=submodules,
+ clone_filter=clone_filter,
+ partial_clone_exclude=self.manifest.PartialCloneExclude,
+ ).success
+ if not success:
+ r = self.GetRemote()
+ print(
+ "fatal: cannot obtain manifest %s" % r.url, file=sys.stderr
+ )
+
+ # Better delete the manifest git dir if we created it; otherwise
+ # next time (when user fixes problems) we won't go through the
+ # "is_new" logic.
+ if is_new:
+ platform_utils.rmtree(self.gitdir)
+ return False
+
+ if manifest_branch:
+ self.MetaBranchSwitch(submodules=submodules)
+
+ syncbuf = SyncBuffer(self.config)
+ self.Sync_LocalHalf(syncbuf, submodules=submodules)
+ syncbuf.Finish()
+
+ if is_new or self.CurrentBranch is None:
+ if not self.StartBranch("default"):
+ print(
+ "fatal: cannot create default in manifest",
+ file=sys.stderr,
+ )
+ return False
+
+ if not manifest_name:
+ print("fatal: manifest name (-m) is required.", file=sys.stderr)
+ return False
+
+ elif is_new:
+ # This is a new standalone manifest.
+ manifest_name = "default.xml"
+ manifest_data = fetch.fetch_file(manifest_url, verbose=verbose)
+ dest = os.path.join(self.worktree, manifest_name)
+ os.makedirs(os.path.dirname(dest), exist_ok=True)
+ with open(dest, "wb") as f:
+ f.write(manifest_data)
+
+ try:
+ self.manifest.Link(manifest_name)
+ except ManifestParseError as e:
+ print(
+ "fatal: manifest '%s' not available" % manifest_name,
+ file=sys.stderr,
+ )
+ print("fatal: %s" % str(e), file=sys.stderr)
+ return False
+
+ if not this_manifest_only:
+ for submanifest in self.manifest.submanifests.values():
+ spec = submanifest.ToSubmanifestSpec()
+ submanifest.repo_client.manifestProject.Sync(
+ manifest_url=spec.manifestUrl,
+ manifest_branch=spec.revision,
+ standalone_manifest=standalone_manifest,
+ groups=self.manifest_groups,
+ platform=platform,
+ mirror=mirror,
+ dissociate=dissociate,
+ reference=reference,
+ worktree=worktree,
+ submodules=submodules,
+ archive=archive,
+ partial_clone=partial_clone,
+ clone_filter=clone_filter,
+ partial_clone_exclude=partial_clone_exclude,
+ clone_bundle=clone_bundle,
+ git_lfs=git_lfs,
+ use_superproject=use_superproject,
+ verbose=verbose,
+ current_branch_only=current_branch_only,
+ tags=tags,
+ depth=depth,
+ git_event_log=git_event_log,
+ manifest_name=spec.manifestName,
+ this_manifest_only=False,
+ outer_manifest=False,
+ )
+
+ # Lastly, if the manifest has a <superproject> then have the
+ # superproject sync it (if it will be used).
+ if git_superproject.UseSuperproject(use_superproject, self.manifest):
+ sync_result = self.manifest.superproject.Sync(git_event_log)
+ if not sync_result.success:
+ submanifest = ""
+ if self.manifest.path_prefix:
+ submanifest = f"for {self.manifest.path_prefix} "
+ print(
+ f"warning: git update of superproject {submanifest}failed, "
+ "repo sync will not use superproject to fetch source; "
+ "while this error is not fatal, and you can continue to "
+ "run repo sync, please run repo init with the "
+ "--no-use-superproject option to stop seeing this warning",
+ file=sys.stderr,
+ )
+ if sync_result.fatal and use_superproject is not None:
+ return False
+
+ return True
+
+ def _ConfigureDepth(self, depth):
+ """Configure the depth we'll sync down.
+
+ Args:
+ depth: an int, how deep of a partial clone to create.
+ """
+ # Opt.depth will be non-None if user actually passed --depth to repo
+ # init.
+ if depth is not None:
+ if depth > 0:
+ # Positive values will set the depth.
+ depth = str(depth)
+ else:
+ # Negative numbers will clear the depth; passing None to
+ # SetString will do that.
+ depth = None
+
+ # We store the depth in the main manifest project.
+ self.config.SetString("repo.depth", depth)