Merge commits up to 224a31a765eb943443640301a715d2d4eb005b79
This pulls in upstream changes:
224a31a765eb 2017-07-10 14:46:25 -0700 init: add missing submodule arg
b54343d9fd68 2017-07-10 10:31:24 +0200 Tell the user if it will upload a draft
8419ab22d69e 2017-06-16 12:09:06 +0200 sync: Continue job if some fetchs failed but force-broken is set
913327f10c73 2017-06-05 15:01:41 +0200 Add a newline after "Fetching projects" progress output
35d22217a5ed 2016-11-01 11:24:52 -0700 Ensure repo waits for child process to terminate
e0684addeeb9 2017-04-05 00:02:59 -0700 sync: Add support to dump a JSON event log of all sync events.
fef9f21b28d3 2016-11-01 18:28:01 -0700 Fix misplaced file separator string.replace call
6a470be220ab 2016-11-01 11:25:15 -0700 Use OS file separator
c79d3b8fd173 2017-01-24 21:47:50 +0900 init: allow relative path on --reference argument
aa90021fbc33 2017-04-05 13:50:52 -0700 Set result if sys.exit() is called by subcommand.
fddfa6fbac4f 2017-01-09 23:47:52 +0000 Adding include element into the top-level element
eec726c6d880 2016-10-07 10:52:08 +0200 Add option REPO_IGNORE_SSH_INFO to ignore ssh_info
666debc5180f 2017-05-26 21:53:34 +0900 gitc_delete: Remove unused imports
c354a9b92268 2017-05-26 21:52:12 +0900 abandon: fix usage of undefined variable
06848b24150c 2017-05-26 21:44:57 +0900 Update .mailmap
e4e94d26ae81 2017-03-21 16:05:12 -0700 init: add --submodules to sync manifest submodules
3d7bbc9edf2e 2017-04-12 19:51:47 +0800 project.py: fix performance issue with --reference when the mirrored repository has many refs
The submodules work conflicted slightly with our --cache-dir change
because of a change in the Sync_NetworkHalf function signature, but
resolving that was straight forward.
BUG=chromium:900461
TEST=`repo sync` locally
Change-Id: I13dd556fd2ac48e459e7268ec92d6b2e6b917138
diff --git a/project.py b/project.py
index 1b6ab0a..d6e11d2 100644
--- a/project.py
+++ b/project.py
@@ -330,13 +330,15 @@
pushUrl=None,
review=None,
revision=None,
- orig_name=None):
+ orig_name=None,
+ fetchUrl=None):
self.name = name
self.url = url
self.pushUrl = pushUrl
self.review = review
self.revision = revision
self.orig_name = orig_name
+ self.fetchUrl = fetchUrl
class RepoHook(object):
@@ -694,7 +696,7 @@
self.gitdir = gitdir.replace('\\', '/')
self.objdir = objdir.replace('\\', '/')
if worktree:
- self.worktree = os.path.normpath(worktree.replace('\\', '/'))
+ self.worktree = os.path.normpath(worktree).replace('\\', '/')
else:
self.worktree = None
self.relpath = relpath
@@ -1272,6 +1274,7 @@
archive=False,
optimized_fetch=False,
prune=False,
+ submodules=False,
cache_dir=None):
"""Perform only the network IO portion of the sync process.
Local working directory/branch state is not affected.
@@ -1364,7 +1367,8 @@
if (need_to_fetch and
not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
current_branch_only=current_branch_only,
- no_tags=no_tags, prune=prune, depth=depth)):
+ no_tags=no_tags, prune=prune, depth=depth,
+ submodules=submodules)):
return False
if self.worktree:
@@ -1420,11 +1424,11 @@
raise ManifestInvalidRevisionError('revision %s in %s not found' %
(self.revisionExpr, self.name))
- def Sync_LocalHalf(self, syncbuf, force_sync=False):
+ def Sync_LocalHalf(self, syncbuf, force_sync=False, submodules=False):
"""Perform only the local IO portion of the sync process.
Network access is not required.
"""
- self._InitWorkTree(force_sync=force_sync)
+ self._InitWorkTree(force_sync=force_sync, submodules=submodules)
all_refs = self.bare_ref.all
self.CleanPublishedCache(all_refs)
revid = self.GetRevisionId(all_refs)
@@ -1433,6 +1437,9 @@
self._FastForward(revid)
self._CopyAndLinkFiles()
+ def _dosubmodules():
+ self._SyncSubmodules(quiet=True)
+
head = self.work_git.GetHead()
if head.startswith(R_HEADS):
branch = head[len(R_HEADS):]
@@ -1466,6 +1473,8 @@
try:
self._Checkout(revid, quiet=True)
+ if submodules:
+ self._SyncSubmodules(quiet=True)
except GitError as e:
syncbuf.fail(self, e)
return
@@ -1490,6 +1499,8 @@
branch.name)
try:
self._Checkout(revid, quiet=True)
+ if submodules:
+ self._SyncSubmodules(quiet=True)
except GitError as e:
syncbuf.fail(self, e)
return
@@ -1515,6 +1526,8 @@
# strict subset. We can fast-forward safely.
#
syncbuf.later1(self, _doff)
+ if submodules:
+ syncbuf.later1(self, _dosubmodules)
return
# Examine the local commits not in the remote. Find the
@@ -1566,19 +1579,28 @@
branch.Save()
if cnt_mine > 0 and self.rebase:
+ def _docopyandlink():
+ self._CopyAndLinkFiles()
+
def _dorebase():
self._Rebase(upstream='%s^1' % last_mine, onto=revid)
- self._CopyAndLinkFiles()
syncbuf.later2(self, _dorebase)
+ if submodules:
+ syncbuf.later2(self, _dosubmodules)
+ syncbuf.later2(self, _docopyandlink)
elif local_changes:
try:
self._ResetHard(revid)
+ if submodules:
+ self._SyncSubmodules(quiet=True)
self._CopyAndLinkFiles()
except GitError as e:
syncbuf.fail(self, e)
return
else:
syncbuf.later1(self, _doff)
+ if submodules:
+ syncbuf.later1(self, _dosubmodules)
def AddCopyFile(self, src, dest, absdest):
# dest should already be an absolute path, but src is project relative
@@ -1981,7 +2003,8 @@
alt_dir=None,
no_tags=False,
prune=False,
- depth=None):
+ depth=None,
+ submodules=False):
is_sha1 = False
tag_name = None
@@ -2052,15 +2075,17 @@
ids.add(ref_id)
tmp.add(r)
- tmp_packed = ''
- old_packed = ''
+ tmp_packed_lines = []
+ old_packed_lines = []
for r in sorted(all_refs):
line = '%s %s\n' % (all_refs[r], r)
- tmp_packed += line
+ tmp_packed_lines.append(line)
if r not in tmp:
- old_packed += line
+ old_packed_lines.append(line)
+ tmp_packed = ''.join(tmp_packed_lines)
+ old_packed = ''.join(old_packed_lines)
_lwrite(packed_refs, tmp_packed)
else:
alt_dir = None
@@ -2093,6 +2118,9 @@
if prune:
cmd.append('--prune')
+ if submodules:
+ cmd.append('--recurse-submodules=on-demand')
+
spec = []
if not current_branch_only:
# Fetch whole repo
@@ -2313,6 +2341,13 @@
if GitCommand(self, cmd).Wait() != 0:
raise GitError('%s reset --hard %s ' % (self.name, rev))
+ def _SyncSubmodules(self, quiet=True):
+ cmd = ['submodule', 'update', '--init', '--recursive']
+ if quiet:
+ cmd.append('-q')
+ if GitCommand(self, cmd).Wait() != 0:
+ raise GitError('%s submodule update --init --recursive %s ' % self.name)
+
def _Rebase(self, upstream, onto=None):
cmd = ['rebase']
if onto is not None:
@@ -2553,7 +2588,7 @@
else:
raise
- def _InitWorkTree(self, force_sync=False):
+ def _InitWorkTree(self, force_sync=False, submodules=False):
dotgit = os.path.join(self.worktree, '.git')
init_dotgit = not os.path.exists(dotgit)
try:
@@ -2568,7 +2603,7 @@
if force_sync:
try:
shutil.rmtree(dotgit)
- return self._InitWorkTree(force_sync=False)
+ return self._InitWorkTree(force_sync=False, submodules=submodules)
except:
raise e
raise e
@@ -2582,6 +2617,8 @@
if GitCommand(self, cmd).Wait() != 0:
raise GitError("cannot initialize work tree")
+ if submodules:
+ self._SyncSubmodules(quiet=True)
self._CopyAndLinkFiles()
except Exception:
if init_dotgit:
@@ -2930,13 +2967,14 @@
self.detach_head = detach_head
self.clean = True
+ self.recent_clean = True
def info(self, project, fmt, *args):
self._messages.append(_InfoMessage(project, fmt % args))
def fail(self, project, err=None):
self._failures.append(_Failure(project, err))
- self.clean = False
+ self._MarkUnclean()
def later1(self, project, what):
self._later_queue1.append(_Later(project, what))
@@ -2950,6 +2988,15 @@
self._PrintMessages()
return self.clean
+ def Recently(self):
+ recent_clean = self.recent_clean
+ self.recent_clean = True
+ return recent_clean
+
+ def _MarkUnclean(self):
+ self.clean = False
+ self.recent_clean = False
+
def _RunLater(self):
for q in ['_later_queue1', '_later_queue2']:
if not self._RunQueue(q):
@@ -2958,7 +3005,7 @@
def _RunQueue(self, queue):
for m in getattr(self, queue):
if not m.Run(self):
- self.clean = False
+ self._MarkUnclean()
return False
setattr(self, queue, [])
return True
@@ -3000,14 +3047,14 @@
self.revisionExpr = base
self.revisionId = None
- def MetaBranchSwitch(self):
+ def MetaBranchSwitch(self, submodules=False):
""" Prepare MetaProject for manifest branch switch
"""
# detach and delete manifest branch, allowing a new
# branch to take over
syncbuf = SyncBuffer(self.config, detach_head=True)
- self.Sync_LocalHalf(syncbuf)
+ self.Sync_LocalHalf(syncbuf, submodules=submodules)
syncbuf.Finish()
return GitCommand(self,