Merge commits up to 224a31a765eb943443640301a715d2d4eb005b79

This pulls in upstream changes:
224a31a765eb 2017-07-10 14:46:25 -0700 init: add missing submodule arg
b54343d9fd68 2017-07-10 10:31:24 +0200 Tell the user if it will upload a draft
8419ab22d69e 2017-06-16 12:09:06 +0200 sync: Continue job if some fetchs failed but force-broken is set
913327f10c73 2017-06-05 15:01:41 +0200 Add a newline after "Fetching projects" progress output
35d22217a5ed 2016-11-01 11:24:52 -0700 Ensure repo waits for child process to terminate
e0684addeeb9 2017-04-05 00:02:59 -0700 sync: Add support to dump a JSON event log of all sync events.
fef9f21b28d3 2016-11-01 18:28:01 -0700 Fix misplaced file separator string.replace call
6a470be220ab 2016-11-01 11:25:15 -0700 Use OS file separator
c79d3b8fd173 2017-01-24 21:47:50 +0900 init: allow relative path on --reference argument
aa90021fbc33 2017-04-05 13:50:52 -0700 Set result if sys.exit() is called by subcommand.
fddfa6fbac4f 2017-01-09 23:47:52 +0000 Adding include element into the top-level element
eec726c6d880 2016-10-07 10:52:08 +0200 Add option REPO_IGNORE_SSH_INFO to ignore ssh_info
666debc5180f 2017-05-26 21:53:34 +0900 gitc_delete: Remove unused imports
c354a9b92268 2017-05-26 21:52:12 +0900 abandon: fix usage of undefined variable
06848b24150c 2017-05-26 21:44:57 +0900 Update .mailmap
e4e94d26ae81 2017-03-21 16:05:12 -0700 init: add --submodules to sync manifest submodules
3d7bbc9edf2e 2017-04-12 19:51:47 +0800 project.py: fix performance issue with --reference when the mirrored repository has many refs

The submodules work conflicted slightly with our --cache-dir change
because of a change in the Sync_NetworkHalf function signature, but
resolving that was straight forward.

BUG=chromium:900461
TEST=`repo sync` locally

Change-Id: I13dd556fd2ac48e459e7268ec92d6b2e6b917138
diff --git a/project.py b/project.py
index 1b6ab0a..d6e11d2 100644
--- a/project.py
+++ b/project.py
@@ -330,13 +330,15 @@
                pushUrl=None,
                review=None,
                revision=None,
-               orig_name=None):
+               orig_name=None,
+               fetchUrl=None):
     self.name = name
     self.url = url
     self.pushUrl = pushUrl
     self.review = review
     self.revision = revision
     self.orig_name = orig_name
+    self.fetchUrl = fetchUrl
 
 
 class RepoHook(object):
@@ -694,7 +696,7 @@
     self.gitdir = gitdir.replace('\\', '/')
     self.objdir = objdir.replace('\\', '/')
     if worktree:
-      self.worktree = os.path.normpath(worktree.replace('\\', '/'))
+      self.worktree = os.path.normpath(worktree).replace('\\', '/')
     else:
       self.worktree = None
     self.relpath = relpath
@@ -1272,6 +1274,7 @@
                        archive=False,
                        optimized_fetch=False,
                        prune=False,
+                       submodules=False,
                        cache_dir=None):
     """Perform only the network IO portion of the sync process.
        Local working directory/branch state is not affected.
@@ -1364,7 +1367,8 @@
     if (need_to_fetch and
         not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
                               current_branch_only=current_branch_only,
-                              no_tags=no_tags, prune=prune, depth=depth)):
+                              no_tags=no_tags, prune=prune, depth=depth,
+                              submodules=submodules)):
       return False
 
     if self.worktree:
@@ -1420,11 +1424,11 @@
       raise ManifestInvalidRevisionError('revision %s in %s not found' %
                                          (self.revisionExpr, self.name))
 
-  def Sync_LocalHalf(self, syncbuf, force_sync=False):
+  def Sync_LocalHalf(self, syncbuf, force_sync=False, submodules=False):
     """Perform only the local IO portion of the sync process.
        Network access is not required.
     """
-    self._InitWorkTree(force_sync=force_sync)
+    self._InitWorkTree(force_sync=force_sync, submodules=submodules)
     all_refs = self.bare_ref.all
     self.CleanPublishedCache(all_refs)
     revid = self.GetRevisionId(all_refs)
@@ -1433,6 +1437,9 @@
       self._FastForward(revid)
       self._CopyAndLinkFiles()
 
+    def _dosubmodules():
+      self._SyncSubmodules(quiet=True)
+
     head = self.work_git.GetHead()
     if head.startswith(R_HEADS):
       branch = head[len(R_HEADS):]
@@ -1466,6 +1473,8 @@
 
       try:
         self._Checkout(revid, quiet=True)
+        if submodules:
+          self._SyncSubmodules(quiet=True)
       except GitError as e:
         syncbuf.fail(self, e)
         return
@@ -1490,6 +1499,8 @@
                    branch.name)
       try:
         self._Checkout(revid, quiet=True)
+        if submodules:
+          self._SyncSubmodules(quiet=True)
       except GitError as e:
         syncbuf.fail(self, e)
         return
@@ -1515,6 +1526,8 @@
         # strict subset.  We can fast-forward safely.
         #
         syncbuf.later1(self, _doff)
+        if submodules:
+          syncbuf.later1(self, _dosubmodules)
         return
 
     # Examine the local commits not in the remote.  Find the
@@ -1566,19 +1579,28 @@
     branch.Save()
 
     if cnt_mine > 0 and self.rebase:
+      def _docopyandlink():
+        self._CopyAndLinkFiles()
+
       def _dorebase():
         self._Rebase(upstream='%s^1' % last_mine, onto=revid)
-        self._CopyAndLinkFiles()
       syncbuf.later2(self, _dorebase)
+      if submodules:
+        syncbuf.later2(self, _dosubmodules)
+      syncbuf.later2(self, _docopyandlink)
     elif local_changes:
       try:
         self._ResetHard(revid)
+        if submodules:
+          self._SyncSubmodules(quiet=True)
         self._CopyAndLinkFiles()
       except GitError as e:
         syncbuf.fail(self, e)
         return
     else:
       syncbuf.later1(self, _doff)
+      if submodules:
+        syncbuf.later1(self, _dosubmodules)
 
   def AddCopyFile(self, src, dest, absdest):
     # dest should already be an absolute path, but src is project relative
@@ -1981,7 +2003,8 @@
                    alt_dir=None,
                    no_tags=False,
                    prune=False,
-                   depth=None):
+                   depth=None,
+                   submodules=False):
 
     is_sha1 = False
     tag_name = None
@@ -2052,15 +2075,17 @@
           ids.add(ref_id)
           tmp.add(r)
 
-        tmp_packed = ''
-        old_packed = ''
+        tmp_packed_lines = []
+        old_packed_lines = []
 
         for r in sorted(all_refs):
           line = '%s %s\n' % (all_refs[r], r)
-          tmp_packed += line
+          tmp_packed_lines.append(line)
           if r not in tmp:
-            old_packed += line
+            old_packed_lines.append(line)
 
+        tmp_packed = ''.join(tmp_packed_lines)
+        old_packed = ''.join(old_packed_lines)
         _lwrite(packed_refs, tmp_packed)
       else:
         alt_dir = None
@@ -2093,6 +2118,9 @@
     if prune:
       cmd.append('--prune')
 
+    if submodules:
+      cmd.append('--recurse-submodules=on-demand')
+
     spec = []
     if not current_branch_only:
       # Fetch whole repo
@@ -2313,6 +2341,13 @@
     if GitCommand(self, cmd).Wait() != 0:
       raise GitError('%s reset --hard %s ' % (self.name, rev))
 
+  def _SyncSubmodules(self, quiet=True):
+    cmd = ['submodule', 'update', '--init', '--recursive']
+    if quiet:
+      cmd.append('-q')
+    if GitCommand(self, cmd).Wait() != 0:
+      raise GitError('%s submodule update --init --recursive %s ' % self.name)
+
   def _Rebase(self, upstream, onto=None):
     cmd = ['rebase']
     if onto is not None:
@@ -2553,7 +2588,7 @@
         else:
           raise
 
-  def _InitWorkTree(self, force_sync=False):
+  def _InitWorkTree(self, force_sync=False, submodules=False):
     dotgit = os.path.join(self.worktree, '.git')
     init_dotgit = not os.path.exists(dotgit)
     try:
@@ -2568,7 +2603,7 @@
         if force_sync:
           try:
             shutil.rmtree(dotgit)
-            return self._InitWorkTree(force_sync=False)
+            return self._InitWorkTree(force_sync=False, submodules=submodules)
           except:
             raise e
         raise e
@@ -2582,6 +2617,8 @@
         if GitCommand(self, cmd).Wait() != 0:
           raise GitError("cannot initialize work tree")
 
+        if submodules:
+          self._SyncSubmodules(quiet=True)
         self._CopyAndLinkFiles()
     except Exception:
       if init_dotgit:
@@ -2930,13 +2967,14 @@
 
     self.detach_head = detach_head
     self.clean = True
+    self.recent_clean = True
 
   def info(self, project, fmt, *args):
     self._messages.append(_InfoMessage(project, fmt % args))
 
   def fail(self, project, err=None):
     self._failures.append(_Failure(project, err))
-    self.clean = False
+    self._MarkUnclean()
 
   def later1(self, project, what):
     self._later_queue1.append(_Later(project, what))
@@ -2950,6 +2988,15 @@
     self._PrintMessages()
     return self.clean
 
+  def Recently(self):
+    recent_clean = self.recent_clean
+    self.recent_clean = True
+    return recent_clean
+
+  def _MarkUnclean(self):
+    self.clean = False
+    self.recent_clean = False
+
   def _RunLater(self):
     for q in ['_later_queue1', '_later_queue2']:
       if not self._RunQueue(q):
@@ -2958,7 +3005,7 @@
   def _RunQueue(self, queue):
     for m in getattr(self, queue):
       if not m.Run(self):
-        self.clean = False
+        self._MarkUnclean()
         return False
     setattr(self, queue, [])
     return True
@@ -3000,14 +3047,14 @@
           self.revisionExpr = base
           self.revisionId = None
 
-  def MetaBranchSwitch(self):
+  def MetaBranchSwitch(self, submodules=False):
     """ Prepare MetaProject for manifest branch switch
     """
 
     # detach and delete manifest branch, allowing a new
     # branch to take over
     syncbuf = SyncBuffer(self.config, detach_head=True)
-    self.Sync_LocalHalf(syncbuf)
+    self.Sync_LocalHalf(syncbuf, submodules=submodules)
     syncbuf.Finish()
 
     return GitCommand(self,