blob: a4c0dfd83b1f2a970d0fd1416032402f3b354b84 [file] [log] [blame]
steveblock@chromium.org93567042012-02-15 01:02:26 +00001# Copyright (c) 2012 The Chromium Authors. All rights reserved.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +00002# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
maruel@chromium.org5f3eee32009-09-17 00:34:30 +00004
maruel@chromium.orgd5800f12009-11-12 20:03:43 +00005"""Gclient-specific SCM-specific operations."""
maruel@chromium.org5f3eee32009-09-17 00:34:30 +00006
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00007from __future__ import print_function
8
John Budorick0f7b2002018-01-19 15:46:17 -08009import collections
10import contextlib
borenet@google.comb2256212014-05-07 20:57:28 +000011import errno
John Budorick0f7b2002018-01-19 15:46:17 -080012import json
maruel@chromium.org754960e2009-09-21 12:31:05 +000013import logging
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000014import os
maruel@chromium.orgee4071d2009-12-22 22:25:37 +000015import posixpath
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000016import re
maruel@chromium.org90541732011-04-01 17:54:18 +000017import sys
ilevy@chromium.org3534aa52013-07-20 01:58:08 +000018import tempfile
John Budorick0f7b2002018-01-19 15:46:17 -080019import threading
zty@chromium.org6279e8a2014-02-13 01:45:25 +000020import traceback
hinoka@google.com2f2ca142014-01-07 03:59:18 +000021import urlparse
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000022
hinoka@google.com2f2ca142014-01-07 03:59:18 +000023import download_from_google_storage
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000024import gclient_utils
szager@chromium.org848fd492014-04-09 19:06:44 +000025import git_cache
maruel@chromium.org31cb48a2011-04-04 18:01:36 +000026import scm
borenet@google.comb2256212014-05-07 20:57:28 +000027import shutil
maruel@chromium.org31cb48a2011-04-04 18:01:36 +000028import subprocess2
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000029
30
szager@chromium.org71cbb502013-04-19 23:30:15 +000031THIS_FILE_PATH = os.path.abspath(__file__)
32
hinoka@google.com2f2ca142014-01-07 03:59:18 +000033GSUTIL_DEFAULT_PATH = os.path.join(
hinoka@chromium.orgb091aa52014-12-20 01:47:31 +000034 os.path.dirname(os.path.abspath(__file__)), 'gsutil.py')
hinoka@google.com2f2ca142014-01-07 03:59:18 +000035
maruel@chromium.org79d62372015-06-01 18:50:55 +000036
smutae7ea312016-07-18 11:59:41 -070037class NoUsableRevError(gclient_utils.Error):
38 """Raised if requested revision isn't found in checkout."""
39
40
haitao.feng@intel.com306080c2012-05-04 13:11:29 +000041class DiffFiltererWrapper(object):
42 """Simple base class which tracks which file is being diffed and
maruel@chromium.orgee4071d2009-12-22 22:25:37 +000043 replaces instances of its file name in the original and
agable41e3a6c2016-10-20 11:36:56 -070044 working copy lines of the git diff output."""
haitao.feng@intel.com306080c2012-05-04 13:11:29 +000045 index_string = None
maruel@chromium.orgee4071d2009-12-22 22:25:37 +000046 original_prefix = "--- "
47 working_prefix = "+++ "
48
szager@chromium.orgfe0d1902014-04-08 20:50:44 +000049 def __init__(self, relpath, print_func):
maruel@chromium.orgee4071d2009-12-22 22:25:37 +000050 # Note that we always use '/' as the path separator to be
agable41e3a6c2016-10-20 11:36:56 -070051 # consistent with cygwin-style output on Windows
maruel@chromium.orgee4071d2009-12-22 22:25:37 +000052 self._relpath = relpath.replace("\\", "/")
haitao.feng@intel.com306080c2012-05-04 13:11:29 +000053 self._current_file = None
szager@chromium.orgfe0d1902014-04-08 20:50:44 +000054 self._print_func = print_func
maruel@chromium.orgee4071d2009-12-22 22:25:37 +000055
maruel@chromium.org6e29d572010-06-04 17:32:20 +000056 def SetCurrentFile(self, current_file):
57 self._current_file = current_file
haitao.feng@intel.com306080c2012-05-04 13:11:29 +000058
iannucci@chromium.org3830a672013-02-19 20:15:14 +000059 @property
60 def _replacement_file(self):
haitao.feng@intel.com306080c2012-05-04 13:11:29 +000061 return posixpath.join(self._relpath, self._current_file)
maruel@chromium.orgee4071d2009-12-22 22:25:37 +000062
maruel@chromium.orgf5d37bf2010-09-02 00:50:34 +000063 def _Replace(self, line):
64 return line.replace(self._current_file, self._replacement_file)
maruel@chromium.orgee4071d2009-12-22 22:25:37 +000065
66 def Filter(self, line):
67 if (line.startswith(self.index_string)):
68 self.SetCurrentFile(line[len(self.index_string):])
maruel@chromium.orgf5d37bf2010-09-02 00:50:34 +000069 line = self._Replace(line)
maruel@chromium.orgee4071d2009-12-22 22:25:37 +000070 else:
71 if (line.startswith(self.original_prefix) or
72 line.startswith(self.working_prefix)):
maruel@chromium.orgf5d37bf2010-09-02 00:50:34 +000073 line = self._Replace(line)
szager@chromium.orgfe0d1902014-04-08 20:50:44 +000074 self._print_func(line)
maruel@chromium.orgee4071d2009-12-22 22:25:37 +000075
76
haitao.feng@intel.com306080c2012-05-04 13:11:29 +000077class GitDiffFilterer(DiffFiltererWrapper):
78 index_string = "diff --git "
79
80 def SetCurrentFile(self, current_file):
81 # Get filename by parsing "a/<filename> b/<filename>"
82 self._current_file = current_file[:(len(current_file)/2)][2:]
83
84 def _Replace(self, line):
85 return re.sub("[a|b]/" + self._current_file, self._replacement_file, line)
86
87
msb@chromium.orgcb5442b2009-09-22 16:51:24 +000088# SCMWrapper base class
89
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000090class SCMWrapper(object):
91 """Add necessary glue between all the supported SCM.
92
msb@chromium.orgd6504212010-01-13 17:34:31 +000093 This is the abstraction layer to bind to different SCM.
94 """
szager@chromium.orgfe0d1902014-04-08 20:50:44 +000095 def __init__(self, url=None, root_dir=None, relpath=None, out_fh=None,
Edward Lemur231f5ea2018-01-31 19:02:36 +010096 out_cb=None, print_outbuf=False):
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000097 self.url = url
maruel@chromium.org5e73b0c2009-09-18 19:47:48 +000098 self._root_dir = root_dir
99 if self._root_dir:
100 self._root_dir = self._root_dir.replace('/', os.sep)
101 self.relpath = relpath
102 if self.relpath:
103 self.relpath = self.relpath.replace('/', os.sep)
msb@chromium.orge28e4982009-09-25 20:51:45 +0000104 if self.relpath and self._root_dir:
105 self.checkout_path = os.path.join(self._root_dir, self.relpath)
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000106 if out_fh is None:
107 out_fh = sys.stdout
108 self.out_fh = out_fh
109 self.out_cb = out_cb
Edward Lemur231f5ea2018-01-31 19:02:36 +0100110 self.print_outbuf = print_outbuf
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000111
112 def Print(self, *args, **kwargs):
113 kwargs.setdefault('file', self.out_fh)
114 if kwargs.pop('timestamp', True):
115 self.out_fh.write('[%s] ' % gclient_utils.Elapsed())
116 print(*args, **kwargs)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000117
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000118 def RunCommand(self, command, options, args, file_list=None):
agabledebf6c82016-12-21 12:50:12 -0800119 commands = ['update', 'updatesingle', 'revert',
tony@chromium.org4b5b1772010-04-08 01:52:56 +0000120 'revinfo', 'status', 'diff', 'pack', 'runhooks']
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000121
122 if not command in commands:
123 raise gclient_utils.Error('Unknown command %s' % command)
124
msb@chromium.orgcb5442b2009-09-22 16:51:24 +0000125 if not command in dir(self):
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000126 raise gclient_utils.Error('Command %s not implemented in %s wrapper' % (
maruel@chromium.org9eda4112010-06-11 18:56:10 +0000127 command, self.__class__.__name__))
msb@chromium.orgcb5442b2009-09-22 16:51:24 +0000128
129 return getattr(self, command)(options, args, file_list)
130
hinoka@chromium.orgfa2b9b42014-08-22 18:08:53 +0000131 @staticmethod
132 def _get_first_remote_url(checkout_path):
133 log = scm.GIT.Capture(
134 ['config', '--local', '--get-regexp', r'remote.*.url'],
135 cwd=checkout_path)
136 # Get the second token of the first line of the log.
137 return log.splitlines()[0].split(' ', 1)[1]
138
levarum@chromium.org27a6f9a2016-05-28 00:21:49 +0000139 def GetCacheMirror(self):
Robert Iannuccia19649b2018-06-29 16:31:45 +0000140 if getattr(self, 'cache_dir', None):
levarum@chromium.org27a6f9a2016-05-28 00:21:49 +0000141 url, _ = gclient_utils.SplitUrlRevision(self.url)
142 return git_cache.Mirror(url)
143 return None
144
smut@google.comd33eab32014-07-07 19:35:18 +0000145 def GetActualRemoteURL(self, options):
borenet@google.com88d10082014-03-21 17:24:48 +0000146 """Attempt to determine the remote URL for this SCMWrapper."""
smut@google.comd33eab32014-07-07 19:35:18 +0000147 # Git
borenet@google.combda475e2014-03-24 19:04:45 +0000148 if os.path.exists(os.path.join(self.checkout_path, '.git')):
hinoka@chromium.orgfa2b9b42014-08-22 18:08:53 +0000149 actual_remote_url = self._get_first_remote_url(self.checkout_path)
borenet@google.com4e9be262014-04-08 19:40:30 +0000150
levarum@chromium.org27a6f9a2016-05-28 00:21:49 +0000151 mirror = self.GetCacheMirror()
152 # If the cache is used, obtain the actual remote URL from there.
153 if (mirror and mirror.exists() and
154 mirror.mirror_path.replace('\\', '/') ==
155 actual_remote_url.replace('\\', '/')):
156 actual_remote_url = self._get_first_remote_url(mirror.mirror_path)
borenet@google.com4e9be262014-04-08 19:40:30 +0000157 return actual_remote_url
borenet@google.com88d10082014-03-21 17:24:48 +0000158 return None
159
borenet@google.com4e9be262014-04-08 19:40:30 +0000160 def DoesRemoteURLMatch(self, options):
borenet@google.com88d10082014-03-21 17:24:48 +0000161 """Determine whether the remote URL of this checkout is the expected URL."""
162 if not os.path.exists(self.checkout_path):
163 # A checkout which doesn't exist can't be broken.
164 return True
165
smut@google.comd33eab32014-07-07 19:35:18 +0000166 actual_remote_url = self.GetActualRemoteURL(options)
borenet@google.com88d10082014-03-21 17:24:48 +0000167 if actual_remote_url:
borenet@google.com8156c9f2014-04-01 16:41:36 +0000168 return (gclient_utils.SplitUrlRevision(actual_remote_url)[0].rstrip('/')
169 == gclient_utils.SplitUrlRevision(self.url)[0].rstrip('/'))
borenet@google.com88d10082014-03-21 17:24:48 +0000170 else:
171 # This may occur if the self.checkout_path exists but does not contain a
agable41e3a6c2016-10-20 11:36:56 -0700172 # valid git checkout.
borenet@google.com88d10082014-03-21 17:24:48 +0000173 return False
174
borenet@google.comb09097a2014-04-09 19:09:08 +0000175 def _DeleteOrMove(self, force):
176 """Delete the checkout directory or move it out of the way.
177
178 Args:
179 force: bool; if True, delete the directory. Otherwise, just move it.
180 """
borenet@google.comb2256212014-05-07 20:57:28 +0000181 if force and os.environ.get('CHROME_HEADLESS') == '1':
182 self.Print('_____ Conflicting directory found in %s. Removing.'
183 % self.checkout_path)
184 gclient_utils.AddWarning('Conflicting directory %s deleted.'
185 % self.checkout_path)
186 gclient_utils.rmtree(self.checkout_path)
187 else:
188 bad_scm_dir = os.path.join(self._root_dir, '_bad_scm',
189 os.path.dirname(self.relpath))
190
191 try:
192 os.makedirs(bad_scm_dir)
193 except OSError as e:
194 if e.errno != errno.EEXIST:
195 raise
196
197 dest_path = tempfile.mkdtemp(
198 prefix=os.path.basename(self.relpath),
199 dir=bad_scm_dir)
200 self.Print('_____ Conflicting directory found in %s. Moving to %s.'
201 % (self.checkout_path, dest_path))
202 gclient_utils.AddWarning('Conflicting directory %s moved to %s.'
203 % (self.checkout_path, dest_path))
204 shutil.move(self.checkout_path, dest_path)
borenet@google.comb09097a2014-04-09 19:09:08 +0000205
msb@chromium.orgcb5442b2009-09-22 16:51:24 +0000206
maruel@chromium.org55e724e2010-03-11 19:36:49 +0000207class GitWrapper(SCMWrapper):
msb@chromium.orge28e4982009-09-25 20:51:45 +0000208 """Wrapper for Git"""
iannucci@chromium.org2702bcd2013-09-24 19:10:07 +0000209 name = 'git'
maruel@chromium.org1a60dca2013-11-26 14:06:26 +0000210 remote = 'origin'
msb@chromium.orge28e4982009-09-25 20:51:45 +0000211
Robert Iannuccia19649b2018-06-29 16:31:45 +0000212 @property
213 def cache_dir(self):
214 try:
215 return git_cache.Mirror.GetCachePath()
216 except RuntimeError:
217 return None
iannucci@chromium.org53456aa2013-07-03 19:38:34 +0000218
John Budorick0f7b2002018-01-19 15:46:17 -0800219 def __init__(self, url=None, *args, **kwargs):
igorgatis@gmail.com4e075672011-11-21 16:35:08 +0000220 """Removes 'git+' fake prefix from git URL."""
221 if url.startswith('git+http://') or url.startswith('git+https://'):
222 url = url[4:]
John Budorick0f7b2002018-01-19 15:46:17 -0800223 SCMWrapper.__init__(self, url, *args, **kwargs)
szager@chromium.org848fd492014-04-09 19:06:44 +0000224 filter_kwargs = { 'time_throttle': 1, 'out_fh': self.out_fh }
225 if self.out_cb:
226 filter_kwargs['predicate'] = self.out_cb
227 self.filter = gclient_utils.GitFilter(**filter_kwargs)
igorgatis@gmail.com4e075672011-11-21 16:35:08 +0000228
mukai@chromium.org9e3e82c2012-04-18 12:55:43 +0000229 @staticmethod
230 def BinaryExists():
231 """Returns true if the command exists."""
232 try:
233 # We assume git is newer than 1.7. See: crbug.com/114483
234 result, version = scm.GIT.AssertVersion('1.7')
235 if not result:
236 raise gclient_utils.Error('Git version is older than 1.7: %s' % version)
237 return result
238 except OSError:
239 return False
240
xusydoc@chromium.org885a9602013-05-31 09:54:40 +0000241 def GetCheckoutRoot(self):
242 return scm.GIT.GetCheckoutRoot(self.checkout_path)
243
iannucci@chromium.org396e1a62013-07-03 19:41:04 +0000244 def GetRevisionDate(self, _revision):
floitsch@google.comeaab7842011-04-28 09:07:58 +0000245 """Returns the given revision's date in ISO-8601 format (which contains the
246 time zone)."""
247 # TODO(floitsch): get the time-stamp of the given revision and not just the
248 # time-stamp of the currently checked out revision.
249 return self._Capture(['log', '-n', '1', '--format=%ai'])
250
Aaron Gablef4068aa2017-12-12 15:14:09 -0800251 def _GetDiffFilenames(self, base):
252 """Returns the names of files modified since base."""
253 return self._Capture(
254 # Filter to remove base if it is None.
255 filter(bool, ['-c', 'core.quotePath=false', 'diff', '--name-only', base])
256 ).split()
257
iannucci@chromium.org396e1a62013-07-03 19:41:04 +0000258 def diff(self, options, _args, _file_list):
Aaron Gable1853f662018-02-12 15:45:56 -0800259 _, revision = gclient_utils.SplitUrlRevision(self.url)
260 if not revision:
261 revision = 'refs/remotes/%s/master' % self.remote
262 self._Run(['-c', 'core.quotePath=false', 'diff', revision], options)
msb@chromium.orge28e4982009-09-25 20:51:45 +0000263
iannucci@chromium.org396e1a62013-07-03 19:41:04 +0000264 def pack(self, _options, _args, _file_list):
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000265 """Generates a patch file which can be applied to the root of the
msb@chromium.orgd6504212010-01-13 17:34:31 +0000266 repository.
267
268 The patch file is generated from a diff of the merge base of HEAD and
269 its upstream branch.
270 """
raphael.kubo.da.costa05c83592016-08-04 08:32:41 -0700271 try:
272 merge_base = [self._Capture(['merge-base', 'HEAD', self.remote])]
273 except subprocess2.CalledProcessError:
274 merge_base = []
maruel@chromium.org17d01792010-09-01 18:07:10 +0000275 gclient_utils.CheckCallAndFilter(
raphael.kubo.da.costa05c83592016-08-04 08:32:41 -0700276 ['git', 'diff'] + merge_base,
maruel@chromium.org8469bf92010-09-03 19:03:15 +0000277 cwd=self.checkout_path,
avakulenko@google.com255f2be2014-12-05 22:19:55 +0000278 filter_fn=GitDiffFilterer(self.relpath, print_func=self.Print).Filter)
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000279
Robert Iannuccic41d8b92017-02-16 17:07:37 -0800280 def _Scrub(self, target, options):
281 """Scrubs out all changes in the local repo, back to the state of target."""
mmoss@chromium.org50fd47f2014-02-13 01:03:19 +0000282 quiet = []
283 if not options.verbose:
284 quiet = ['--quiet']
Robert Iannuccic41d8b92017-02-16 17:07:37 -0800285 self._Run(['reset', '--hard', target] + quiet, options)
286 if options.force and options.delete_unversioned_trees:
287 # where `target` is a commit that contains both upper and lower case
288 # versions of the same file on a case insensitive filesystem, we are
289 # actually in a broken state here. The index will have both 'a' and 'A',
290 # but only one of them will exist on the disk. To progress, we delete
291 # everything that status thinks is modified.
Aaron Gable7817f022017-12-12 09:43:17 -0800292 output = self._Capture([
293 '-c', 'core.quotePath=false', 'status', '--porcelain'], strip=False)
Robert Iannuccia7a9ceb2017-02-16 17:38:06 -0800294 for line in output.splitlines():
Robert Iannuccic41d8b92017-02-16 17:07:37 -0800295 # --porcelain (v1) looks like:
296 # XY filename
297 try:
298 filename = line[3:]
299 self.Print('_____ Deleting residual after reset: %r.' % filename)
300 gclient_utils.rm_file_or_tree(
Robert Iannuccia7a9ceb2017-02-16 17:38:06 -0800301 os.path.join(self.checkout_path, filename))
Robert Iannuccic41d8b92017-02-16 17:07:37 -0800302 except OSError:
303 pass
304
John Budorick882c91e2018-07-12 22:11:41 +0000305 def _FetchAndReset(self, revision, file_list, options):
Robert Iannuccic41d8b92017-02-16 17:07:37 -0800306 """Equivalent to git fetch; git reset."""
Edward Lemur579c9862018-07-13 23:17:51 +0000307 self._SetFetchConfig(options)
mmoss@chromium.org50fd47f2014-02-13 01:03:19 +0000308
dnj@chromium.org680f2172014-06-25 00:39:32 +0000309 self._Fetch(options, prune=True, quiet=options.verbose)
John Budorick882c91e2018-07-12 22:11:41 +0000310 self._Scrub(revision, options)
mmoss@chromium.org50fd47f2014-02-13 01:03:19 +0000311 if file_list is not None:
Aaron Gable7817f022017-12-12 09:43:17 -0800312 files = self._Capture(
313 ['-c', 'core.quotePath=false', 'ls-files']).splitlines()
mmoss@chromium.org50fd47f2014-02-13 01:03:19 +0000314 file_list.extend([os.path.join(self.checkout_path, f) for f in files])
315
szager@chromium.org8a139702014-06-20 15:55:01 +0000316 def _DisableHooks(self):
317 hook_dir = os.path.join(self.checkout_path, '.git', 'hooks')
318 if not os.path.isdir(hook_dir):
319 return
320 for f in os.listdir(hook_dir):
321 if not f.endswith('.sample') and not f.endswith('.disabled'):
primiano@chromium.org41265562015-04-08 09:14:46 +0000322 disabled_hook_path = os.path.join(hook_dir, f + '.disabled')
323 if os.path.exists(disabled_hook_path):
324 os.remove(disabled_hook_path)
325 os.rename(os.path.join(hook_dir, f), disabled_hook_path)
szager@chromium.org8a139702014-06-20 15:55:01 +0000326
iannucci@chromium.org30a07982016-04-07 21:35:19 +0000327 def _maybe_break_locks(self, options):
328 """This removes all .lock files from this repo's .git directory, if the
329 user passed the --break_repo_locks command line flag.
330
331 In particular, this will cleanup index.lock files, as well as ref lock
332 files.
333 """
334 if options.break_repo_locks:
335 git_dir = os.path.join(self.checkout_path, '.git')
336 for path, _, filenames in os.walk(git_dir):
337 for filename in filenames:
338 if filename.endswith('.lock'):
339 to_break = os.path.join(path, filename)
340 self.Print('breaking lock: %s' % (to_break,))
341 try:
342 os.remove(to_break)
343 except OSError as ex:
344 self.Print('FAILED to break lock: %s: %s' % (to_break, ex))
345 raise
346
Edward Lemur6a4e31b2018-08-10 19:59:02 +0000347 # TODO(ehmaldonado): Remove after bot_update is modified to pass the patch's
348 # branch.
349 def _GetTargetBranchForCommit(self, commit):
Edward Lemurca7d8812018-07-24 17:42:45 +0000350 """Get the remote branch a commit is part of."""
Edward Lemur6a4e31b2018-08-10 19:59:02 +0000351 _WELL_KNOWN_BRANCHES = [
352 'refs/remotes/origin/master',
353 'refs/remotes/origin/infra/config',
354 'refs/remotes/origin/lkgr',
355 ]
356 for branch in _WELL_KNOWN_BRANCHES:
357 if scm.GIT.IsAncestor(self.checkout_path, commit, branch):
358 return branch
Edward Lemurca7d8812018-07-24 17:42:45 +0000359 remote_refs = self._Capture(
Edward Lemur6a4e31b2018-08-10 19:59:02 +0000360 ['for-each-ref', 'refs/remotes/%s' % self.remote,
Edward Lemurca7d8812018-07-24 17:42:45 +0000361 '--format=%(refname)']).splitlines()
Edward Lemur6a4e31b2018-08-10 19:59:02 +0000362 for ref in sorted(remote_refs, reverse=True):
Edward Lemurca7d8812018-07-24 17:42:45 +0000363 if scm.GIT.IsAncestor(self.checkout_path, commit, ref):
364 return ref
365 self.Print('Failed to find a remote ref that contains %s. '
366 'Candidate refs were %s.' % (commit, remote_refs))
367 # Fallback to the commit we got.
368 # This means that apply_path_ref will try to find the merge-base between the
369 # patch and the commit (which is most likely the commit) and cherry-pick
370 # everything in between.
371 return commit
372
Edward Lemur6a4e31b2018-08-10 19:59:02 +0000373 def apply_patch_ref(self, patch_repo, patch_ref, target_branch, options,
374 file_list):
375 """Apply a patch on top of the revision we're synced at.
376
377 The patch ref is given by |patch_repo|@|patch_ref|, and the current revision
378 is |base_rev|.
379 We also need the |target_branch| that the patch was uploaded against. We use
380 it to find a merge base between |patch_rev| and |base_rev|, so we can find
381 what commits constitute the patch:
382
383 Graphically, it looks like this:
384
385 ... -> merge_base -> [possibly already landed commits] -> target_branch
386 \
387 -> [possibly not yet landed dependent CLs] -> patch_rev
388
389 Next, we apply the commits |merge_base..patch_rev| on top of whatever is
390 currently checked out, denoted |base_rev|. Typically, it'd be a revision
391 from |target_branch|, but this is not required.
392
393 Graphically, we cherry pick |merge_base..patch_rev| on top of |base_rev|:
394
395 ... -> base_rev -> [possibly not yet landed dependent CLs] -> patch_rev
396
397 After application, if |options.reset_patch_ref| is specified, we soft reset
398 the just cherry-picked changes, keeping them in git index only.
399
400 Args:
401 patch_repo: The patch origin. e.g. 'https://foo.googlesource.com/bar'
402 patch_ref: The ref to the patch. e.g. 'refs/changes/1234/34/1'.
403 target_branch: The branch the patch was uploaded against.
404 e.g. 'refs/heads/master' or 'refs/heads/infra/config'.
405 options: The options passed to gclient.
406 file_list: A list where modified files will be appended.
407 """
408
Edward Lemurca7d8812018-07-24 17:42:45 +0000409 # Abort any cherry-picks in progress.
410 try:
411 self._Capture(['cherry-pick', '--abort'])
412 except subprocess2.CalledProcessError:
413 pass
414
Edward Lesmesc621b212018-03-21 20:26:56 -0400415 base_rev = self._Capture(['rev-parse', 'HEAD'])
Edward Lemur6a4e31b2018-08-10 19:59:02 +0000416 target_branch = target_branch or self._GetTargetBranchForCommit(base_rev)
Edward Lesmesc621b212018-03-21 20:26:56 -0400417 self.Print('===Applying patch ref===')
Edward Lemur6a4e31b2018-08-10 19:59:02 +0000418 self.Print('Patch ref is %r @ %r. Target branch for patch is %r. '
419 'Current HEAD is %r. Current dir is %r' % (
420 patch_repo, patch_ref, target_branch, base_rev,
421 self.checkout_path))
Edward Lesmesc621b212018-03-21 20:26:56 -0400422 self._Capture(['reset', '--hard'])
423 self._Capture(['fetch', patch_repo, patch_ref])
Edward Lemurca7d8812018-07-24 17:42:45 +0000424 patch_rev = self._Capture(['rev-parse', 'FETCH_HEAD'])
Edward Lesmesc621b212018-03-21 20:26:56 -0400425
Edward Lemurca7d8812018-07-24 17:42:45 +0000426 try:
427 if not options.rebase_patch_ref:
428 self._Capture(['checkout', patch_rev])
429 else:
430 # Find the merge-base between the branch_rev and patch_rev to find out
431 # the changes we need to cherry-pick on top of base_rev.
Edward Lemur6a4e31b2018-08-10 19:59:02 +0000432 merge_base = self._Capture(['merge-base', target_branch, patch_rev])
Edward Lemurca7d8812018-07-24 17:42:45 +0000433 self.Print('Merge base of %s and %s is %s' % (
Edward Lemur6a4e31b2018-08-10 19:59:02 +0000434 target_branch, patch_rev, merge_base))
Edward Lemurca7d8812018-07-24 17:42:45 +0000435 if merge_base == patch_rev:
436 # If the merge-base is patch_rev, it means patch_rev is already part
437 # of the history, so just check it out.
438 self._Capture(['checkout', patch_rev])
439 else:
440 # If a change was uploaded on top of another change, which has already
441 # landed, one of the commits in the cherry-pick range will be
442 # redundant, since it has already landed and its changes incorporated
443 # in the tree.
444 # We pass '--keep-redundant-commits' to ignore those changes.
445 self._Capture(['cherry-pick', merge_base + '..' + patch_rev,
446 '--keep-redundant-commits'])
447
448 if file_list is not None:
449 file_list.extend(self._GetDiffFilenames(base_rev))
450
451 except subprocess2.CalledProcessError as e:
Edward Lemur6a4e31b2018-08-10 19:59:02 +0000452 self.Print('Failed to apply patch.')
453 self.Print('Patch ref is %r @ %r. Target branch for patch is %r. '
454 'Current HEAD is %r. Current dir is %r' % (
455 patch_repo, patch_ref, target_branch, base_rev,
456 self.checkout_path))
Edward Lemurca7d8812018-07-24 17:42:45 +0000457 self.Print('git returned non-zero exit status %s:\n%s' % (
458 e.returncode, e.stderr))
459 # Print the current status so that developers know what changes caused the
460 # patch failure, since git cherry-pick doesn't show that information.
461 self.Print(self._Capture(['status']))
John Budorick2c984a02018-07-18 23:24:13 +0000462 try:
Edward Lemurca7d8812018-07-24 17:42:45 +0000463 self._Capture(['cherry-pick', '--abort'])
464 except subprocess2.CalledProcessError:
465 pass
466 raise
467
Edward Lesmesc621b212018-03-21 20:26:56 -0400468 if options.reset_patch_ref:
469 self._Capture(['reset', '--soft', base_rev])
470
msb@chromium.orge28e4982009-09-25 20:51:45 +0000471 def update(self, options, args, file_list):
472 """Runs git to update or transparently checkout the working copy.
473
474 All updated files will be appended to file_list.
475
476 Raises:
477 Error: if can't get URL for relative path.
478 """
msb@chromium.orge28e4982009-09-25 20:51:45 +0000479 if args:
480 raise gclient_utils.Error("Unsupported argument(s): %s" % ",".join(args))
481
nasser@codeaurora.orgece406f2010-02-23 17:29:15 +0000482 self._CheckMinVersion("1.6.6")
msb@chromium.org923a0372009-12-11 20:42:43 +0000483
John Budorick882c91e2018-07-12 22:11:41 +0000484 # If a dependency is not pinned, track the default remote branch.
485 default_rev = 'refs/remotes/%s/master' % self.remote
nasser@codeaurora.org7080e942010-03-15 15:06:16 +0000486 url, deps_revision = gclient_utils.SplitUrlRevision(self.url)
nasser@codeaurora.org7080e942010-03-15 15:06:16 +0000487 revision = deps_revision
cmp@chromium.orgeb2756d2011-09-20 20:17:51 +0000488 managed = True
msb@chromium.orge28e4982009-09-25 20:51:45 +0000489 if options.revision:
msb@chromium.orgac915bb2009-11-13 17:03:01 +0000490 # Override the revision number.
491 revision = str(options.revision)
cmp@chromium.orgeb2756d2011-09-20 20:17:51 +0000492 if revision == 'unmanaged':
romain.pokrzywka@gmail.com483a0ba2014-05-30 00:06:07 +0000493 # Check again for a revision in case an initial ref was specified
494 # in the url, for example bla.git@refs/heads/custombranch
495 revision = deps_revision
cmp@chromium.orgeb2756d2011-09-20 20:17:51 +0000496 managed = False
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000497 if not revision:
498 revision = default_rev
msb@chromium.orge28e4982009-09-25 20:51:45 +0000499
szager@chromium.org8a139702014-06-20 15:55:01 +0000500 if managed:
501 self._DisableHooks()
502
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000503 printed_path = False
504 verbose = []
msb@chromium.orgb1a22bf2009-11-07 02:33:50 +0000505 if options.verbose:
agable83faed02016-10-24 14:37:10 -0700506 self.Print('_____ %s at %s' % (self.relpath, revision), timestamp=False)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000507 verbose = ['--verbose']
508 printed_path = True
509
John Budorick882c91e2018-07-12 22:11:41 +0000510 remote_ref = scm.GIT.RefToRemoteRef(revision, self.remote)
511 if remote_ref:
512 # Rewrite remote refs to their local equivalents.
513 revision = ''.join(remote_ref)
514 rev_type = "branch"
515 elif revision.startswith('refs/'):
516 # Local branch? We probably don't want to support, since DEPS should
517 # always specify branches as they are in the upstream repo.
518 rev_type = "branch"
519 else:
520 # hash is also a tag, only make a distinction at checkout
521 rev_type = "hash"
smut@google.comd33eab32014-07-07 19:35:18 +0000522
John Budorick882c91e2018-07-12 22:11:41 +0000523 mirror = self._GetMirror(url, options)
szager@chromium.orgb0a13a22014-06-18 00:52:25 +0000524 if mirror:
525 url = mirror.mirror_path
526
primiano@chromium.org1c127382015-02-17 11:15:40 +0000527 # If we are going to introduce a new project, there is a possibility that
528 # we are syncing back to a state where the project was originally a
529 # sub-project rolled by DEPS (realistic case: crossing the Blink merge point
530 # syncing backwards, when Blink was a DEPS entry and not part of src.git).
531 # In such case, we might have a backup of the former .git folder, which can
532 # be used to avoid re-fetching the entire repo again (useful for bisects).
533 backup_dir = self.GetGitBackupDirPath()
534 target_dir = os.path.join(self.checkout_path, '.git')
535 if os.path.exists(backup_dir) and not os.path.exists(target_dir):
536 gclient_utils.safe_makedirs(self.checkout_path)
537 os.rename(backup_dir, target_dir)
538 # Reset to a clean state
Robert Iannuccic41d8b92017-02-16 17:07:37 -0800539 self._Scrub('HEAD', options)
primiano@chromium.org1c127382015-02-17 11:15:40 +0000540
szager@chromium.org6c2b49d2014-02-26 23:57:38 +0000541 if (not os.path.exists(self.checkout_path) or
542 (os.path.isdir(self.checkout_path) and
543 not os.path.exists(os.path.join(self.checkout_path, '.git')))):
szager@chromium.orgb0a13a22014-06-18 00:52:25 +0000544 if mirror:
John Budorick882c91e2018-07-12 22:11:41 +0000545 self._UpdateMirrorIfNotContains(mirror, options, rev_type, revision)
borenet@google.com90fe58b2014-05-01 18:22:00 +0000546 try:
John Budorick882c91e2018-07-12 22:11:41 +0000547 self._Clone(revision, url, options)
borenet@google.com90fe58b2014-05-01 18:22:00 +0000548 except subprocess2.CalledProcessError:
549 self._DeleteOrMove(options.force)
John Budorick882c91e2018-07-12 22:11:41 +0000550 self._Clone(revision, url, options)
iannucci@chromium.org396e1a62013-07-03 19:41:04 +0000551 if file_list is not None:
Aaron Gable7817f022017-12-12 09:43:17 -0800552 files = self._Capture(
553 ['-c', 'core.quotePath=false', 'ls-files']).splitlines()
iannucci@chromium.org396e1a62013-07-03 19:41:04 +0000554 file_list.extend([os.path.join(self.checkout_path, f) for f in files])
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000555 if not verbose:
556 # Make the output a little prettier. It's nice to have some whitespace
557 # between projects when cloning.
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000558 self.Print('')
iannucci@chromium.org2702bcd2013-09-24 19:10:07 +0000559 return self._Capture(['rev-parse', '--verify', 'HEAD'])
msb@chromium.orge28e4982009-09-25 20:51:45 +0000560
szager@chromium.org3dc5cb72014-06-17 15:06:05 +0000561 if not managed:
Edward Lemur579c9862018-07-13 23:17:51 +0000562 self._SetFetchConfig(options)
szager@chromium.org3dc5cb72014-06-17 15:06:05 +0000563 self.Print('________ unmanaged solution; skipping %s' % self.relpath)
564 return self._Capture(['rev-parse', '--verify', 'HEAD'])
565
iannucci@chromium.org30a07982016-04-07 21:35:19 +0000566 self._maybe_break_locks(options)
567
szager@chromium.orgb0a13a22014-06-18 00:52:25 +0000568 if mirror:
John Budorick882c91e2018-07-12 22:11:41 +0000569 self._UpdateMirrorIfNotContains(mirror, options, rev_type, revision)
szager@chromium.orgb0a13a22014-06-18 00:52:25 +0000570
szager@chromium.org6c2b49d2014-02-26 23:57:38 +0000571 # See if the url has changed (the unittests use git://foo for the url, let
572 # that through).
573 current_url = self._Capture(['config', 'remote.%s.url' % self.remote])
574 return_early = False
575 # TODO(maruel): Delete url != 'git://foo' since it's just to make the
576 # unit test pass. (and update the comment above)
577 # Skip url auto-correction if remote.origin.gclient-auto-fix-url is set.
578 # This allows devs to use experimental repos which have a different url
579 # but whose branch(s) are the same as official repos.
borenet@google.comb09097a2014-04-09 19:09:08 +0000580 if (current_url.rstrip('/') != url.rstrip('/') and
szager@chromium.org6c2b49d2014-02-26 23:57:38 +0000581 url != 'git://foo' and
582 subprocess2.capture(
583 ['git', 'config', 'remote.%s.gclient-auto-fix-url' % self.remote],
584 cwd=self.checkout_path).strip() != 'False'):
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000585 self.Print('_____ switching %s to a new upstream' % self.relpath)
iannucci@chromium.org78514212014-08-20 23:08:00 +0000586 if not (options.force or options.reset):
587 # Make sure it's clean
agable83faed02016-10-24 14:37:10 -0700588 self._CheckClean(revision)
szager@chromium.org6c2b49d2014-02-26 23:57:38 +0000589 # Switch over to the new upstream
590 self._Run(['remote', 'set-url', self.remote, url], options)
szager@chromium.org8dd35462015-06-08 22:56:05 +0000591 if mirror:
592 with open(os.path.join(
593 self.checkout_path, '.git', 'objects', 'info', 'alternates'),
594 'w') as fh:
595 fh.write(os.path.join(url, 'objects'))
John Budorick882c91e2018-07-12 22:11:41 +0000596 self._EnsureValidHeadObjectOrCheckout(revision, options, url)
597 self._FetchAndReset(revision, file_list, options)
tandrii@chromium.orgc438c142015-08-24 22:55:55 +0000598
szager@chromium.org6c2b49d2014-02-26 23:57:38 +0000599 return_early = True
tandrii@chromium.orgc438c142015-08-24 22:55:55 +0000600 else:
John Budorick882c91e2018-07-12 22:11:41 +0000601 self._EnsureValidHeadObjectOrCheckout(revision, options, url)
szager@chromium.org6c2b49d2014-02-26 23:57:38 +0000602
mmoss@chromium.org50fd47f2014-02-13 01:03:19 +0000603 if return_early:
604 return self._Capture(['rev-parse', '--verify', 'HEAD'])
605
msb@chromium.org5bde4852009-12-14 16:47:12 +0000606 cur_branch = self._GetCurrentBranch()
607
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000608 # Cases:
msb@chromium.org786fb682010-06-02 15:16:23 +0000609 # 0) HEAD is detached. Probably from our initial clone.
610 # - make sure HEAD is contained by a named ref, then update.
611 # Cases 1-4. HEAD is a branch.
agable41e3a6c2016-10-20 11:36:56 -0700612 # 1) current branch is not tracking a remote branch
msb@chromium.org786fb682010-06-02 15:16:23 +0000613 # - try to rebase onto the new hash or branch
614 # 2) current branch is tracking a remote branch with local committed
615 # changes, but the DEPS file switched to point to a hash
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000616 # - rebase those changes on top of the hash
mmoss@chromium.org6e7202b2014-09-09 18:23:39 +0000617 # 3) current branch is tracking a remote branch w/or w/out changes, and
618 # no DEPS switch
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000619 # - see if we can FF, if not, prompt the user for rebase, merge, or stop
mmoss@chromium.org6e7202b2014-09-09 18:23:39 +0000620 # 4) current branch is tracking a remote branch, but DEPS switches to a
621 # different remote branch, and
622 # a) current branch has no local changes, and --force:
623 # - checkout new branch
624 # b) current branch has local changes, and --force and --reset:
625 # - checkout new branch
626 # c) otherwise exit
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000627
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000628 # GetUpstreamBranch returns something like 'refs/remotes/origin/master' for
629 # a tracking branch
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000630 # or 'master' if not a tracking branch (it's based on a specific rev/hash)
631 # or it returns None if it couldn't find an upstream
msb@chromium.org786fb682010-06-02 15:16:23 +0000632 if cur_branch is None:
633 upstream_branch = None
634 current_type = "detached"
635 logging.debug("Detached HEAD")
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000636 else:
msb@chromium.org786fb682010-06-02 15:16:23 +0000637 upstream_branch = scm.GIT.GetUpstreamBranch(self.checkout_path)
638 if not upstream_branch or not upstream_branch.startswith('refs/remotes'):
639 current_type = "hash"
640 logging.debug("Current branch is not tracking an upstream (remote)"
641 " branch.")
642 elif upstream_branch.startswith('refs/remotes'):
643 current_type = "branch"
644 else:
645 raise gclient_utils.Error('Invalid Upstream: %s' % upstream_branch)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000646
Edward Lemur579c9862018-07-13 23:17:51 +0000647 self._SetFetchConfig(options)
648 self._Fetch(options, prune=options.force)
649
John Budorick882c91e2018-07-12 22:11:41 +0000650 if not scm.GIT.IsValidRevision(self.checkout_path, revision, sha_only=True):
bauerb@chromium.orgcbd20a42012-06-27 13:49:27 +0000651 # Update the remotes first so we have all the refs.
ilevy@chromium.orga41249c2013-07-03 00:09:12 +0000652 remote_output = scm.GIT.Capture(['remote'] + verbose + ['update'],
bauerb@chromium.orgcbd20a42012-06-27 13:49:27 +0000653 cwd=self.checkout_path)
bauerb@chromium.orgcbd20a42012-06-27 13:49:27 +0000654 if verbose:
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000655 self.Print(remote_output)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000656
John Budorick882c91e2018-07-12 22:11:41 +0000657 revision = self._AutoFetchRef(options, revision)
Paweł Hajdan, Jr63b8c2a2017-09-05 17:59:08 +0200658
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000659 # This is a big hammer, debatable if it should even be here...
davemoore@chromium.org793796d2010-02-19 17:27:41 +0000660 if options.force or options.reset:
iannucci@chromium.orgd4fffee2013-06-28 00:35:26 +0000661 target = 'HEAD'
662 if options.upstream and upstream_branch:
663 target = upstream_branch
Robert Iannuccic41d8b92017-02-16 17:07:37 -0800664 self._Scrub(target, options)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000665
msb@chromium.org786fb682010-06-02 15:16:23 +0000666 if current_type == 'detached':
667 # case 0
Robert Iannuccic41d8b92017-02-16 17:07:37 -0800668 # We just did a Scrub, this is as clean as it's going to get. In
669 # particular if HEAD is a commit that contains two versions of the same
670 # file on a case-insensitive filesystem (e.g. 'a' and 'A'), there's no way
671 # to actually "Clean" the checkout; that commit is uncheckoutable on this
672 # system. The best we can do is carry forward to the checkout step.
673 if not (options.force or options.reset):
John Budorick882c91e2018-07-12 22:11:41 +0000674 self._CheckClean(revision)
675 self._CheckDetachedHead(revision, options)
smut@google.comd33eab32014-07-07 19:35:18 +0000676 if self._Capture(['rev-list', '-n', '1', 'HEAD']) == revision:
szager@chromium.org848fd492014-04-09 19:06:44 +0000677 self.Print('Up-to-date; skipping checkout.')
678 else:
vadimsh@chromium.org2b7d3ed2014-06-20 18:15:37 +0000679 # 'git checkout' may need to overwrite existing untracked files. Allow
680 # it only when nuclear options are enabled.
dnj@chromium.orgbb424c02014-06-23 22:42:51 +0000681 self._Checkout(
682 options,
John Budorick882c91e2018-07-12 22:11:41 +0000683 revision,
smut@google.com34b4e982016-05-16 19:06:07 +0000684 force=(options.force and options.delete_unversioned_trees),
dnj@chromium.orgbb424c02014-06-23 22:42:51 +0000685 quiet=True,
686 )
msb@chromium.org786fb682010-06-02 15:16:23 +0000687 if not printed_path:
John Budorick882c91e2018-07-12 22:11:41 +0000688 self.Print('_____ %s at %s' % (self.relpath, revision), timestamp=False)
msb@chromium.org786fb682010-06-02 15:16:23 +0000689 elif current_type == 'hash':
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000690 # case 1
agable41e3a6c2016-10-20 11:36:56 -0700691 # Can't find a merge-base since we don't know our upstream. That makes
692 # this command VERY likely to produce a rebase failure. For now we
693 # assume origin is our upstream since that's what the old behavior was.
John Budorick882c91e2018-07-12 22:11:41 +0000694 upstream_branch = self.remote
695 if options.revision or deps_revision:
696 upstream_branch = revision
agable1a8439a2016-10-24 16:36:14 -0700697 self._AttemptRebase(upstream_branch, file_list, options,
agable41e3a6c2016-10-20 11:36:56 -0700698 printed_path=printed_path, merge=options.merge)
699 printed_path = True
John Budorick882c91e2018-07-12 22:11:41 +0000700 elif rev_type == 'hash':
701 # case 2
702 self._AttemptRebase(upstream_branch, file_list, options,
703 newbase=revision, printed_path=printed_path,
704 merge=options.merge)
705 printed_path = True
706 elif remote_ref and ''.join(remote_ref) != upstream_branch:
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000707 # case 4
John Budorick882c91e2018-07-12 22:11:41 +0000708 new_base = ''.join(remote_ref)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000709 if not printed_path:
John Budorick882c91e2018-07-12 22:11:41 +0000710 self.Print('_____ %s at %s' % (self.relpath, revision), timestamp=False)
mmoss@chromium.org6e7202b2014-09-09 18:23:39 +0000711 switch_error = ("Could not switch upstream branch from %s to %s\n"
John Budorick882c91e2018-07-12 22:11:41 +0000712 % (upstream_branch, new_base) +
mmoss@chromium.org6e7202b2014-09-09 18:23:39 +0000713 "Please use --force or merge or rebase manually:\n" +
John Budorick882c91e2018-07-12 22:11:41 +0000714 "cd %s; git rebase %s\n" % (self.checkout_path, new_base) +
715 "OR git checkout -b <some new branch> %s" % new_base)
mmoss@chromium.org6e7202b2014-09-09 18:23:39 +0000716 force_switch = False
717 if options.force:
718 try:
John Budorick882c91e2018-07-12 22:11:41 +0000719 self._CheckClean(revision)
mmoss@chromium.org6e7202b2014-09-09 18:23:39 +0000720 # case 4a
721 force_switch = True
722 except gclient_utils.Error as e:
723 if options.reset:
724 # case 4b
725 force_switch = True
726 else:
727 switch_error = '%s\n%s' % (e.message, switch_error)
728 if force_switch:
729 self.Print("Switching upstream branch from %s to %s" %
John Budorick882c91e2018-07-12 22:11:41 +0000730 (upstream_branch, new_base))
731 switch_branch = 'gclient_' + remote_ref[1]
732 self._Capture(['branch', '-f', switch_branch, new_base])
mmoss@chromium.org6e7202b2014-09-09 18:23:39 +0000733 self._Checkout(options, switch_branch, force=True, quiet=True)
734 else:
735 # case 4c
736 raise gclient_utils.Error(switch_error)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000737 else:
738 # case 3 - the default case
Aaron Gablef4068aa2017-12-12 15:14:09 -0800739 rebase_files = self._GetDiffFilenames(upstream_branch)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000740 if verbose:
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000741 self.Print('Trying fast-forward merge to branch : %s' % upstream_branch)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000742 try:
bauerb@chromium.org2aad1b22011-07-22 12:00:41 +0000743 merge_args = ['merge']
bauerb@chromium.org30c46d62014-01-23 12:11:56 +0000744 if options.merge:
745 merge_args.append('--ff')
746 else:
bauerb@chromium.org2aad1b22011-07-22 12:00:41 +0000747 merge_args.append('--ff-only')
748 merge_args.append(upstream_branch)
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000749 merge_output = self._Capture(merge_args)
bratell@opera.com18fa4542013-05-21 13:30:46 +0000750 except subprocess2.CalledProcessError as e:
agable1a8439a2016-10-24 16:36:14 -0700751 rebase_files = []
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000752 if re.match('fatal: Not possible to fast-forward, aborting.', e.stderr):
753 if not printed_path:
John Budorick882c91e2018-07-12 22:11:41 +0000754 self.Print('_____ %s at %s' % (self.relpath, revision),
agable83faed02016-10-24 14:37:10 -0700755 timestamp=False)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000756 printed_path = True
757 while True:
dnj@chromium.org5b23e872015-02-20 21:25:57 +0000758 if not options.auto_rebase:
759 try:
760 action = self._AskForData(
761 'Cannot %s, attempt to rebase? '
762 '(y)es / (q)uit / (s)kip : ' %
763 ('merge' if options.merge else 'fast-forward merge'),
764 options)
765 except ValueError:
766 raise gclient_utils.Error('Invalid Character')
767 if options.auto_rebase or re.match(r'yes|y', action, re.I):
agable1a8439a2016-10-24 16:36:14 -0700768 self._AttemptRebase(upstream_branch, rebase_files, options,
bauerb@chromium.org30c46d62014-01-23 12:11:56 +0000769 printed_path=printed_path, merge=False)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000770 printed_path = True
771 break
772 elif re.match(r'quit|q', action, re.I):
773 raise gclient_utils.Error("Can't fast-forward, please merge or "
774 "rebase manually.\n"
775 "cd %s && git " % self.checkout_path
776 + "rebase %s" % upstream_branch)
777 elif re.match(r'skip|s', action, re.I):
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000778 self.Print('Skipping %s' % self.relpath)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000779 return
780 else:
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000781 self.Print('Input not recognized')
smut@google.com27c9c8a2014-09-11 19:57:55 +0000782 elif re.match("error: Your local changes to '.*' would be "
783 "overwritten by merge. Aborting.\nPlease, commit your "
784 "changes or stash them before you can merge.\n",
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000785 e.stderr):
786 if not printed_path:
John Budorick882c91e2018-07-12 22:11:41 +0000787 self.Print('_____ %s at %s' % (self.relpath, revision),
agable83faed02016-10-24 14:37:10 -0700788 timestamp=False)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000789 printed_path = True
790 raise gclient_utils.Error(e.stderr)
791 else:
792 # Some other problem happened with the merge
793 logging.error("Error during fast-forward merge in %s!" % self.relpath)
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000794 self.Print(e.stderr)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000795 raise
796 else:
797 # Fast-forward merge was successful
798 if not re.match('Already up-to-date.', merge_output) or verbose:
799 if not printed_path:
agable83faed02016-10-24 14:37:10 -0700800 self.Print('_____ %s at %s' % (self.relpath, revision),
801 timestamp=False)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000802 printed_path = True
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000803 self.Print(merge_output.strip())
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000804 if not verbose:
805 # Make the output a little prettier. It's nice to have some
806 # whitespace between projects when syncing.
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000807 self.Print('')
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000808
agablec3937b92016-10-25 10:13:03 -0700809 if file_list is not None:
810 file_list.extend(
811 [os.path.join(self.checkout_path, f) for f in rebase_files])
msb@chromium.org5bde4852009-12-14 16:47:12 +0000812
813 # If the rebase generated a conflict, abort and ask user to fix
msb@chromium.org786fb682010-06-02 15:16:23 +0000814 if self._IsRebasing():
agable83faed02016-10-24 14:37:10 -0700815 raise gclient_utils.Error('\n____ %s at %s\n'
msb@chromium.org5bde4852009-12-14 16:47:12 +0000816 '\nConflict while rebasing this branch.\n'
817 'Fix the conflict and run gclient again.\n'
818 'See man git-rebase for details.\n'
agable83faed02016-10-24 14:37:10 -0700819 % (self.relpath, revision))
msb@chromium.org5bde4852009-12-14 16:47:12 +0000820
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000821 if verbose:
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000822 self.Print('Checked out revision %s' % self.revinfo(options, (), None),
823 timestamp=False)
msb@chromium.orge28e4982009-09-25 20:51:45 +0000824
steveblock@chromium.org98e69452012-02-16 16:36:43 +0000825 # If --reset and --delete_unversioned_trees are specified, remove any
826 # untracked directories.
827 if options.reset and options.delete_unversioned_trees:
828 # GIT.CaptureStatus() uses 'dit diff' to compare to a specific SHA1 (the
829 # merge-base by default), so doesn't include untracked files. So we use
830 # 'git ls-files --directory --others --exclude-standard' here directly.
831 paths = scm.GIT.Capture(
Aaron Gable7817f022017-12-12 09:43:17 -0800832 ['-c', 'core.quotePath=false', 'ls-files',
833 '--directory', '--others', '--exclude-standard'],
steveblock@chromium.org98e69452012-02-16 16:36:43 +0000834 self.checkout_path)
835 for path in (p for p in paths.splitlines() if p.endswith('/')):
836 full_path = os.path.join(self.checkout_path, path)
837 if not os.path.islink(full_path):
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000838 self.Print('_____ removing unversioned directory %s' % path)
digit@chromium.orgdc112ac2013-04-24 13:00:19 +0000839 gclient_utils.rmtree(full_path)
steveblock@chromium.org98e69452012-02-16 16:36:43 +0000840
iannucci@chromium.org2702bcd2013-09-24 19:10:07 +0000841 return self._Capture(['rev-parse', '--verify', 'HEAD'])
842
iannucci@chromium.org396e1a62013-07-03 19:41:04 +0000843 def revert(self, options, _args, file_list):
msb@chromium.orge28e4982009-09-25 20:51:45 +0000844 """Reverts local modifications.
845
846 All reverted files will be appended to file_list.
847 """
maruel@chromium.org8469bf92010-09-03 19:03:15 +0000848 if not os.path.isdir(self.checkout_path):
msb@chromium.org260c6532009-10-28 03:22:35 +0000849 # revert won't work if the directory doesn't exist. It needs to
850 # checkout instead.
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000851 self.Print('_____ %s is missing, synching instead' % self.relpath)
msb@chromium.org260c6532009-10-28 03:22:35 +0000852 # Don't reuse the args.
853 return self.update(options, [], file_list)
nasser@codeaurora.orgb2b46312010-04-30 20:58:03 +0000854
855 default_rev = "refs/heads/master"
iannucci@chromium.orgd4fffee2013-06-28 00:35:26 +0000856 if options.upstream:
857 if self._GetCurrentBranch():
858 upstream_branch = scm.GIT.GetUpstreamBranch(self.checkout_path)
859 default_rev = upstream_branch or default_rev
maruel@chromium.org6e29d572010-06-04 17:32:20 +0000860 _, deps_revision = gclient_utils.SplitUrlRevision(self.url)
nasser@codeaurora.orgb2b46312010-04-30 20:58:03 +0000861 if not deps_revision:
862 deps_revision = default_rev
smut@google.comd33eab32014-07-07 19:35:18 +0000863 if deps_revision.startswith('refs/heads/'):
864 deps_revision = deps_revision.replace('refs/heads/', self.remote + '/')
smutae7ea312016-07-18 11:59:41 -0700865 try:
866 deps_revision = self.GetUsableRev(deps_revision, options)
867 except NoUsableRevError as e:
868 # If the DEPS entry's url and hash changed, try to update the origin.
869 # See also http://crbug.com/520067.
870 logging.warn(
871 'Couldn\'t find usable revision, will retrying to update instead: %s',
872 e.message)
873 return self.update(options, [], file_list)
nasser@codeaurora.orgb2b46312010-04-30 20:58:03 +0000874
iannucci@chromium.org396e1a62013-07-03 19:41:04 +0000875 if file_list is not None:
Aaron Gablef4068aa2017-12-12 15:14:09 -0800876 files = self._GetDiffFilenames(deps_revision)
iannucci@chromium.org396e1a62013-07-03 19:41:04 +0000877
Robert Iannuccic41d8b92017-02-16 17:07:37 -0800878 self._Scrub(deps_revision, options)
lliabraa@chromium.orgade83db2012-09-27 14:06:49 +0000879 self._Run(['clean', '-f', '-d'], options)
msb@chromium.orge28e4982009-09-25 20:51:45 +0000880
iannucci@chromium.org396e1a62013-07-03 19:41:04 +0000881 if file_list is not None:
882 file_list.extend([os.path.join(self.checkout_path, f) for f in files])
883
884 def revinfo(self, _options, _args, _file_list):
maruel@chromium.org6cafa132010-09-07 14:17:26 +0000885 """Returns revision"""
886 return self._Capture(['rev-parse', 'HEAD'])
msb@chromium.org0f282062009-11-06 20:14:02 +0000887
msb@chromium.orge28e4982009-09-25 20:51:45 +0000888 def runhooks(self, options, args, file_list):
889 self.status(options, args, file_list)
890
iannucci@chromium.org396e1a62013-07-03 19:41:04 +0000891 def status(self, options, _args, file_list):
msb@chromium.orge28e4982009-09-25 20:51:45 +0000892 """Display status information."""
893 if not os.path.isdir(self.checkout_path):
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000894 self.Print('________ couldn\'t run status in %s:\n'
895 'The directory does not exist.' % self.checkout_path)
msb@chromium.orge28e4982009-09-25 20:51:45 +0000896 else:
raphael.kubo.da.costa05c83592016-08-04 08:32:41 -0700897 try:
898 merge_base = [self._Capture(['merge-base', 'HEAD', self.remote])]
899 except subprocess2.CalledProcessError:
900 merge_base = []
Aaron Gablef4068aa2017-12-12 15:14:09 -0800901 self._Run(
902 ['-c', 'core.quotePath=false', 'diff', '--name-status'] + merge_base,
903 options, stdout=self.out_fh, always=options.verbose)
iannucci@chromium.org396e1a62013-07-03 19:41:04 +0000904 if file_list is not None:
Aaron Gablef4068aa2017-12-12 15:14:09 -0800905 files = self._GetDiffFilenames(merge_base[0] if merge_base else None)
iannucci@chromium.org396e1a62013-07-03 19:41:04 +0000906 file_list.extend([os.path.join(self.checkout_path, f) for f in files])
msb@chromium.orge28e4982009-09-25 20:51:45 +0000907
smutae7ea312016-07-18 11:59:41 -0700908 def GetUsableRev(self, rev, options):
agable41e3a6c2016-10-20 11:36:56 -0700909 """Finds a useful revision for this repository."""
smutae7ea312016-07-18 11:59:41 -0700910 sha1 = None
911 if not os.path.isdir(self.checkout_path):
912 raise NoUsableRevError(
agablea98a6cd2016-11-15 14:30:10 -0800913 'This is not a git repo, so we cannot get a usable rev.')
agable41e3a6c2016-10-20 11:36:56 -0700914
915 if scm.GIT.IsValidRevision(cwd=self.checkout_path, rev=rev):
916 sha1 = rev
smutae7ea312016-07-18 11:59:41 -0700917 else:
agable41e3a6c2016-10-20 11:36:56 -0700918 # May exist in origin, but we don't have it yet, so fetch and look
919 # again.
920 self._Fetch(options)
smutae7ea312016-07-18 11:59:41 -0700921 if scm.GIT.IsValidRevision(cwd=self.checkout_path, rev=rev):
922 sha1 = rev
smutae7ea312016-07-18 11:59:41 -0700923
924 if not sha1:
925 raise NoUsableRevError(
agablea98a6cd2016-11-15 14:30:10 -0800926 'Hash %s does not appear to be a valid hash in this repo.' % rev)
smutae7ea312016-07-18 11:59:41 -0700927
928 return sha1
929
primiano@chromium.org1c127382015-02-17 11:15:40 +0000930 def GetGitBackupDirPath(self):
931 """Returns the path where the .git folder for the current project can be
932 staged/restored. Use case: subproject moved from DEPS <-> outer project."""
933 return os.path.join(self._root_dir,
934 'old_' + self.relpath.replace(os.sep, '_')) + '.git'
935
John Budorick882c91e2018-07-12 22:11:41 +0000936 def _GetMirror(self, url, options):
szager@chromium.orgb0a13a22014-06-18 00:52:25 +0000937 """Get a git_cache.Mirror object for the argument url."""
Robert Iannuccia19649b2018-06-29 16:31:45 +0000938 if not self.cache_dir:
szager@chromium.orgb0a13a22014-06-18 00:52:25 +0000939 return None
hinoka@google.comb1b54572014-04-16 22:29:23 +0000940 mirror_kwargs = {
941 'print_func': self.filter,
John Budorick882c91e2018-07-12 22:11:41 +0000942 'refs': []
hinoka@google.comb1b54572014-04-16 22:29:23 +0000943 }
hinoka@google.comb1b54572014-04-16 22:29:23 +0000944 if hasattr(options, 'with_branch_heads') and options.with_branch_heads:
945 mirror_kwargs['refs'].append('refs/branch-heads/*')
szager@chromium.org8d3348f2014-08-19 22:49:16 +0000946 if hasattr(options, 'with_tags') and options.with_tags:
947 mirror_kwargs['refs'].append('refs/tags/*')
szager@chromium.orgb0a13a22014-06-18 00:52:25 +0000948 return git_cache.Mirror(url, **mirror_kwargs)
949
John Budorick882c91e2018-07-12 22:11:41 +0000950 def _UpdateMirrorIfNotContains(self, mirror, options, rev_type, revision):
Andrii Shyshkalov46a672b2017-11-24 18:04:43 -0800951 """Update a git mirror by fetching the latest commits from the remote,
952 unless mirror already contains revision whose type is sha1 hash.
953 """
John Budorick882c91e2018-07-12 22:11:41 +0000954 if rev_type == 'hash' and mirror.contains_revision(revision):
Andrii Shyshkalov46a672b2017-11-24 18:04:43 -0800955 if options.verbose:
956 self.Print('skipping mirror update, it has rev=%s already' % revision,
957 timestamp=False)
958 return
959
szager@chromium.org3ec84f62014-08-22 21:00:22 +0000960 if getattr(options, 'shallow', False):
hinoka@chromium.org46b87412014-05-15 00:42:05 +0000961 # HACK(hinoka): These repositories should be super shallow.
szager@chromium.orgb0a13a22014-06-18 00:52:25 +0000962 if 'flash' in mirror.url:
hinoka@chromium.org46b87412014-05-15 00:42:05 +0000963 depth = 10
964 else:
965 depth = 10000
966 else:
967 depth = None
e.hakkinen@samsung.come8bc1aa2015-04-08 08:00:37 +0000968 mirror.populate(verbose=options.verbose,
969 bootstrap=not getattr(options, 'no_bootstrap', False),
Vadim Shtayura08049e22017-10-11 00:14:52 +0000970 depth=depth,
971 ignore_lock=getattr(options, 'ignore_locks', False),
972 lock_timeout=getattr(options, 'lock_timeout', 0))
973 mirror.unlock()
iannucci@chromium.org53456aa2013-07-03 19:38:34 +0000974
John Budorick882c91e2018-07-12 22:11:41 +0000975 def _Clone(self, revision, url, options):
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000976 """Clone a git repository from the given URL.
977
msb@chromium.org786fb682010-06-02 15:16:23 +0000978 Once we've cloned the repo, we checkout a working branch if the specified
979 revision is a branch head. If it is a tag or a specific commit, then we
980 leave HEAD detached as it makes future updates simpler -- in this case the
981 user should first create a new branch or switch to an existing branch before
982 making changes in the repo."""
maruel@chromium.orgf5d37bf2010-09-02 00:50:34 +0000983 if not options.verbose:
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000984 # git clone doesn't seem to insert a newline properly before printing
985 # to stdout
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000986 self.Print('')
szager@chromium.orgf1d73eb2014-04-21 17:07:04 +0000987 cfg = gclient_utils.DefaultIndexPackConfig(url)
szager@chromium.org8a139702014-06-20 15:55:01 +0000988 clone_cmd = cfg + ['clone', '--no-checkout', '--progress']
iannucci@chromium.org53456aa2013-07-03 19:38:34 +0000989 if self.cache_dir:
990 clone_cmd.append('--shared')
maruel@chromium.orgf5d37bf2010-09-02 00:50:34 +0000991 if options.verbose:
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000992 clone_cmd.append('--verbose')
ilevy@chromium.org3534aa52013-07-20 01:58:08 +0000993 clone_cmd.append(url)
nsylvain@chromium.org328c3c72011-06-01 20:50:27 +0000994 # If the parent directory does not exist, Git clone on Windows will not
995 # create it, so we need to do it manually.
996 parent_dir = os.path.dirname(self.checkout_path)
ilevy@chromium.org3534aa52013-07-20 01:58:08 +0000997 gclient_utils.safe_makedirs(parent_dir)
primiano@chromium.org5439ea52014-08-06 17:18:18 +0000998
999 template_dir = None
1000 if hasattr(options, 'no_history') and options.no_history:
John Budorick882c91e2018-07-12 22:11:41 +00001001 if gclient_utils.IsGitSha(revision):
primiano@chromium.org5439ea52014-08-06 17:18:18 +00001002 # In the case of a subproject, the pinned sha is not necessarily the
1003 # head of the remote branch (so we can't just use --depth=N). Instead,
1004 # we tell git to fetch all the remote objects from SHA..HEAD by means of
1005 # a template git dir which has a 'shallow' file pointing to the sha.
1006 template_dir = tempfile.mkdtemp(
1007 prefix='_gclient_gittmp_%s' % os.path.basename(self.checkout_path),
1008 dir=parent_dir)
1009 self._Run(['init', '--bare', template_dir], options, cwd=self._root_dir)
1010 with open(os.path.join(template_dir, 'shallow'), 'w') as template_file:
1011 template_file.write(revision)
1012 clone_cmd.append('--template=' + template_dir)
1013 else:
1014 # Otherwise, we're just interested in the HEAD. Just use --depth.
1015 clone_cmd.append('--depth=1')
1016
ilevy@chromium.org3534aa52013-07-20 01:58:08 +00001017 tmp_dir = tempfile.mkdtemp(
1018 prefix='_gclient_%s_' % os.path.basename(self.checkout_path),
1019 dir=parent_dir)
1020 try:
1021 clone_cmd.append(tmp_dir)
Edward Lemur231f5ea2018-01-31 19:02:36 +01001022 if self.print_outbuf:
1023 print_stdout = True
1024 stdout = gclient_utils.WriteToStdout(self.out_fh)
1025 else:
1026 print_stdout = False
1027 stdout = self.out_fh
1028 self._Run(clone_cmd, options, cwd=self._root_dir, retry=True,
1029 print_stdout=print_stdout, stdout=stdout)
ilevy@chromium.org3534aa52013-07-20 01:58:08 +00001030 gclient_utils.safe_makedirs(self.checkout_path)
cyrille@nnamrak.orgef509e42013-09-20 13:19:08 +00001031 gclient_utils.safe_rename(os.path.join(tmp_dir, '.git'),
1032 os.path.join(self.checkout_path, '.git'))
zty@chromium.org6279e8a2014-02-13 01:45:25 +00001033 except:
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001034 traceback.print_exc(file=self.out_fh)
zty@chromium.org6279e8a2014-02-13 01:45:25 +00001035 raise
ilevy@chromium.org3534aa52013-07-20 01:58:08 +00001036 finally:
1037 if os.listdir(tmp_dir):
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001038 self.Print('_____ removing non-empty tmp dir %s' % tmp_dir)
ilevy@chromium.org3534aa52013-07-20 01:58:08 +00001039 gclient_utils.rmtree(tmp_dir)
primiano@chromium.org5439ea52014-08-06 17:18:18 +00001040 if template_dir:
1041 gclient_utils.rmtree(template_dir)
Edward Lemur579c9862018-07-13 23:17:51 +00001042 self._SetFetchConfig(options)
1043 self._Fetch(options, prune=options.force)
John Budorick882c91e2018-07-12 22:11:41 +00001044 revision = self._AutoFetchRef(options, revision)
1045 remote_ref = scm.GIT.RefToRemoteRef(revision, self.remote)
1046 self._Checkout(options, ''.join(remote_ref or revision), quiet=True)
romain.pokrzywka@gmail.com483a0ba2014-05-30 00:06:07 +00001047 if self._GetCurrentBranch() is None:
msb@chromium.org786fb682010-06-02 15:16:23 +00001048 # Squelch git's very verbose detached HEAD warning and use our own
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001049 self.Print(
smut@google.comd33eab32014-07-07 19:35:18 +00001050 ('Checked out %s to a detached HEAD. Before making any commits\n'
1051 'in this repo, you should use \'git checkout <branch>\' to switch to\n'
1052 'an existing branch or use \'git checkout %s -b <branch>\' to\n'
John Budorick882c91e2018-07-12 22:11:41 +00001053 'create a new branch for your work.') % (revision, self.remote))
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001054
szager@chromium.org6cd41b62014-04-21 23:55:22 +00001055 def _AskForData(self, prompt, options):
bauerb@chromium.org30c46d62014-01-23 12:11:56 +00001056 if options.jobs > 1:
szager@chromium.org6cd41b62014-04-21 23:55:22 +00001057 self.Print(prompt)
bauerb@chromium.org30c46d62014-01-23 12:11:56 +00001058 raise gclient_utils.Error("Background task requires input. Rerun "
1059 "gclient with --jobs=1 so that\n"
1060 "interaction is possible.")
1061 try:
1062 return raw_input(prompt)
1063 except KeyboardInterrupt:
1064 # Hide the exception.
1065 sys.exit(1)
1066
1067
maruel@chromium.orgf5d37bf2010-09-02 00:50:34 +00001068 def _AttemptRebase(self, upstream, files, options, newbase=None,
bauerb@chromium.org30c46d62014-01-23 12:11:56 +00001069 branch=None, printed_path=False, merge=False):
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001070 """Attempt to rebase onto either upstream or, if specified, newbase."""
iannucci@chromium.org396e1a62013-07-03 19:41:04 +00001071 if files is not None:
Aaron Gablef4068aa2017-12-12 15:14:09 -08001072 files.extend(self._GetDiffFilenames(upstream))
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001073 revision = upstream
1074 if newbase:
1075 revision = newbase
bauerb@chromium.org30c46d62014-01-23 12:11:56 +00001076 action = 'merge' if merge else 'rebase'
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001077 if not printed_path:
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001078 self.Print('_____ %s : Attempting %s onto %s...' % (
bauerb@chromium.org30c46d62014-01-23 12:11:56 +00001079 self.relpath, action, revision))
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001080 printed_path = True
1081 else:
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001082 self.Print('Attempting %s onto %s...' % (action, revision))
bauerb@chromium.org30c46d62014-01-23 12:11:56 +00001083
1084 if merge:
1085 merge_output = self._Capture(['merge', revision])
1086 if options.verbose:
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001087 self.Print(merge_output)
bauerb@chromium.org30c46d62014-01-23 12:11:56 +00001088 return
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001089
1090 # Build the rebase command here using the args
1091 # git rebase [options] [--onto <newbase>] <upstream> [<branch>]
1092 rebase_cmd = ['rebase']
maruel@chromium.orgf5d37bf2010-09-02 00:50:34 +00001093 if options.verbose:
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001094 rebase_cmd.append('--verbose')
1095 if newbase:
1096 rebase_cmd.extend(['--onto', newbase])
1097 rebase_cmd.append(upstream)
1098 if branch:
1099 rebase_cmd.append(branch)
1100
1101 try:
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +00001102 rebase_output = scm.GIT.Capture(rebase_cmd, cwd=self.checkout_path)
maruel@chromium.orgbffad372011-09-08 17:54:22 +00001103 except subprocess2.CalledProcessError, e:
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +00001104 if (re.match(r'cannot rebase: you have unstaged changes', e.stderr) or
1105 re.match(r'cannot rebase: your index contains uncommitted changes',
1106 e.stderr)):
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001107 while True:
bauerb@chromium.org30c46d62014-01-23 12:11:56 +00001108 rebase_action = self._AskForData(
maruel@chromium.org90541732011-04-01 17:54:18 +00001109 'Cannot rebase because of unstaged changes.\n'
1110 '\'git reset --hard HEAD\' ?\n'
1111 'WARNING: destroys any uncommitted work in your current branch!'
bratell@opera.com18fa4542013-05-21 13:30:46 +00001112 ' (y)es / (q)uit / (s)how : ', options)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001113 if re.match(r'yes|y', rebase_action, re.I):
Robert Iannuccic41d8b92017-02-16 17:07:37 -08001114 self._Scrub('HEAD', options)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001115 # Should this be recursive?
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +00001116 rebase_output = scm.GIT.Capture(rebase_cmd, cwd=self.checkout_path)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001117 break
1118 elif re.match(r'quit|q', rebase_action, re.I):
1119 raise gclient_utils.Error("Please merge or rebase manually\n"
1120 "cd %s && git " % self.checkout_path
1121 + "%s" % ' '.join(rebase_cmd))
1122 elif re.match(r'show|s', rebase_action, re.I):
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001123 self.Print('%s' % e.stderr.strip())
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001124 continue
1125 else:
1126 gclient_utils.Error("Input not recognized")
1127 continue
1128 elif re.search(r'^CONFLICT', e.stdout, re.M):
1129 raise gclient_utils.Error("Conflict while rebasing this branch.\n"
1130 "Fix the conflict and run gclient again.\n"
1131 "See 'man git-rebase' for details.\n")
1132 else:
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001133 self.Print(e.stdout.strip())
1134 self.Print('Rebase produced error output:\n%s' % e.stderr.strip())
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001135 raise gclient_utils.Error("Unrecognized error, please merge or rebase "
1136 "manually.\ncd %s && git " %
1137 self.checkout_path
1138 + "%s" % ' '.join(rebase_cmd))
1139
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001140 self.Print(rebase_output.strip())
maruel@chromium.orgf5d37bf2010-09-02 00:50:34 +00001141 if not options.verbose:
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001142 # Make the output a little prettier. It's nice to have some
1143 # whitespace between projects when syncing.
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001144 self.Print('')
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001145
maruel@chromium.org6e29d572010-06-04 17:32:20 +00001146 @staticmethod
1147 def _CheckMinVersion(min_version):
maruel@chromium.orgd0f854a2010-03-11 19:35:53 +00001148 (ok, current_version) = scm.GIT.AssertVersion(min_version)
1149 if not ok:
1150 raise gclient_utils.Error('git version %s < minimum required %s' %
1151 (current_version, min_version))
msb@chromium.org923a0372009-12-11 20:42:43 +00001152
John Budorick882c91e2018-07-12 22:11:41 +00001153 def _EnsureValidHeadObjectOrCheckout(self, revision, options, url):
tandrii@chromium.orgc438c142015-08-24 22:55:55 +00001154 # Special case handling if all 3 conditions are met:
1155 # * the mirros have recently changed, but deps destination remains same,
1156 # * the git histories of mirrors are conflicting.
1157 # * git cache is used
1158 # This manifests itself in current checkout having invalid HEAD commit on
1159 # most git operations. Since git cache is used, just deleted the .git
1160 # folder, and re-create it by cloning.
1161 try:
1162 self._Capture(['rev-list', '-n', '1', 'HEAD'])
1163 except subprocess2.CalledProcessError as e:
1164 if ('fatal: bad object HEAD' in e.stderr
1165 and self.cache_dir and self.cache_dir in url):
1166 self.Print((
1167 'Likely due to DEPS change with git cache_dir, '
1168 'the current commit points to no longer existing object.\n'
1169 '%s' % e)
1170 )
1171 self._DeleteOrMove(options.force)
John Budorick882c91e2018-07-12 22:11:41 +00001172 self._Clone(revision, url, options)
tandrii@chromium.orgc438c142015-08-24 22:55:55 +00001173 else:
1174 raise
1175
msb@chromium.org786fb682010-06-02 15:16:23 +00001176 def _IsRebasing(self):
1177 # Check for any of REBASE-i/REBASE-m/REBASE/AM. Unfortunately git doesn't
1178 # have a plumbing command to determine whether a rebase is in progress, so
1179 # for now emualate (more-or-less) git-rebase.sh / git-completion.bash
1180 g = os.path.join(self.checkout_path, '.git')
1181 return (
1182 os.path.isdir(os.path.join(g, "rebase-merge")) or
1183 os.path.isdir(os.path.join(g, "rebase-apply")))
1184
Robert Iannuccic41d8b92017-02-16 17:07:37 -08001185 def _CheckClean(self, revision, fixup=False):
iannucci@chromium.orgd9b318c2015-12-04 20:03:08 +00001186 lockfile = os.path.join(self.checkout_path, ".git", "index.lock")
1187 if os.path.exists(lockfile):
1188 raise gclient_utils.Error(
agable83faed02016-10-24 14:37:10 -07001189 '\n____ %s at %s\n'
iannucci@chromium.orgd9b318c2015-12-04 20:03:08 +00001190 '\tYour repo is locked, possibly due to a concurrent git process.\n'
1191 '\tIf no git executable is running, then clean up %r and try again.\n'
agable83faed02016-10-24 14:37:10 -07001192 % (self.relpath, revision, lockfile))
iannucci@chromium.orgd9b318c2015-12-04 20:03:08 +00001193
msb@chromium.org786fb682010-06-02 15:16:23 +00001194 # Make sure the tree is clean; see git-rebase.sh for reference
1195 try:
1196 scm.GIT.Capture(['update-index', '--ignore-submodules', '--refresh'],
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +00001197 cwd=self.checkout_path)
maruel@chromium.orgbffad372011-09-08 17:54:22 +00001198 except subprocess2.CalledProcessError:
agable83faed02016-10-24 14:37:10 -07001199 raise gclient_utils.Error('\n____ %s at %s\n'
maruel@chromium.org6e29d572010-06-04 17:32:20 +00001200 '\tYou have unstaged changes.\n'
1201 '\tPlease commit, stash, or reset.\n'
agable83faed02016-10-24 14:37:10 -07001202 % (self.relpath, revision))
msb@chromium.org786fb682010-06-02 15:16:23 +00001203 try:
1204 scm.GIT.Capture(['diff-index', '--cached', '--name-status', '-r',
smut@google.com27c9c8a2014-09-11 19:57:55 +00001205 '--ignore-submodules', 'HEAD', '--'],
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +00001206 cwd=self.checkout_path)
maruel@chromium.orgbffad372011-09-08 17:54:22 +00001207 except subprocess2.CalledProcessError:
agable83faed02016-10-24 14:37:10 -07001208 raise gclient_utils.Error('\n____ %s at %s\n'
maruel@chromium.org6e29d572010-06-04 17:32:20 +00001209 '\tYour index contains uncommitted changes\n'
1210 '\tPlease commit, stash, or reset.\n'
agable83faed02016-10-24 14:37:10 -07001211 % (self.relpath, revision))
msb@chromium.org786fb682010-06-02 15:16:23 +00001212
agable83faed02016-10-24 14:37:10 -07001213 def _CheckDetachedHead(self, revision, _options):
msb@chromium.org786fb682010-06-02 15:16:23 +00001214 # HEAD is detached. Make sure it is safe to move away from (i.e., it is
1215 # reference by a commit). If not, error out -- most likely a rebase is
1216 # in progress, try to detect so we can give a better error.
1217 try:
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +00001218 scm.GIT.Capture(['name-rev', '--no-undefined', 'HEAD'],
1219 cwd=self.checkout_path)
maruel@chromium.orgbffad372011-09-08 17:54:22 +00001220 except subprocess2.CalledProcessError:
msb@chromium.org786fb682010-06-02 15:16:23 +00001221 # Commit is not contained by any rev. See if the user is rebasing:
1222 if self._IsRebasing():
1223 # Punt to the user
agable83faed02016-10-24 14:37:10 -07001224 raise gclient_utils.Error('\n____ %s at %s\n'
msb@chromium.org786fb682010-06-02 15:16:23 +00001225 '\tAlready in a conflict, i.e. (no branch).\n'
1226 '\tFix the conflict and run gclient again.\n'
1227 '\tOr to abort run:\n\t\tgit-rebase --abort\n'
1228 '\tSee man git-rebase for details.\n'
agable83faed02016-10-24 14:37:10 -07001229 % (self.relpath, revision))
msb@chromium.org786fb682010-06-02 15:16:23 +00001230 # Let's just save off the commit so we can proceed.
maruel@chromium.org6cafa132010-09-07 14:17:26 +00001231 name = ('saved-by-gclient-' +
1232 self._Capture(['rev-parse', '--short', 'HEAD']))
mmoss@chromium.org77bd7362013-09-25 23:46:14 +00001233 self._Capture(['branch', '-f', name])
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001234 self.Print('_____ found an unreferenced commit and saved it as \'%s\'' %
maruel@chromium.orgf5d37bf2010-09-02 00:50:34 +00001235 name)
msb@chromium.org786fb682010-06-02 15:16:23 +00001236
msb@chromium.org5bde4852009-12-14 16:47:12 +00001237 def _GetCurrentBranch(self):
msb@chromium.org786fb682010-06-02 15:16:23 +00001238 # Returns name of current branch or None for detached HEAD
maruel@chromium.org6cafa132010-09-07 14:17:26 +00001239 branch = self._Capture(['rev-parse', '--abbrev-ref=strict', 'HEAD'])
msb@chromium.org786fb682010-06-02 15:16:23 +00001240 if branch == 'HEAD':
msb@chromium.org5bde4852009-12-14 16:47:12 +00001241 return None
1242 return branch
1243
borenet@google.comc3e09d22014-04-10 13:58:18 +00001244 def _Capture(self, args, **kwargs):
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001245 kwargs.setdefault('cwd', self.checkout_path)
1246 kwargs.setdefault('stderr', subprocess2.PIPE)
Robert Iannuccia7a9ceb2017-02-16 17:38:06 -08001247 strip = kwargs.pop('strip', True)
szager@chromium.org6d8115d2014-04-23 20:59:23 +00001248 env = scm.GIT.ApplyEnvVars(kwargs)
Robert Iannuccia7a9ceb2017-02-16 17:38:06 -08001249 ret = subprocess2.check_output(['git'] + args, env=env, **kwargs)
1250 if strip:
1251 ret = ret.strip()
1252 return ret
maruel@chromium.org6cafa132010-09-07 14:17:26 +00001253
dnj@chromium.orgbb424c02014-06-23 22:42:51 +00001254 def _Checkout(self, options, ref, force=False, quiet=None):
1255 """Performs a 'git-checkout' operation.
1256
1257 Args:
1258 options: The configured option set
1259 ref: (str) The branch/commit to checkout
1260 quiet: (bool/None) Whether or not the checkout shoud pass '--quiet'; if
1261 'None', the behavior is inferred from 'options.verbose'.
1262 Returns: (str) The output of the checkout operation
1263 """
1264 if quiet is None:
1265 quiet = (not options.verbose)
1266 checkout_args = ['checkout']
1267 if force:
1268 checkout_args.append('--force')
1269 if quiet:
1270 checkout_args.append('--quiet')
1271 checkout_args.append(ref)
1272 return self._Capture(checkout_args)
1273
Paweł Hajdan, Jr63b8c2a2017-09-05 17:59:08 +02001274 def _Fetch(self, options, remote=None, prune=False, quiet=False,
1275 refspec=None):
dnj@chromium.org680f2172014-06-25 00:39:32 +00001276 cfg = gclient_utils.DefaultIndexPackConfig(self.url)
Edward Lemurd64781e2018-07-11 23:09:55 +00001277 # When a mirror is configured, it fetches only the refs/heads, and possibly
1278 # the refs/branch-heads and refs/tags, but not the refs/changes. So, if
1279 # we're asked to fetch a refs/changes ref from the mirror, it won't have it.
1280 # This makes sure that we always fetch refs/changes directly from the
1281 # repository and not from the mirror.
1282 if refspec and refspec.startswith('refs/changes'):
1283 remote, _ = gclient_utils.SplitUrlRevision(self.url)
1284 # Make sure that we fetch the (remote) refs/changes/xx ref to the (local)
1285 # refs/changes/xx ref.
1286 if ':' not in refspec:
1287 refspec += ':' + refspec
dnj@chromium.org680f2172014-06-25 00:39:32 +00001288 fetch_cmd = cfg + [
1289 'fetch',
1290 remote or self.remote,
1291 ]
Paweł Hajdan, Jr63b8c2a2017-09-05 17:59:08 +02001292 if refspec:
1293 fetch_cmd.append(refspec)
dnj@chromium.org680f2172014-06-25 00:39:32 +00001294
1295 if prune:
1296 fetch_cmd.append('--prune')
1297 if options.verbose:
1298 fetch_cmd.append('--verbose')
1299 elif quiet:
1300 fetch_cmd.append('--quiet')
tandrii64103db2016-10-11 05:30:05 -07001301 self._Run(fetch_cmd, options, show_header=options.verbose, retry=True)
dnj@chromium.org680f2172014-06-25 00:39:32 +00001302
1303 # Return the revision that was fetched; this will be stored in 'FETCH_HEAD'
1304 return self._Capture(['rev-parse', '--verify', 'FETCH_HEAD'])
1305
Edward Lemur579c9862018-07-13 23:17:51 +00001306 def _SetFetchConfig(self, options):
szager@chromium.org8d3348f2014-08-19 22:49:16 +00001307 """Adds, and optionally fetches, "branch-heads" and "tags" refspecs
1308 if requested."""
Edward Lemur2f38df62018-07-14 02:13:21 +00001309 if options.force or options.reset:
Edward Lemur579c9862018-07-13 23:17:51 +00001310 try:
1311 self._Run(['config', '--unset-all', 'remote.%s.fetch' % self.remote],
1312 options)
1313 self._Run(['config', 'remote.%s.fetch' % self.remote,
1314 '+refs/heads/*:refs/remotes/%s/*' % self.remote], options)
1315 except subprocess2.CalledProcessError as e:
1316 # If exit code was 5, it means we attempted to unset a config that
1317 # didn't exist. Ignore it.
1318 if e.returncode != 5:
1319 raise
mmoss@chromium.orge409df62013-04-16 17:28:57 +00001320 if hasattr(options, 'with_branch_heads') and options.with_branch_heads:
maruel@chromium.org1a60dca2013-11-26 14:06:26 +00001321 config_cmd = ['config', 'remote.%s.fetch' % self.remote,
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +00001322 '+refs/branch-heads/*:refs/remotes/branch-heads/*',
1323 '^\\+refs/branch-heads/\\*:.*$']
1324 self._Run(config_cmd, options)
szager@chromium.org8d3348f2014-08-19 22:49:16 +00001325 if hasattr(options, 'with_tags') and options.with_tags:
1326 config_cmd = ['config', 'remote.%s.fetch' % self.remote,
1327 '+refs/tags/*:refs/tags/*',
1328 '^\\+refs/tags/\\*:.*$']
1329 self._Run(config_cmd, options)
mmoss@chromium.orge409df62013-04-16 17:28:57 +00001330
John Budorick882c91e2018-07-12 22:11:41 +00001331 def _AutoFetchRef(self, options, revision):
Paweł Hajdan, Jr63b8c2a2017-09-05 17:59:08 +02001332 """Attempts to fetch |revision| if not available in local repo.
1333
1334 Returns possibly updated revision."""
John Budorick882c91e2018-07-12 22:11:41 +00001335 try:
1336 self._Capture(['rev-parse', revision])
1337 except subprocess2.CalledProcessError:
Paweł Hajdan, Jr63b8c2a2017-09-05 17:59:08 +02001338 self._Fetch(options, refspec=revision)
1339 revision = self._Capture(['rev-parse', 'FETCH_HEAD'])
1340 return revision
1341
dnj@chromium.org680f2172014-06-25 00:39:32 +00001342 def _Run(self, args, options, show_header=True, **kwargs):
Quinten Yearsleyb2cc4a92016-12-15 13:53:26 -08001343 # Disable 'unused options' warning | pylint: disable=unused-argument
sbc@chromium.org2cd0b8e2014-09-22 21:17:59 +00001344 kwargs.setdefault('cwd', self.checkout_path)
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001345 kwargs.setdefault('stdout', self.out_fh)
szager@chromium.org848fd492014-04-09 19:06:44 +00001346 kwargs['filter_fn'] = self.filter
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001347 kwargs.setdefault('print_stdout', False)
szager@chromium.org6d8115d2014-04-23 20:59:23 +00001348 env = scm.GIT.ApplyEnvVars(kwargs)
agable@chromium.org772efaf2014-04-01 02:35:44 +00001349 cmd = ['git'] + args
dnj@chromium.org680f2172014-06-25 00:39:32 +00001350 if show_header:
sbc@chromium.org2cd0b8e2014-09-22 21:17:59 +00001351 gclient_utils.CheckCallAndFilterAndHeader(cmd, env=env, **kwargs)
1352 else:
1353 gclient_utils.CheckCallAndFilter(cmd, env=env, **kwargs)
John Budorick0f7b2002018-01-19 15:46:17 -08001354
1355
1356class CipdPackage(object):
1357 """A representation of a single CIPD package."""
1358
John Budorickd3ba72b2018-03-20 12:27:42 -07001359 def __init__(self, name, version, authority_for_subdir):
John Budorick0f7b2002018-01-19 15:46:17 -08001360 self._authority_for_subdir = authority_for_subdir
1361 self._name = name
1362 self._version = version
1363
1364 @property
John Budorick0f7b2002018-01-19 15:46:17 -08001365 def authority_for_subdir(self):
1366 """Whether this package has authority to act on behalf of its subdir.
1367
1368 Some operations should only be performed once per subdirectory. A package
1369 that has authority for its subdirectory is the only package that should
1370 perform such operations.
1371
1372 Returns:
1373 bool; whether this package has subdir authority.
1374 """
1375 return self._authority_for_subdir
1376
1377 @property
1378 def name(self):
1379 return self._name
1380
1381 @property
1382 def version(self):
1383 return self._version
1384
1385
1386class CipdRoot(object):
1387 """A representation of a single CIPD root."""
1388 def __init__(self, root_dir, service_url):
1389 self._all_packages = set()
1390 self._mutator_lock = threading.Lock()
1391 self._packages_by_subdir = collections.defaultdict(list)
1392 self._root_dir = root_dir
1393 self._service_url = service_url
1394
1395 def add_package(self, subdir, package, version):
1396 """Adds a package to this CIPD root.
1397
1398 As far as clients are concerned, this grants both root and subdir authority
1399 to packages arbitrarily. (The implementation grants root authority to the
1400 first package added and subdir authority to the first package added for that
1401 subdir, but clients should not depend on or expect that behavior.)
1402
1403 Args:
1404 subdir: str; relative path to where the package should be installed from
1405 the cipd root directory.
1406 package: str; the cipd package name.
1407 version: str; the cipd package version.
1408 Returns:
1409 CipdPackage; the package that was created and added to this root.
1410 """
1411 with self._mutator_lock:
1412 cipd_package = CipdPackage(
1413 package, version,
John Budorick0f7b2002018-01-19 15:46:17 -08001414 not self._packages_by_subdir[subdir])
1415 self._all_packages.add(cipd_package)
1416 self._packages_by_subdir[subdir].append(cipd_package)
1417 return cipd_package
1418
1419 def packages(self, subdir):
1420 """Get the list of configured packages for the given subdir."""
1421 return list(self._packages_by_subdir[subdir])
1422
1423 def clobber(self):
1424 """Remove the .cipd directory.
1425
1426 This is useful for forcing ensure to redownload and reinitialize all
1427 packages.
1428 """
1429 with self._mutator_lock:
John Budorickd3ba72b2018-03-20 12:27:42 -07001430 cipd_cache_dir = os.path.join(self.root_dir, '.cipd')
John Budorick0f7b2002018-01-19 15:46:17 -08001431 try:
1432 gclient_utils.rmtree(os.path.join(cipd_cache_dir))
1433 except OSError:
1434 if os.path.exists(cipd_cache_dir):
1435 raise
1436
1437 @contextlib.contextmanager
1438 def _create_ensure_file(self):
1439 try:
1440 ensure_file = None
1441 with tempfile.NamedTemporaryFile(
1442 suffix='.ensure', delete=False) as ensure_file:
John Budorick302bb842018-07-17 23:49:17 +00001443 ensure_file.write('$ParanoidMode CheckPresence\n\n')
John Budorick0f7b2002018-01-19 15:46:17 -08001444 for subdir, packages in sorted(self._packages_by_subdir.iteritems()):
1445 ensure_file.write('@Subdir %s\n' % subdir)
Edward Lemurfbb06aa2018-06-11 20:43:06 +00001446 for package in sorted(packages, key=lambda p: p.name):
John Budorick0f7b2002018-01-19 15:46:17 -08001447 ensure_file.write('%s %s\n' % (package.name, package.version))
1448 ensure_file.write('\n')
1449 yield ensure_file.name
1450 finally:
1451 if ensure_file is not None and os.path.exists(ensure_file.name):
1452 os.remove(ensure_file.name)
1453
1454 def ensure(self):
1455 """Run `cipd ensure`."""
1456 with self._mutator_lock:
1457 with self._create_ensure_file() as ensure_file:
1458 cmd = [
1459 'cipd', 'ensure',
1460 '-log-level', 'error',
1461 '-root', self.root_dir,
1462 '-ensure-file', ensure_file,
1463 ]
1464 gclient_utils.CheckCallAndFilterAndHeader(cmd)
1465
John Budorickd3ba72b2018-03-20 12:27:42 -07001466 def run(self, command):
1467 if command == 'update':
1468 self.ensure()
1469 elif command == 'revert':
1470 self.clobber()
1471 self.ensure()
1472
John Budorick0f7b2002018-01-19 15:46:17 -08001473 def created_package(self, package):
1474 """Checks whether this root created the given package.
1475
1476 Args:
1477 package: CipdPackage; the package to check.
1478 Returns:
1479 bool; whether this root created the given package.
1480 """
1481 return package in self._all_packages
1482
1483 @property
1484 def root_dir(self):
1485 return self._root_dir
1486
1487 @property
1488 def service_url(self):
1489 return self._service_url
1490
1491
1492class CipdWrapper(SCMWrapper):
1493 """Wrapper for CIPD.
1494
1495 Currently only supports chrome-infra-packages.appspot.com.
1496 """
John Budorick3929e9e2018-02-04 18:18:07 -08001497 name = 'cipd'
John Budorick0f7b2002018-01-19 15:46:17 -08001498
1499 def __init__(self, url=None, root_dir=None, relpath=None, out_fh=None,
1500 out_cb=None, root=None, package=None):
1501 super(CipdWrapper, self).__init__(
1502 url=url, root_dir=root_dir, relpath=relpath, out_fh=out_fh,
1503 out_cb=out_cb)
1504 assert root.created_package(package)
1505 self._package = package
1506 self._root = root
1507
1508 #override
1509 def GetCacheMirror(self):
1510 return None
1511
1512 #override
1513 def GetActualRemoteURL(self, options):
1514 return self._root.service_url
1515
1516 #override
1517 def DoesRemoteURLMatch(self, options):
1518 del options
1519 return True
1520
1521 def revert(self, options, args, file_list):
John Budorickd3ba72b2018-03-20 12:27:42 -07001522 """Does nothing.
1523
1524 CIPD packages should be reverted at the root by running
1525 `CipdRoot.run('revert')`.
1526 """
1527 pass
John Budorick0f7b2002018-01-19 15:46:17 -08001528
1529 def diff(self, options, args, file_list):
1530 """CIPD has no notion of diffing."""
1531 pass
1532
1533 def pack(self, options, args, file_list):
1534 """CIPD has no notion of diffing."""
1535 pass
1536
1537 def revinfo(self, options, args, file_list):
1538 """Grab the instance ID."""
1539 try:
1540 tmpdir = tempfile.mkdtemp()
1541 describe_json_path = os.path.join(tmpdir, 'describe.json')
1542 cmd = [
1543 'cipd', 'describe',
1544 self._package.name,
1545 '-log-level', 'error',
1546 '-version', self._package.version,
1547 '-json-output', describe_json_path
1548 ]
1549 gclient_utils.CheckCallAndFilter(
1550 cmd, filter_fn=lambda _line: None, print_stdout=False)
1551 with open(describe_json_path) as f:
1552 describe_json = json.load(f)
1553 return describe_json.get('result', {}).get('pin', {}).get('instance_id')
1554 finally:
1555 gclient_utils.rmtree(tmpdir)
1556
1557 def status(self, options, args, file_list):
1558 pass
1559
1560 def update(self, options, args, file_list):
John Budorickd3ba72b2018-03-20 12:27:42 -07001561 """Does nothing.
1562
1563 CIPD packages should be updated at the root by running
1564 `CipdRoot.run('update')`.
1565 """
1566 pass