blob: e7edc00b6866105d42a38007ffcc70276baf3809 [file] [log] [blame]
steveblock@chromium.org93567042012-02-15 01:02:26 +00001# Copyright (c) 2012 The Chromium Authors. All rights reserved.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +00002# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
maruel@chromium.org5f3eee32009-09-17 00:34:30 +00004
maruel@chromium.orgd5800f12009-11-12 20:03:43 +00005"""Gclient-specific SCM-specific operations."""
maruel@chromium.org5f3eee32009-09-17 00:34:30 +00006
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00007from __future__ import print_function
8
John Budorick0f7b2002018-01-19 15:46:17 -08009import collections
10import contextlib
borenet@google.comb2256212014-05-07 20:57:28 +000011import errno
John Budorick0f7b2002018-01-19 15:46:17 -080012import json
maruel@chromium.org754960e2009-09-21 12:31:05 +000013import logging
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000014import os
Tomasz Wiszkowskid4e66882021-08-19 21:35:09 +000015import platform
maruel@chromium.orgee4071d2009-12-22 22:25:37 +000016import posixpath
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000017import re
maruel@chromium.org90541732011-04-01 17:54:18 +000018import sys
ilevy@chromium.org3534aa52013-07-20 01:58:08 +000019import tempfile
John Budorick0f7b2002018-01-19 15:46:17 -080020import threading
zty@chromium.org6279e8a2014-02-13 01:45:25 +000021import traceback
Raul Tambreb946b232019-03-26 14:48:46 +000022
23try:
24 import urlparse
25except ImportError: # For Py3 compatibility
26 import urllib.parse as urlparse
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000027
28import gclient_utils
Ravi Mistryecda7822022-02-28 16:22:20 +000029import gerrit_util
szager@chromium.org848fd492014-04-09 19:06:44 +000030import git_cache
maruel@chromium.org31cb48a2011-04-04 18:01:36 +000031import scm
borenet@google.comb2256212014-05-07 20:57:28 +000032import shutil
maruel@chromium.org31cb48a2011-04-04 18:01:36 +000033import subprocess2
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000034
35
smutae7ea312016-07-18 11:59:41 -070036class NoUsableRevError(gclient_utils.Error):
37 """Raised if requested revision isn't found in checkout."""
38
39
haitao.feng@intel.com306080c2012-05-04 13:11:29 +000040class DiffFiltererWrapper(object):
41 """Simple base class which tracks which file is being diffed and
maruel@chromium.orgee4071d2009-12-22 22:25:37 +000042 replaces instances of its file name in the original and
agable41e3a6c2016-10-20 11:36:56 -070043 working copy lines of the git diff output."""
haitao.feng@intel.com306080c2012-05-04 13:11:29 +000044 index_string = None
maruel@chromium.orgee4071d2009-12-22 22:25:37 +000045 original_prefix = "--- "
46 working_prefix = "+++ "
47
szager@chromium.orgfe0d1902014-04-08 20:50:44 +000048 def __init__(self, relpath, print_func):
maruel@chromium.orgee4071d2009-12-22 22:25:37 +000049 # Note that we always use '/' as the path separator to be
agable41e3a6c2016-10-20 11:36:56 -070050 # consistent with cygwin-style output on Windows
maruel@chromium.orgee4071d2009-12-22 22:25:37 +000051 self._relpath = relpath.replace("\\", "/")
haitao.feng@intel.com306080c2012-05-04 13:11:29 +000052 self._current_file = None
szager@chromium.orgfe0d1902014-04-08 20:50:44 +000053 self._print_func = print_func
maruel@chromium.orgee4071d2009-12-22 22:25:37 +000054
maruel@chromium.org6e29d572010-06-04 17:32:20 +000055 def SetCurrentFile(self, current_file):
56 self._current_file = current_file
haitao.feng@intel.com306080c2012-05-04 13:11:29 +000057
iannucci@chromium.org3830a672013-02-19 20:15:14 +000058 @property
59 def _replacement_file(self):
haitao.feng@intel.com306080c2012-05-04 13:11:29 +000060 return posixpath.join(self._relpath, self._current_file)
maruel@chromium.orgee4071d2009-12-22 22:25:37 +000061
maruel@chromium.orgf5d37bf2010-09-02 00:50:34 +000062 def _Replace(self, line):
63 return line.replace(self._current_file, self._replacement_file)
maruel@chromium.orgee4071d2009-12-22 22:25:37 +000064
65 def Filter(self, line):
66 if (line.startswith(self.index_string)):
67 self.SetCurrentFile(line[len(self.index_string):])
maruel@chromium.orgf5d37bf2010-09-02 00:50:34 +000068 line = self._Replace(line)
maruel@chromium.orgee4071d2009-12-22 22:25:37 +000069 else:
70 if (line.startswith(self.original_prefix) or
71 line.startswith(self.working_prefix)):
maruel@chromium.orgf5d37bf2010-09-02 00:50:34 +000072 line = self._Replace(line)
szager@chromium.orgfe0d1902014-04-08 20:50:44 +000073 self._print_func(line)
maruel@chromium.orgee4071d2009-12-22 22:25:37 +000074
75
haitao.feng@intel.com306080c2012-05-04 13:11:29 +000076class GitDiffFilterer(DiffFiltererWrapper):
77 index_string = "diff --git "
78
79 def SetCurrentFile(self, current_file):
80 # Get filename by parsing "a/<filename> b/<filename>"
81 self._current_file = current_file[:(len(current_file)/2)][2:]
82
83 def _Replace(self, line):
84 return re.sub("[a|b]/" + self._current_file, self._replacement_file, line)
85
86
msb@chromium.orgcb5442b2009-09-22 16:51:24 +000087# SCMWrapper base class
88
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000089class SCMWrapper(object):
90 """Add necessary glue between all the supported SCM.
91
msb@chromium.orgd6504212010-01-13 17:34:31 +000092 This is the abstraction layer to bind to different SCM.
93 """
szager@chromium.orgfe0d1902014-04-08 20:50:44 +000094 def __init__(self, url=None, root_dir=None, relpath=None, out_fh=None,
Edward Lemur231f5ea2018-01-31 19:02:36 +010095 out_cb=None, print_outbuf=False):
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000096 self.url = url
maruel@chromium.org5e73b0c2009-09-18 19:47:48 +000097 self._root_dir = root_dir
98 if self._root_dir:
99 self._root_dir = self._root_dir.replace('/', os.sep)
100 self.relpath = relpath
101 if self.relpath:
102 self.relpath = self.relpath.replace('/', os.sep)
msb@chromium.orge28e4982009-09-25 20:51:45 +0000103 if self.relpath and self._root_dir:
104 self.checkout_path = os.path.join(self._root_dir, self.relpath)
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000105 if out_fh is None:
106 out_fh = sys.stdout
107 self.out_fh = out_fh
108 self.out_cb = out_cb
Edward Lemur231f5ea2018-01-31 19:02:36 +0100109 self.print_outbuf = print_outbuf
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000110
111 def Print(self, *args, **kwargs):
112 kwargs.setdefault('file', self.out_fh)
113 if kwargs.pop('timestamp', True):
114 self.out_fh.write('[%s] ' % gclient_utils.Elapsed())
115 print(*args, **kwargs)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000116
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000117 def RunCommand(self, command, options, args, file_list=None):
agabledebf6c82016-12-21 12:50:12 -0800118 commands = ['update', 'updatesingle', 'revert',
tony@chromium.org4b5b1772010-04-08 01:52:56 +0000119 'revinfo', 'status', 'diff', 'pack', 'runhooks']
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000120
121 if not command in commands:
122 raise gclient_utils.Error('Unknown command %s' % command)
123
msb@chromium.orgcb5442b2009-09-22 16:51:24 +0000124 if not command in dir(self):
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000125 raise gclient_utils.Error('Command %s not implemented in %s wrapper' % (
maruel@chromium.org9eda4112010-06-11 18:56:10 +0000126 command, self.__class__.__name__))
msb@chromium.orgcb5442b2009-09-22 16:51:24 +0000127
128 return getattr(self, command)(options, args, file_list)
129
hinoka@chromium.orgfa2b9b42014-08-22 18:08:53 +0000130 @staticmethod
131 def _get_first_remote_url(checkout_path):
132 log = scm.GIT.Capture(
133 ['config', '--local', '--get-regexp', r'remote.*.url'],
134 cwd=checkout_path)
135 # Get the second token of the first line of the log.
136 return log.splitlines()[0].split(' ', 1)[1]
137
levarum@chromium.org27a6f9a2016-05-28 00:21:49 +0000138 def GetCacheMirror(self):
Robert Iannuccia19649b2018-06-29 16:31:45 +0000139 if getattr(self, 'cache_dir', None):
levarum@chromium.org27a6f9a2016-05-28 00:21:49 +0000140 url, _ = gclient_utils.SplitUrlRevision(self.url)
141 return git_cache.Mirror(url)
142 return None
143
smut@google.comd33eab32014-07-07 19:35:18 +0000144 def GetActualRemoteURL(self, options):
borenet@google.com88d10082014-03-21 17:24:48 +0000145 """Attempt to determine the remote URL for this SCMWrapper."""
smut@google.comd33eab32014-07-07 19:35:18 +0000146 # Git
borenet@google.combda475e2014-03-24 19:04:45 +0000147 if os.path.exists(os.path.join(self.checkout_path, '.git')):
hinoka@chromium.orgfa2b9b42014-08-22 18:08:53 +0000148 actual_remote_url = self._get_first_remote_url(self.checkout_path)
borenet@google.com4e9be262014-04-08 19:40:30 +0000149
levarum@chromium.org27a6f9a2016-05-28 00:21:49 +0000150 mirror = self.GetCacheMirror()
151 # If the cache is used, obtain the actual remote URL from there.
152 if (mirror and mirror.exists() and
153 mirror.mirror_path.replace('\\', '/') ==
154 actual_remote_url.replace('\\', '/')):
155 actual_remote_url = self._get_first_remote_url(mirror.mirror_path)
borenet@google.com4e9be262014-04-08 19:40:30 +0000156 return actual_remote_url
borenet@google.com88d10082014-03-21 17:24:48 +0000157 return None
158
borenet@google.com4e9be262014-04-08 19:40:30 +0000159 def DoesRemoteURLMatch(self, options):
borenet@google.com88d10082014-03-21 17:24:48 +0000160 """Determine whether the remote URL of this checkout is the expected URL."""
161 if not os.path.exists(self.checkout_path):
162 # A checkout which doesn't exist can't be broken.
163 return True
164
smut@google.comd33eab32014-07-07 19:35:18 +0000165 actual_remote_url = self.GetActualRemoteURL(options)
borenet@google.com88d10082014-03-21 17:24:48 +0000166 if actual_remote_url:
borenet@google.com8156c9f2014-04-01 16:41:36 +0000167 return (gclient_utils.SplitUrlRevision(actual_remote_url)[0].rstrip('/')
168 == gclient_utils.SplitUrlRevision(self.url)[0].rstrip('/'))
Aravind Vasudevanc5f0cbb2022-01-24 23:56:57 +0000169
170 # This may occur if the self.checkout_path exists but does not contain a
171 # valid git checkout.
172 return False
borenet@google.com88d10082014-03-21 17:24:48 +0000173
borenet@google.comb09097a2014-04-09 19:09:08 +0000174 def _DeleteOrMove(self, force):
175 """Delete the checkout directory or move it out of the way.
176
177 Args:
178 force: bool; if True, delete the directory. Otherwise, just move it.
179 """
borenet@google.comb2256212014-05-07 20:57:28 +0000180 if force and os.environ.get('CHROME_HEADLESS') == '1':
181 self.Print('_____ Conflicting directory found in %s. Removing.'
182 % self.checkout_path)
183 gclient_utils.AddWarning('Conflicting directory %s deleted.'
184 % self.checkout_path)
185 gclient_utils.rmtree(self.checkout_path)
186 else:
187 bad_scm_dir = os.path.join(self._root_dir, '_bad_scm',
188 os.path.dirname(self.relpath))
189
190 try:
191 os.makedirs(bad_scm_dir)
192 except OSError as e:
193 if e.errno != errno.EEXIST:
194 raise
195
196 dest_path = tempfile.mkdtemp(
197 prefix=os.path.basename(self.relpath),
198 dir=bad_scm_dir)
199 self.Print('_____ Conflicting directory found in %s. Moving to %s.'
200 % (self.checkout_path, dest_path))
201 gclient_utils.AddWarning('Conflicting directory %s moved to %s.'
202 % (self.checkout_path, dest_path))
203 shutil.move(self.checkout_path, dest_path)
borenet@google.comb09097a2014-04-09 19:09:08 +0000204
msb@chromium.orgcb5442b2009-09-22 16:51:24 +0000205
maruel@chromium.org55e724e2010-03-11 19:36:49 +0000206class GitWrapper(SCMWrapper):
msb@chromium.orge28e4982009-09-25 20:51:45 +0000207 """Wrapper for Git"""
iannucci@chromium.org2702bcd2013-09-24 19:10:07 +0000208 name = 'git'
maruel@chromium.org1a60dca2013-11-26 14:06:26 +0000209 remote = 'origin'
msb@chromium.orge28e4982009-09-25 20:51:45 +0000210
Aravind Vasudevan14e6d232022-06-02 20:42:16 +0000211 _is_env_cog = None
212
213 @staticmethod
214 def _IsCog():
215 """Returns true if the env is cog"""
216 if not GitWrapper._is_env_cog:
217 GitWrapper._is_env_cog = os.getcwd().startswith('/google/src/cloud')
218
219 return GitWrapper._is_env_cog
220
Robert Iannuccia19649b2018-06-29 16:31:45 +0000221 @property
222 def cache_dir(self):
223 try:
224 return git_cache.Mirror.GetCachePath()
225 except RuntimeError:
226 return None
iannucci@chromium.org53456aa2013-07-03 19:38:34 +0000227
John Budorick0f7b2002018-01-19 15:46:17 -0800228 def __init__(self, url=None, *args, **kwargs):
igorgatis@gmail.com4e075672011-11-21 16:35:08 +0000229 """Removes 'git+' fake prefix from git URL."""
Henrique Ferreiroe72279d2019-04-17 12:01:50 +0000230 if url and (url.startswith('git+http://') or
231 url.startswith('git+https://')):
igorgatis@gmail.com4e075672011-11-21 16:35:08 +0000232 url = url[4:]
John Budorick0f7b2002018-01-19 15:46:17 -0800233 SCMWrapper.__init__(self, url, *args, **kwargs)
szager@chromium.org848fd492014-04-09 19:06:44 +0000234 filter_kwargs = { 'time_throttle': 1, 'out_fh': self.out_fh }
235 if self.out_cb:
236 filter_kwargs['predicate'] = self.out_cb
237 self.filter = gclient_utils.GitFilter(**filter_kwargs)
Nico Weberc49c88a2020-07-08 17:36:02 +0000238 self._running_under_rosetta = None
igorgatis@gmail.com4e075672011-11-21 16:35:08 +0000239
xusydoc@chromium.org885a9602013-05-31 09:54:40 +0000240 def GetCheckoutRoot(self):
241 return scm.GIT.GetCheckoutRoot(self.checkout_path)
242
iannucci@chromium.org396e1a62013-07-03 19:41:04 +0000243 def GetRevisionDate(self, _revision):
floitsch@google.comeaab7842011-04-28 09:07:58 +0000244 """Returns the given revision's date in ISO-8601 format (which contains the
245 time zone)."""
246 # TODO(floitsch): get the time-stamp of the given revision and not just the
247 # time-stamp of the currently checked out revision.
248 return self._Capture(['log', '-n', '1', '--format=%ai'])
249
Aaron Gablef4068aa2017-12-12 15:14:09 -0800250 def _GetDiffFilenames(self, base):
251 """Returns the names of files modified since base."""
252 return self._Capture(
Raul Tambrecd862e32019-05-10 21:19:00 +0000253 # Filter to remove base if it is None.
254 list(filter(bool, ['-c', 'core.quotePath=false', 'diff', '--name-only',
255 base])
256 )).split()
Aaron Gablef4068aa2017-12-12 15:14:09 -0800257
iannucci@chromium.org396e1a62013-07-03 19:41:04 +0000258 def diff(self, options, _args, _file_list):
Aaron Gable1853f662018-02-12 15:45:56 -0800259 _, revision = gclient_utils.SplitUrlRevision(self.url)
260 if not revision:
Josip Sokcevic7e133ff2021-07-13 17:44:53 +0000261 revision = 'refs/remotes/%s/main' % self.remote
Aaron Gable1853f662018-02-12 15:45:56 -0800262 self._Run(['-c', 'core.quotePath=false', 'diff', revision], options)
msb@chromium.orge28e4982009-09-25 20:51:45 +0000263
iannucci@chromium.org396e1a62013-07-03 19:41:04 +0000264 def pack(self, _options, _args, _file_list):
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000265 """Generates a patch file which can be applied to the root of the
msb@chromium.orgd6504212010-01-13 17:34:31 +0000266 repository.
267
268 The patch file is generated from a diff of the merge base of HEAD and
269 its upstream branch.
270 """
raphael.kubo.da.costa05c83592016-08-04 08:32:41 -0700271 try:
272 merge_base = [self._Capture(['merge-base', 'HEAD', self.remote])]
273 except subprocess2.CalledProcessError:
274 merge_base = []
maruel@chromium.org17d01792010-09-01 18:07:10 +0000275 gclient_utils.CheckCallAndFilter(
raphael.kubo.da.costa05c83592016-08-04 08:32:41 -0700276 ['git', 'diff'] + merge_base,
maruel@chromium.org8469bf92010-09-03 19:03:15 +0000277 cwd=self.checkout_path,
avakulenko@google.com255f2be2014-12-05 22:19:55 +0000278 filter_fn=GitDiffFilterer(self.relpath, print_func=self.Print).Filter)
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000279
Robert Iannuccic41d8b92017-02-16 17:07:37 -0800280 def _Scrub(self, target, options):
281 """Scrubs out all changes in the local repo, back to the state of target."""
mmoss@chromium.org50fd47f2014-02-13 01:03:19 +0000282 quiet = []
283 if not options.verbose:
284 quiet = ['--quiet']
Robert Iannuccic41d8b92017-02-16 17:07:37 -0800285 self._Run(['reset', '--hard', target] + quiet, options)
286 if options.force and options.delete_unversioned_trees:
287 # where `target` is a commit that contains both upper and lower case
288 # versions of the same file on a case insensitive filesystem, we are
289 # actually in a broken state here. The index will have both 'a' and 'A',
290 # but only one of them will exist on the disk. To progress, we delete
291 # everything that status thinks is modified.
Aaron Gable7817f022017-12-12 09:43:17 -0800292 output = self._Capture([
293 '-c', 'core.quotePath=false', 'status', '--porcelain'], strip=False)
Robert Iannuccia7a9ceb2017-02-16 17:38:06 -0800294 for line in output.splitlines():
Robert Iannuccic41d8b92017-02-16 17:07:37 -0800295 # --porcelain (v1) looks like:
296 # XY filename
297 try:
298 filename = line[3:]
299 self.Print('_____ Deleting residual after reset: %r.' % filename)
300 gclient_utils.rm_file_or_tree(
Robert Iannuccia7a9ceb2017-02-16 17:38:06 -0800301 os.path.join(self.checkout_path, filename))
Robert Iannuccic41d8b92017-02-16 17:07:37 -0800302 except OSError:
303 pass
304
John Budorick882c91e2018-07-12 22:11:41 +0000305 def _FetchAndReset(self, revision, file_list, options):
Robert Iannuccic41d8b92017-02-16 17:07:37 -0800306 """Equivalent to git fetch; git reset."""
Edward Lemur579c9862018-07-13 23:17:51 +0000307 self._SetFetchConfig(options)
mmoss@chromium.org50fd47f2014-02-13 01:03:19 +0000308
dnj@chromium.org680f2172014-06-25 00:39:32 +0000309 self._Fetch(options, prune=True, quiet=options.verbose)
John Budorick882c91e2018-07-12 22:11:41 +0000310 self._Scrub(revision, options)
mmoss@chromium.org50fd47f2014-02-13 01:03:19 +0000311 if file_list is not None:
Aaron Gable7817f022017-12-12 09:43:17 -0800312 files = self._Capture(
313 ['-c', 'core.quotePath=false', 'ls-files']).splitlines()
Raul Tambreb946b232019-03-26 14:48:46 +0000314 file_list.extend(
Edward Lemur26a8b9f2019-08-15 20:46:44 +0000315 [os.path.join(self.checkout_path, f) for f in files])
mmoss@chromium.org50fd47f2014-02-13 01:03:19 +0000316
szager@chromium.org8a139702014-06-20 15:55:01 +0000317 def _DisableHooks(self):
318 hook_dir = os.path.join(self.checkout_path, '.git', 'hooks')
319 if not os.path.isdir(hook_dir):
320 return
321 for f in os.listdir(hook_dir):
322 if not f.endswith('.sample') and not f.endswith('.disabled'):
primiano@chromium.org41265562015-04-08 09:14:46 +0000323 disabled_hook_path = os.path.join(hook_dir, f + '.disabled')
324 if os.path.exists(disabled_hook_path):
325 os.remove(disabled_hook_path)
326 os.rename(os.path.join(hook_dir, f), disabled_hook_path)
szager@chromium.org8a139702014-06-20 15:55:01 +0000327
iannucci@chromium.org30a07982016-04-07 21:35:19 +0000328 def _maybe_break_locks(self, options):
329 """This removes all .lock files from this repo's .git directory, if the
330 user passed the --break_repo_locks command line flag.
331
332 In particular, this will cleanup index.lock files, as well as ref lock
333 files.
334 """
335 if options.break_repo_locks:
336 git_dir = os.path.join(self.checkout_path, '.git')
337 for path, _, filenames in os.walk(git_dir):
338 for filename in filenames:
339 if filename.endswith('.lock'):
340 to_break = os.path.join(path, filename)
341 self.Print('breaking lock: %s' % (to_break,))
342 try:
343 os.remove(to_break)
344 except OSError as ex:
345 self.Print('FAILED to break lock: %s: %s' % (to_break, ex))
346 raise
347
Ravi Mistryc848a4e2022-03-10 18:19:59 +0000348 def _download_topics(self, patch_rev, googlesource_url):
349 """This method returns new patch_revs to process that have the same topic.
350
351 It does the following:
352 1. Finds the topic of the Gerrit change specified in the patch_rev.
353 2. Find all changes with that topic.
354 3. Append patch_rev of the changes with the same topic to the patch_revs
355 to process.
356 4. Returns the new patch_revs to process.
357 """
358 patch_revs_to_process = []
359 # Parse the patch_rev to extract the CL and patchset.
360 patch_rev_tokens = patch_rev.split('/')
361 change = patch_rev_tokens[-2]
362 # Parse the googlesource_url.
363 tokens = re.search(
364 '//(.+).googlesource.com/(.+?)(?:\.git)?$', googlesource_url)
365 if not tokens or len(tokens.groups()) != 2:
366 # googlesource_url is not in the expected format.
367 return patch_revs_to_process
368
369 # parse the gerrit host and repo out of googlesource_url.
370 host, repo = tokens.groups()[:2]
371 gerrit_host_url = '%s-review.googlesource.com' % host
372
373 # 1. Find the topic of the Gerrit change specified in the patch_rev.
374 change_object = gerrit_util.GetChange(gerrit_host_url, change)
375 topic = change_object.get('topic')
376 if not topic:
377 # This change has no topic set.
378 return patch_revs_to_process
379
380 # 2. Find all changes with that topic.
381 changes_with_same_topic = gerrit_util.QueryChanges(
382 gerrit_host_url,
383 [('topic', topic), ('status', 'open'), ('repo', repo)],
384 o_params=['ALL_REVISIONS'])
385 for c in changes_with_same_topic:
386 if str(c['_number']) == change:
387 # This change is already in the patch_rev.
388 continue
389 self.Print('Found CL %d with the topic name %s' % (
390 c['_number'], topic))
391 # 3. Append patch_rev of the changes with the same topic to the
392 # patch_revs to process.
393 curr_rev = c['current_revision']
394 new_patch_rev = c['revisions'][curr_rev]['ref']
395 patch_revs_to_process.append(new_patch_rev)
396
Joanna Wang1a977bd2022-06-02 21:51:17 +0000397 # 4. Return the new patch_revs to process.
Ravi Mistryc848a4e2022-03-10 18:19:59 +0000398 return patch_revs_to_process
399
Edward Lemur3acbc742019-05-30 17:57:35 +0000400 def apply_patch_ref(self, patch_repo, patch_rev, target_rev, options,
Edward Lemur6a4e31b2018-08-10 19:59:02 +0000401 file_list):
Joanna Wangf3edc502022-07-20 00:12:10 +0000402 # type: (str, str, str, optparse.Values, Collection[str]) -> str
Edward Lemur6a4e31b2018-08-10 19:59:02 +0000403 """Apply a patch on top of the revision we're synced at.
404
Edward Lemur3acbc742019-05-30 17:57:35 +0000405 The patch ref is given by |patch_repo|@|patch_rev|.
406 |target_rev| is usually the branch that the |patch_rev| was uploaded against
Josip Sokcevic9c0dc302020-11-20 18:41:25 +0000407 (e.g. 'refs/heads/main'), but this is not required.
Edward Lemur3acbc742019-05-30 17:57:35 +0000408
409 We cherry-pick all commits reachable from |patch_rev| on top of the curret
410 HEAD, excluding those reachable from |target_rev|
411 (i.e. git cherry-pick target_rev..patch_rev).
Edward Lemur6a4e31b2018-08-10 19:59:02 +0000412
413 Graphically, it looks like this:
414
Edward Lemur3acbc742019-05-30 17:57:35 +0000415 ... -> o -> [possibly already landed commits] -> target_rev
416 \
417 -> [possibly not yet landed dependent CLs] -> patch_rev
Edward Lemur6a4e31b2018-08-10 19:59:02 +0000418
Edward Lemur3acbc742019-05-30 17:57:35 +0000419 The final checkout state is then:
Edward Lemur6a4e31b2018-08-10 19:59:02 +0000420
Edward Lemur3acbc742019-05-30 17:57:35 +0000421 ... -> HEAD -> [possibly not yet landed dependent CLs] -> patch_rev
Edward Lemur6a4e31b2018-08-10 19:59:02 +0000422
423 After application, if |options.reset_patch_ref| is specified, we soft reset
Edward Lemur3acbc742019-05-30 17:57:35 +0000424 the cherry-picked changes, keeping them in git index only.
Edward Lemur6a4e31b2018-08-10 19:59:02 +0000425
426 Args:
Edward Lemur3acbc742019-05-30 17:57:35 +0000427 patch_repo: The patch origin.
428 e.g. 'https://foo.googlesource.com/bar'
429 patch_rev: The revision to patch.
430 e.g. 'refs/changes/1234/34/1'.
431 target_rev: The revision to use when finding the merge base.
432 Typically, the branch that the patch was uploaded against.
Josip Sokcevic9c0dc302020-11-20 18:41:25 +0000433 e.g. 'refs/heads/main' or 'refs/heads/infra/config'.
Edward Lemur6a4e31b2018-08-10 19:59:02 +0000434 options: The options passed to gclient.
435 file_list: A list where modified files will be appended.
436 """
437
Edward Lemurca7d8812018-07-24 17:42:45 +0000438 # Abort any cherry-picks in progress.
439 try:
440 self._Capture(['cherry-pick', '--abort'])
441 except subprocess2.CalledProcessError:
442 pass
443
Joanna Wangf3edc502022-07-20 00:12:10 +0000444 base_rev = self.revinfo(None, None, None)
Edward Lemura0ffbe42019-05-01 16:52:18 +0000445
Edward Lemur3acbc742019-05-30 17:57:35 +0000446 if not target_rev:
Edward Lemur4c5c8ab2019-06-07 15:58:13 +0000447 raise gclient_utils.Error('A target revision for the patch must be given')
Aravind Vasudevanc5f0cbb2022-01-24 23:56:57 +0000448
449 if target_rev.startswith(('refs/heads/', 'refs/branch-heads')):
Edward Lesmesf627d9f2020-07-23 19:50:50 +0000450 # If |target_rev| is in refs/heads/** or refs/branch-heads/**, try first
451 # to find the corresponding remote ref for it, since |target_rev| might
452 # point to a local ref which is not up to date with the corresponding
453 # remote ref.
Edward Lemur3acbc742019-05-30 17:57:35 +0000454 remote_ref = ''.join(scm.GIT.RefToRemoteRef(target_rev, self.remote))
Quinten Yearsley925cedb2020-04-13 17:49:39 +0000455 self.Print('Trying the corresponding remote ref for %r: %r\n' % (
Edward Lemur3acbc742019-05-30 17:57:35 +0000456 target_rev, remote_ref))
457 if scm.GIT.IsValidRevision(self.checkout_path, remote_ref):
Josip Sokcevic3d7cbce2021-10-05 20:48:04 +0000458 # refs/remotes may need to be updated to cleanly cherry-pick changes.
459 # See https://crbug.com/1255178.
460 self._Capture(['fetch', '--no-tags', self.remote, target_rev])
Edward Lemur3acbc742019-05-30 17:57:35 +0000461 target_rev = remote_ref
462 elif not scm.GIT.IsValidRevision(self.checkout_path, target_rev):
463 # Fetch |target_rev| if it's not already available.
464 url, _ = gclient_utils.SplitUrlRevision(self.url)
Edward Lesmes07a68342021-04-20 23:39:30 +0000465 mirror = self._GetMirror(url, options, target_rev, target_rev)
Edward Lemur3acbc742019-05-30 17:57:35 +0000466 if mirror:
467 rev_type = 'branch' if target_rev.startswith('refs/') else 'hash'
468 self._UpdateMirrorIfNotContains(mirror, options, rev_type, target_rev)
469 self._Fetch(options, refspec=target_rev)
Edward Lemura0ffbe42019-05-01 16:52:18 +0000470
Ravi Mistryecda7822022-02-28 16:22:20 +0000471 patch_revs_to_process = [patch_rev]
472
473 if hasattr(options, 'download_topics') and options.download_topics:
Ravi Mistryc848a4e2022-03-10 18:19:59 +0000474 patch_revs_to_process_from_topics = self._download_topics(
475 patch_rev, self.url)
476 patch_revs_to_process.extend(patch_revs_to_process_from_topics)
Ravi Mistryecda7822022-02-28 16:22:20 +0000477
Edward Lesmesc621b212018-03-21 20:26:56 -0400478 self._Capture(['reset', '--hard'])
Ravi Mistryecda7822022-02-28 16:22:20 +0000479 for pr in patch_revs_to_process:
480 self.Print('===Applying patch===')
481 self.Print('Revision to patch is %r @ %r.' % (patch_repo, pr))
482 self.Print('Current dir is %r' % self.checkout_path)
483 self._Capture(['fetch', '--no-tags', patch_repo, pr])
484 pr = self._Capture(['rev-parse', 'FETCH_HEAD'])
Edward Lesmesc621b212018-03-21 20:26:56 -0400485
Ravi Mistryecda7822022-02-28 16:22:20 +0000486 if not options.rebase_patch_ref:
487 self._Capture(['checkout', pr])
488 # Adjust base_rev to be the first parent of our checked out patch ref;
489 # This will allow us to correctly extend `file_list`, and will show the
490 # correct file-list to programs which do `git diff --cached` expecting
491 # to see the patch diff.
492 base_rev = self._Capture(['rev-parse', pr+'~'])
493 else:
494 self.Print('Will cherrypick %r .. %r on top of %r.' % (
495 target_rev, pr, base_rev))
Edward Lemur3acbc742019-05-30 17:57:35 +0000496 try:
Ravi Mistryecda7822022-02-28 16:22:20 +0000497 if scm.GIT.IsAncestor(self.checkout_path, pr, target_rev):
498 if len(patch_revs_to_process) > 1:
499 # If there are multiple patch_revs_to_process then we do not want
500 # want to invalidate a previous patch so throw an error.
501 raise gclient_utils.Error(
502 'patch_rev %s is an ancestor of target_rev %s. This '
503 'situation is unsupported when we need to apply multiple '
504 'patch_revs: %s' % (pr, target_rev, patch_revs_to_process))
505 # If |patch_rev| is an ancestor of |target_rev|, check it out.
506 self._Capture(['checkout', pr])
507 else:
508 # If a change was uploaded on top of another change, which has
509 # already landed, one of the commits in the cherry-pick range will
510 # be redundant, since it has already landed and its changes
511 # incorporated in the tree.
512 # We pass '--keep-redundant-commits' to ignore those changes.
513 self._Capture(['cherry-pick', target_rev + '..' + pr,
514 '--keep-redundant-commits'])
Edward Lemurca7d8812018-07-24 17:42:45 +0000515
Ravi Mistryecda7822022-02-28 16:22:20 +0000516 except subprocess2.CalledProcessError as e:
517 self.Print('Failed to apply patch.')
518 self.Print('Revision to patch was %r @ %r.' % (patch_repo, pr))
519 self.Print('Tried to cherrypick %r .. %r on top of %r.' % (
520 target_rev, pr, base_rev))
521 self.Print('Current dir is %r' % self.checkout_path)
522 self.Print('git returned non-zero exit status %s:\n%s' % (
523 e.returncode, e.stderr.decode('utf-8')))
524 # Print the current status so that developers know what changes caused
525 # the patch failure, since git cherry-pick doesn't show that
526 # information.
527 self.Print(self._Capture(['status']))
528 try:
529 self._Capture(['cherry-pick', '--abort'])
530 except subprocess2.CalledProcessError:
531 pass
532 raise
533
534 if file_list is not None:
535 file_list.extend(self._GetDiffFilenames(base_rev))
Edward Lemurca7d8812018-07-24 17:42:45 +0000536
Joanna Wangf3edc502022-07-20 00:12:10 +0000537 latest_commit = self.revinfo(None, None, None)
Edward Lesmesc621b212018-03-21 20:26:56 -0400538 if options.reset_patch_ref:
539 self._Capture(['reset', '--soft', base_rev])
Joanna Wangf3edc502022-07-20 00:12:10 +0000540 return latest_commit
Edward Lesmesc621b212018-03-21 20:26:56 -0400541
Joanna Wang5a7c8242022-07-01 19:09:00 +0000542 def check_diff(self, previous_commit, files=None):
543 # type: (str, Optional[List[str]]) -> bool
544 """Check if a diff exists between the current commit and `previous_commit`.
545
546 Returns True if there were diffs or if an error was encountered.
547 """
548 cmd = ['diff', previous_commit, '--quiet']
549 if files:
550 cmd += ['--'] + files
551 try:
552 self._Capture(cmd)
553 return False
554 except subprocess2.CalledProcessError as e:
555 # git diff --quiet exits with 1 if there were diffs.
556 if e.returncode != 1:
557 self.Print('git returned non-zero exit status %s:\n%s' %
558 (e.returncode, e.stderr.decode('utf-8')))
559 return True
560
msb@chromium.orge28e4982009-09-25 20:51:45 +0000561 def update(self, options, args, file_list):
562 """Runs git to update or transparently checkout the working copy.
563
564 All updated files will be appended to file_list.
565
566 Raises:
567 Error: if can't get URL for relative path.
568 """
msb@chromium.orge28e4982009-09-25 20:51:45 +0000569 if args:
570 raise gclient_utils.Error("Unsupported argument(s): %s" % ",".join(args))
571
nasser@codeaurora.orgece406f2010-02-23 17:29:15 +0000572 self._CheckMinVersion("1.6.6")
msb@chromium.org923a0372009-12-11 20:42:43 +0000573
nasser@codeaurora.org7080e942010-03-15 15:06:16 +0000574 url, deps_revision = gclient_utils.SplitUrlRevision(self.url)
nasser@codeaurora.org7080e942010-03-15 15:06:16 +0000575 revision = deps_revision
cmp@chromium.orgeb2756d2011-09-20 20:17:51 +0000576 managed = True
msb@chromium.orge28e4982009-09-25 20:51:45 +0000577 if options.revision:
msb@chromium.orgac915bb2009-11-13 17:03:01 +0000578 # Override the revision number.
579 revision = str(options.revision)
cmp@chromium.orgeb2756d2011-09-20 20:17:51 +0000580 if revision == 'unmanaged':
romain.pokrzywka@gmail.com483a0ba2014-05-30 00:06:07 +0000581 # Check again for a revision in case an initial ref was specified
582 # in the url, for example bla.git@refs/heads/custombranch
583 revision = deps_revision
cmp@chromium.orgeb2756d2011-09-20 20:17:51 +0000584 managed = False
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000585 if not revision:
Josip Sokcevic091f5ac2021-01-14 23:14:21 +0000586 # If a dependency is not pinned, track the default remote branch.
587 revision = scm.GIT.GetRemoteHeadRef(self.checkout_path, self.url,
588 self.remote)
Edward Lesmes4ea67bb2021-04-20 17:33:52 +0000589 if revision.startswith('origin/'):
590 revision = 'refs/remotes/' + revision
msb@chromium.orge28e4982009-09-25 20:51:45 +0000591
Tomasz Wiszkowskid4e66882021-08-19 21:35:09 +0000592 if managed and platform.system() == 'Windows':
szager@chromium.org8a139702014-06-20 15:55:01 +0000593 self._DisableHooks()
594
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000595 printed_path = False
596 verbose = []
msb@chromium.orgb1a22bf2009-11-07 02:33:50 +0000597 if options.verbose:
agable83faed02016-10-24 14:37:10 -0700598 self.Print('_____ %s at %s' % (self.relpath, revision), timestamp=False)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000599 verbose = ['--verbose']
600 printed_path = True
601
Edward Lemurbdbe07f2019-03-28 22:14:45 +0000602 revision_ref = revision
603 if ':' in revision:
604 revision_ref, _, revision = revision.partition(':')
605
Edward Lesmes8073a502020-04-15 02:11:14 +0000606 if revision_ref.startswith('refs/branch-heads'):
607 options.with_branch_heads = True
608
Edward Lesmes07a68342021-04-20 23:39:30 +0000609 mirror = self._GetMirror(url, options, revision, revision_ref)
Edward Lemurbdbe07f2019-03-28 22:14:45 +0000610 if mirror:
611 url = mirror.mirror_path
Edward Lemurdb5c5ad2019-03-07 16:00:24 +0000612
John Budorick882c91e2018-07-12 22:11:41 +0000613 remote_ref = scm.GIT.RefToRemoteRef(revision, self.remote)
614 if remote_ref:
615 # Rewrite remote refs to their local equivalents.
616 revision = ''.join(remote_ref)
617 rev_type = "branch"
618 elif revision.startswith('refs/'):
619 # Local branch? We probably don't want to support, since DEPS should
620 # always specify branches as they are in the upstream repo.
621 rev_type = "branch"
622 else:
623 # hash is also a tag, only make a distinction at checkout
624 rev_type = "hash"
smut@google.comd33eab32014-07-07 19:35:18 +0000625
primiano@chromium.org1c127382015-02-17 11:15:40 +0000626 # If we are going to introduce a new project, there is a possibility that
627 # we are syncing back to a state where the project was originally a
628 # sub-project rolled by DEPS (realistic case: crossing the Blink merge point
629 # syncing backwards, when Blink was a DEPS entry and not part of src.git).
630 # In such case, we might have a backup of the former .git folder, which can
631 # be used to avoid re-fetching the entire repo again (useful for bisects).
632 backup_dir = self.GetGitBackupDirPath()
633 target_dir = os.path.join(self.checkout_path, '.git')
634 if os.path.exists(backup_dir) and not os.path.exists(target_dir):
635 gclient_utils.safe_makedirs(self.checkout_path)
636 os.rename(backup_dir, target_dir)
637 # Reset to a clean state
Robert Iannuccic41d8b92017-02-16 17:07:37 -0800638 self._Scrub('HEAD', options)
primiano@chromium.org1c127382015-02-17 11:15:40 +0000639
szager@chromium.org6c2b49d2014-02-26 23:57:38 +0000640 if (not os.path.exists(self.checkout_path) or
641 (os.path.isdir(self.checkout_path) and
642 not os.path.exists(os.path.join(self.checkout_path, '.git')))):
szager@chromium.orgb0a13a22014-06-18 00:52:25 +0000643 if mirror:
John Budorick882c91e2018-07-12 22:11:41 +0000644 self._UpdateMirrorIfNotContains(mirror, options, rev_type, revision)
borenet@google.com90fe58b2014-05-01 18:22:00 +0000645 try:
John Budorick882c91e2018-07-12 22:11:41 +0000646 self._Clone(revision, url, options)
Joanna Wang47fd5672022-08-05 20:53:31 +0000647 except subprocess2.CalledProcessError as e:
648 logging.warning('Clone failed due to: %s', e)
borenet@google.com90fe58b2014-05-01 18:22:00 +0000649 self._DeleteOrMove(options.force)
John Budorick882c91e2018-07-12 22:11:41 +0000650 self._Clone(revision, url, options)
iannucci@chromium.org396e1a62013-07-03 19:41:04 +0000651 if file_list is not None:
Aaron Gable7817f022017-12-12 09:43:17 -0800652 files = self._Capture(
John Budorick21a51b32018-09-19 19:39:20 +0000653 ['-c', 'core.quotePath=false', 'ls-files']).splitlines()
Raul Tambrec2f74c12019-03-19 05:55:53 +0000654 file_list.extend(
Edward Lemur979fa782019-08-13 22:44:05 +0000655 [os.path.join(self.checkout_path, f) for f in files])
John Budorick21a51b32018-09-19 19:39:20 +0000656 if mirror:
657 self._Capture(
658 ['remote', 'set-url', '--push', 'origin', mirror.url])
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000659 if not verbose:
660 # Make the output a little prettier. It's nice to have some whitespace
661 # between projects when cloning.
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000662 self.Print('')
iannucci@chromium.org2702bcd2013-09-24 19:10:07 +0000663 return self._Capture(['rev-parse', '--verify', 'HEAD'])
msb@chromium.orge28e4982009-09-25 20:51:45 +0000664
John Budorick21a51b32018-09-19 19:39:20 +0000665 if mirror:
666 self._Capture(
667 ['remote', 'set-url', '--push', 'origin', mirror.url])
668
szager@chromium.org3dc5cb72014-06-17 15:06:05 +0000669 if not managed:
Edward Lemur579c9862018-07-13 23:17:51 +0000670 self._SetFetchConfig(options)
szager@chromium.org3dc5cb72014-06-17 15:06:05 +0000671 self.Print('________ unmanaged solution; skipping %s' % self.relpath)
672 return self._Capture(['rev-parse', '--verify', 'HEAD'])
673
iannucci@chromium.org30a07982016-04-07 21:35:19 +0000674 self._maybe_break_locks(options)
675
szager@chromium.orgb0a13a22014-06-18 00:52:25 +0000676 if mirror:
John Budorick882c91e2018-07-12 22:11:41 +0000677 self._UpdateMirrorIfNotContains(mirror, options, rev_type, revision)
szager@chromium.orgb0a13a22014-06-18 00:52:25 +0000678
szager@chromium.org6c2b49d2014-02-26 23:57:38 +0000679 # See if the url has changed (the unittests use git://foo for the url, let
680 # that through).
681 current_url = self._Capture(['config', 'remote.%s.url' % self.remote])
682 return_early = False
683 # TODO(maruel): Delete url != 'git://foo' since it's just to make the
684 # unit test pass. (and update the comment above)
685 # Skip url auto-correction if remote.origin.gclient-auto-fix-url is set.
686 # This allows devs to use experimental repos which have a different url
687 # but whose branch(s) are the same as official repos.
Raul Tambrecd862e32019-05-10 21:19:00 +0000688 if (current_url.rstrip('/') != url.rstrip('/') and url != 'git://foo' and
szager@chromium.org6c2b49d2014-02-26 23:57:38 +0000689 subprocess2.capture(
Aaron Gableac9b0f32019-04-18 17:38:37 +0000690 ['git', 'config', 'remote.%s.gclient-auto-fix-url' % self.remote],
szager@chromium.org6c2b49d2014-02-26 23:57:38 +0000691 cwd=self.checkout_path).strip() != 'False'):
Anthony Polito486f1812020-08-04 23:40:33 +0000692 self.Print('_____ switching %s from %s to new upstream %s' % (
693 self.relpath, current_url, url))
iannucci@chromium.org78514212014-08-20 23:08:00 +0000694 if not (options.force or options.reset):
695 # Make sure it's clean
agable83faed02016-10-24 14:37:10 -0700696 self._CheckClean(revision)
szager@chromium.org6c2b49d2014-02-26 23:57:38 +0000697 # Switch over to the new upstream
698 self._Run(['remote', 'set-url', self.remote, url], options)
szager@chromium.org8dd35462015-06-08 22:56:05 +0000699 if mirror:
700 with open(os.path.join(
701 self.checkout_path, '.git', 'objects', 'info', 'alternates'),
702 'w') as fh:
703 fh.write(os.path.join(url, 'objects'))
John Budorick882c91e2018-07-12 22:11:41 +0000704 self._EnsureValidHeadObjectOrCheckout(revision, options, url)
705 self._FetchAndReset(revision, file_list, options)
tandrii@chromium.orgc438c142015-08-24 22:55:55 +0000706
szager@chromium.org6c2b49d2014-02-26 23:57:38 +0000707 return_early = True
tandrii@chromium.orgc438c142015-08-24 22:55:55 +0000708 else:
John Budorick882c91e2018-07-12 22:11:41 +0000709 self._EnsureValidHeadObjectOrCheckout(revision, options, url)
szager@chromium.org6c2b49d2014-02-26 23:57:38 +0000710
mmoss@chromium.org50fd47f2014-02-13 01:03:19 +0000711 if return_early:
712 return self._Capture(['rev-parse', '--verify', 'HEAD'])
713
msb@chromium.org5bde4852009-12-14 16:47:12 +0000714 cur_branch = self._GetCurrentBranch()
715
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000716 # Cases:
msb@chromium.org786fb682010-06-02 15:16:23 +0000717 # 0) HEAD is detached. Probably from our initial clone.
718 # - make sure HEAD is contained by a named ref, then update.
719 # Cases 1-4. HEAD is a branch.
agable41e3a6c2016-10-20 11:36:56 -0700720 # 1) current branch is not tracking a remote branch
msb@chromium.org786fb682010-06-02 15:16:23 +0000721 # - try to rebase onto the new hash or branch
722 # 2) current branch is tracking a remote branch with local committed
723 # changes, but the DEPS file switched to point to a hash
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000724 # - rebase those changes on top of the hash
mmoss@chromium.org6e7202b2014-09-09 18:23:39 +0000725 # 3) current branch is tracking a remote branch w/or w/out changes, and
726 # no DEPS switch
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000727 # - see if we can FF, if not, prompt the user for rebase, merge, or stop
mmoss@chromium.org6e7202b2014-09-09 18:23:39 +0000728 # 4) current branch is tracking a remote branch, but DEPS switches to a
729 # different remote branch, and
730 # a) current branch has no local changes, and --force:
731 # - checkout new branch
732 # b) current branch has local changes, and --force and --reset:
733 # - checkout new branch
734 # c) otherwise exit
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000735
Josip Sokcevic9c0dc302020-11-20 18:41:25 +0000736 # GetUpstreamBranch returns something like 'refs/remotes/origin/main' for
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000737 # a tracking branch
Josip Sokcevic9c0dc302020-11-20 18:41:25 +0000738 # or 'main' if not a tracking branch (it's based on a specific rev/hash)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000739 # or it returns None if it couldn't find an upstream
msb@chromium.org786fb682010-06-02 15:16:23 +0000740 if cur_branch is None:
741 upstream_branch = None
742 current_type = "detached"
743 logging.debug("Detached HEAD")
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000744 else:
msb@chromium.org786fb682010-06-02 15:16:23 +0000745 upstream_branch = scm.GIT.GetUpstreamBranch(self.checkout_path)
746 if not upstream_branch or not upstream_branch.startswith('refs/remotes'):
747 current_type = "hash"
748 logging.debug("Current branch is not tracking an upstream (remote)"
749 " branch.")
750 elif upstream_branch.startswith('refs/remotes'):
751 current_type = "branch"
752 else:
753 raise gclient_utils.Error('Invalid Upstream: %s' % upstream_branch)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000754
Edward Lemur579c9862018-07-13 23:17:51 +0000755 self._SetFetchConfig(options)
Edward Lemur579c9862018-07-13 23:17:51 +0000756
Michael Spang73fac912019-03-08 18:44:19 +0000757 # Fetch upstream if we don't already have |revision|.
John Budorick882c91e2018-07-12 22:11:41 +0000758 if not scm.GIT.IsValidRevision(self.checkout_path, revision, sha_only=True):
Michael Spang73fac912019-03-08 18:44:19 +0000759 self._Fetch(options, prune=options.force)
760
761 if not scm.GIT.IsValidRevision(self.checkout_path, revision,
762 sha_only=True):
763 # Update the remotes first so we have all the refs.
764 remote_output = scm.GIT.Capture(['remote'] + verbose + ['update'],
765 cwd=self.checkout_path)
766 if verbose:
767 self.Print(remote_output)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000768
John Budorick882c91e2018-07-12 22:11:41 +0000769 revision = self._AutoFetchRef(options, revision)
Paweł Hajdan, Jr63b8c2a2017-09-05 17:59:08 +0200770
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000771 # This is a big hammer, debatable if it should even be here...
davemoore@chromium.org793796d2010-02-19 17:27:41 +0000772 if options.force or options.reset:
iannucci@chromium.orgd4fffee2013-06-28 00:35:26 +0000773 target = 'HEAD'
774 if options.upstream and upstream_branch:
775 target = upstream_branch
Robert Iannuccic41d8b92017-02-16 17:07:37 -0800776 self._Scrub(target, options)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000777
msb@chromium.org786fb682010-06-02 15:16:23 +0000778 if current_type == 'detached':
779 # case 0
Robert Iannuccic41d8b92017-02-16 17:07:37 -0800780 # We just did a Scrub, this is as clean as it's going to get. In
781 # particular if HEAD is a commit that contains two versions of the same
782 # file on a case-insensitive filesystem (e.g. 'a' and 'A'), there's no way
783 # to actually "Clean" the checkout; that commit is uncheckoutable on this
784 # system. The best we can do is carry forward to the checkout step.
785 if not (options.force or options.reset):
John Budorick882c91e2018-07-12 22:11:41 +0000786 self._CheckClean(revision)
787 self._CheckDetachedHead(revision, options)
smut@google.comd33eab32014-07-07 19:35:18 +0000788 if self._Capture(['rev-list', '-n', '1', 'HEAD']) == revision:
szager@chromium.org848fd492014-04-09 19:06:44 +0000789 self.Print('Up-to-date; skipping checkout.')
790 else:
vadimsh@chromium.org2b7d3ed2014-06-20 18:15:37 +0000791 # 'git checkout' may need to overwrite existing untracked files. Allow
792 # it only when nuclear options are enabled.
dnj@chromium.orgbb424c02014-06-23 22:42:51 +0000793 self._Checkout(
794 options,
John Budorick882c91e2018-07-12 22:11:41 +0000795 revision,
smut@google.com34b4e982016-05-16 19:06:07 +0000796 force=(options.force and options.delete_unversioned_trees),
dnj@chromium.orgbb424c02014-06-23 22:42:51 +0000797 quiet=True,
798 )
msb@chromium.org786fb682010-06-02 15:16:23 +0000799 if not printed_path:
John Budorick882c91e2018-07-12 22:11:41 +0000800 self.Print('_____ %s at %s' % (self.relpath, revision), timestamp=False)
msb@chromium.org786fb682010-06-02 15:16:23 +0000801 elif current_type == 'hash':
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000802 # case 1
agable41e3a6c2016-10-20 11:36:56 -0700803 # Can't find a merge-base since we don't know our upstream. That makes
804 # this command VERY likely to produce a rebase failure. For now we
805 # assume origin is our upstream since that's what the old behavior was.
John Budorick882c91e2018-07-12 22:11:41 +0000806 upstream_branch = self.remote
807 if options.revision or deps_revision:
808 upstream_branch = revision
agable1a8439a2016-10-24 16:36:14 -0700809 self._AttemptRebase(upstream_branch, file_list, options,
agable41e3a6c2016-10-20 11:36:56 -0700810 printed_path=printed_path, merge=options.merge)
811 printed_path = True
John Budorick882c91e2018-07-12 22:11:41 +0000812 elif rev_type == 'hash':
813 # case 2
814 self._AttemptRebase(upstream_branch, file_list, options,
815 newbase=revision, printed_path=printed_path,
816 merge=options.merge)
817 printed_path = True
818 elif remote_ref and ''.join(remote_ref) != upstream_branch:
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000819 # case 4
John Budorick882c91e2018-07-12 22:11:41 +0000820 new_base = ''.join(remote_ref)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000821 if not printed_path:
John Budorick882c91e2018-07-12 22:11:41 +0000822 self.Print('_____ %s at %s' % (self.relpath, revision), timestamp=False)
mmoss@chromium.org6e7202b2014-09-09 18:23:39 +0000823 switch_error = ("Could not switch upstream branch from %s to %s\n"
John Budorick882c91e2018-07-12 22:11:41 +0000824 % (upstream_branch, new_base) +
mmoss@chromium.org6e7202b2014-09-09 18:23:39 +0000825 "Please use --force or merge or rebase manually:\n" +
John Budorick882c91e2018-07-12 22:11:41 +0000826 "cd %s; git rebase %s\n" % (self.checkout_path, new_base) +
827 "OR git checkout -b <some new branch> %s" % new_base)
mmoss@chromium.org6e7202b2014-09-09 18:23:39 +0000828 force_switch = False
829 if options.force:
830 try:
John Budorick882c91e2018-07-12 22:11:41 +0000831 self._CheckClean(revision)
mmoss@chromium.org6e7202b2014-09-09 18:23:39 +0000832 # case 4a
833 force_switch = True
834 except gclient_utils.Error as e:
835 if options.reset:
836 # case 4b
837 force_switch = True
838 else:
839 switch_error = '%s\n%s' % (e.message, switch_error)
840 if force_switch:
841 self.Print("Switching upstream branch from %s to %s" %
John Budorick882c91e2018-07-12 22:11:41 +0000842 (upstream_branch, new_base))
843 switch_branch = 'gclient_' + remote_ref[1]
844 self._Capture(['branch', '-f', switch_branch, new_base])
mmoss@chromium.org6e7202b2014-09-09 18:23:39 +0000845 self._Checkout(options, switch_branch, force=True, quiet=True)
846 else:
847 # case 4c
848 raise gclient_utils.Error(switch_error)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000849 else:
850 # case 3 - the default case
Aaron Gablef4068aa2017-12-12 15:14:09 -0800851 rebase_files = self._GetDiffFilenames(upstream_branch)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000852 if verbose:
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000853 self.Print('Trying fast-forward merge to branch : %s' % upstream_branch)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000854 try:
bauerb@chromium.org2aad1b22011-07-22 12:00:41 +0000855 merge_args = ['merge']
bauerb@chromium.org30c46d62014-01-23 12:11:56 +0000856 if options.merge:
857 merge_args.append('--ff')
858 else:
bauerb@chromium.org2aad1b22011-07-22 12:00:41 +0000859 merge_args.append('--ff-only')
860 merge_args.append(upstream_branch)
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000861 merge_output = self._Capture(merge_args)
bratell@opera.com18fa4542013-05-21 13:30:46 +0000862 except subprocess2.CalledProcessError as e:
agable1a8439a2016-10-24 16:36:14 -0700863 rebase_files = []
Edward Lemur979fa782019-08-13 22:44:05 +0000864 if re.match(b'fatal: Not possible to fast-forward, aborting.',
865 e.stderr):
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000866 if not printed_path:
John Budorick882c91e2018-07-12 22:11:41 +0000867 self.Print('_____ %s at %s' % (self.relpath, revision),
agable83faed02016-10-24 14:37:10 -0700868 timestamp=False)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000869 printed_path = True
870 while True:
dnj@chromium.org5b23e872015-02-20 21:25:57 +0000871 if not options.auto_rebase:
872 try:
873 action = self._AskForData(
874 'Cannot %s, attempt to rebase? '
875 '(y)es / (q)uit / (s)kip : ' %
876 ('merge' if options.merge else 'fast-forward merge'),
877 options)
878 except ValueError:
879 raise gclient_utils.Error('Invalid Character')
880 if options.auto_rebase or re.match(r'yes|y', action, re.I):
agable1a8439a2016-10-24 16:36:14 -0700881 self._AttemptRebase(upstream_branch, rebase_files, options,
bauerb@chromium.org30c46d62014-01-23 12:11:56 +0000882 printed_path=printed_path, merge=False)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000883 printed_path = True
884 break
Aravind Vasudevanc5f0cbb2022-01-24 23:56:57 +0000885
886 if re.match(r'quit|q', action, re.I):
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000887 raise gclient_utils.Error("Can't fast-forward, please merge or "
888 "rebase manually.\n"
889 "cd %s && git " % self.checkout_path
890 + "rebase %s" % upstream_branch)
Aravind Vasudevanc5f0cbb2022-01-24 23:56:57 +0000891
892 if re.match(r'skip|s', action, re.I):
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000893 self.Print('Skipping %s' % self.relpath)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000894 return
Aravind Vasudevanc5f0cbb2022-01-24 23:56:57 +0000895
896 self.Print('Input not recognized')
Edward Lemur979fa782019-08-13 22:44:05 +0000897 elif re.match(b"error: Your local changes to '.*' would be "
898 b"overwritten by merge. Aborting.\nPlease, commit your "
899 b"changes or stash them before you can merge.\n",
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000900 e.stderr):
901 if not printed_path:
John Budorick882c91e2018-07-12 22:11:41 +0000902 self.Print('_____ %s at %s' % (self.relpath, revision),
agable83faed02016-10-24 14:37:10 -0700903 timestamp=False)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000904 printed_path = True
Edward Lemur979fa782019-08-13 22:44:05 +0000905 raise gclient_utils.Error(e.stderr.decode('utf-8'))
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000906 else:
907 # Some other problem happened with the merge
908 logging.error("Error during fast-forward merge in %s!" % self.relpath)
Edward Lemur979fa782019-08-13 22:44:05 +0000909 self.Print(e.stderr.decode('utf-8'))
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000910 raise
911 else:
912 # Fast-forward merge was successful
913 if not re.match('Already up-to-date.', merge_output) or verbose:
914 if not printed_path:
agable83faed02016-10-24 14:37:10 -0700915 self.Print('_____ %s at %s' % (self.relpath, revision),
916 timestamp=False)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000917 printed_path = True
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000918 self.Print(merge_output.strip())
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000919 if not verbose:
920 # Make the output a little prettier. It's nice to have some
921 # whitespace between projects when syncing.
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000922 self.Print('')
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000923
agablec3937b92016-10-25 10:13:03 -0700924 if file_list is not None:
925 file_list.extend(
926 [os.path.join(self.checkout_path, f) for f in rebase_files])
msb@chromium.org5bde4852009-12-14 16:47:12 +0000927
928 # If the rebase generated a conflict, abort and ask user to fix
msb@chromium.org786fb682010-06-02 15:16:23 +0000929 if self._IsRebasing():
agable83faed02016-10-24 14:37:10 -0700930 raise gclient_utils.Error('\n____ %s at %s\n'
msb@chromium.org5bde4852009-12-14 16:47:12 +0000931 '\nConflict while rebasing this branch.\n'
932 'Fix the conflict and run gclient again.\n'
933 'See man git-rebase for details.\n'
agable83faed02016-10-24 14:37:10 -0700934 % (self.relpath, revision))
msb@chromium.org5bde4852009-12-14 16:47:12 +0000935
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +0000936 if verbose:
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000937 self.Print('Checked out revision %s' % self.revinfo(options, (), None),
938 timestamp=False)
msb@chromium.orge28e4982009-09-25 20:51:45 +0000939
steveblock@chromium.org98e69452012-02-16 16:36:43 +0000940 # If --reset and --delete_unversioned_trees are specified, remove any
941 # untracked directories.
942 if options.reset and options.delete_unversioned_trees:
943 # GIT.CaptureStatus() uses 'dit diff' to compare to a specific SHA1 (the
944 # merge-base by default), so doesn't include untracked files. So we use
945 # 'git ls-files --directory --others --exclude-standard' here directly.
946 paths = scm.GIT.Capture(
Aaron Gable7817f022017-12-12 09:43:17 -0800947 ['-c', 'core.quotePath=false', 'ls-files',
948 '--directory', '--others', '--exclude-standard'],
steveblock@chromium.org98e69452012-02-16 16:36:43 +0000949 self.checkout_path)
950 for path in (p for p in paths.splitlines() if p.endswith('/')):
951 full_path = os.path.join(self.checkout_path, path)
952 if not os.path.islink(full_path):
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000953 self.Print('_____ removing unversioned directory %s' % path)
digit@chromium.orgdc112ac2013-04-24 13:00:19 +0000954 gclient_utils.rmtree(full_path)
steveblock@chromium.org98e69452012-02-16 16:36:43 +0000955
iannucci@chromium.org2702bcd2013-09-24 19:10:07 +0000956 return self._Capture(['rev-parse', '--verify', 'HEAD'])
957
iannucci@chromium.org396e1a62013-07-03 19:41:04 +0000958 def revert(self, options, _args, file_list):
msb@chromium.orge28e4982009-09-25 20:51:45 +0000959 """Reverts local modifications.
960
961 All reverted files will be appended to file_list.
962 """
maruel@chromium.org8469bf92010-09-03 19:03:15 +0000963 if not os.path.isdir(self.checkout_path):
msb@chromium.org260c6532009-10-28 03:22:35 +0000964 # revert won't work if the directory doesn't exist. It needs to
965 # checkout instead.
Quinten Yearsley925cedb2020-04-13 17:49:39 +0000966 self.Print('_____ %s is missing, syncing instead' % self.relpath)
msb@chromium.org260c6532009-10-28 03:22:35 +0000967 # Don't reuse the args.
968 return self.update(options, [], file_list)
nasser@codeaurora.orgb2b46312010-04-30 20:58:03 +0000969
Josip Sokcevic7e133ff2021-07-13 17:44:53 +0000970 default_rev = "refs/heads/main"
iannucci@chromium.orgd4fffee2013-06-28 00:35:26 +0000971 if options.upstream:
972 if self._GetCurrentBranch():
973 upstream_branch = scm.GIT.GetUpstreamBranch(self.checkout_path)
974 default_rev = upstream_branch or default_rev
maruel@chromium.org6e29d572010-06-04 17:32:20 +0000975 _, deps_revision = gclient_utils.SplitUrlRevision(self.url)
nasser@codeaurora.orgb2b46312010-04-30 20:58:03 +0000976 if not deps_revision:
977 deps_revision = default_rev
smut@google.comd33eab32014-07-07 19:35:18 +0000978 if deps_revision.startswith('refs/heads/'):
979 deps_revision = deps_revision.replace('refs/heads/', self.remote + '/')
smutae7ea312016-07-18 11:59:41 -0700980 try:
981 deps_revision = self.GetUsableRev(deps_revision, options)
982 except NoUsableRevError as e:
983 # If the DEPS entry's url and hash changed, try to update the origin.
984 # See also http://crbug.com/520067.
John Budorickd94f8ea2020-03-27 15:55:24 +0000985 logging.warning(
986 "Couldn't find usable revision, will retrying to update instead: %s",
smutae7ea312016-07-18 11:59:41 -0700987 e.message)
988 return self.update(options, [], file_list)
nasser@codeaurora.orgb2b46312010-04-30 20:58:03 +0000989
iannucci@chromium.org396e1a62013-07-03 19:41:04 +0000990 if file_list is not None:
Aaron Gablef4068aa2017-12-12 15:14:09 -0800991 files = self._GetDiffFilenames(deps_revision)
iannucci@chromium.org396e1a62013-07-03 19:41:04 +0000992
Robert Iannuccic41d8b92017-02-16 17:07:37 -0800993 self._Scrub(deps_revision, options)
lliabraa@chromium.orgade83db2012-09-27 14:06:49 +0000994 self._Run(['clean', '-f', '-d'], options)
msb@chromium.orge28e4982009-09-25 20:51:45 +0000995
iannucci@chromium.org396e1a62013-07-03 19:41:04 +0000996 if file_list is not None:
997 file_list.extend([os.path.join(self.checkout_path, f) for f in files])
998
999 def revinfo(self, _options, _args, _file_list):
maruel@chromium.org6cafa132010-09-07 14:17:26 +00001000 """Returns revision"""
1001 return self._Capture(['rev-parse', 'HEAD'])
msb@chromium.org0f282062009-11-06 20:14:02 +00001002
msb@chromium.orge28e4982009-09-25 20:51:45 +00001003 def runhooks(self, options, args, file_list):
1004 self.status(options, args, file_list)
1005
iannucci@chromium.org396e1a62013-07-03 19:41:04 +00001006 def status(self, options, _args, file_list):
msb@chromium.orge28e4982009-09-25 20:51:45 +00001007 """Display status information."""
1008 if not os.path.isdir(self.checkout_path):
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001009 self.Print('________ couldn\'t run status in %s:\n'
1010 'The directory does not exist.' % self.checkout_path)
msb@chromium.orge28e4982009-09-25 20:51:45 +00001011 else:
Anthony Politobb457342019-11-15 22:26:01 +00001012 merge_base = []
1013 if self.url:
1014 _, base_rev = gclient_utils.SplitUrlRevision(self.url)
1015 if base_rev:
1016 merge_base = [base_rev]
Aaron Gablef4068aa2017-12-12 15:14:09 -08001017 self._Run(
1018 ['-c', 'core.quotePath=false', 'diff', '--name-status'] + merge_base,
Edward Lemur24146be2019-08-01 21:44:52 +00001019 options, always_show_header=options.verbose)
iannucci@chromium.org396e1a62013-07-03 19:41:04 +00001020 if file_list is not None:
Aaron Gablef4068aa2017-12-12 15:14:09 -08001021 files = self._GetDiffFilenames(merge_base[0] if merge_base else None)
iannucci@chromium.org396e1a62013-07-03 19:41:04 +00001022 file_list.extend([os.path.join(self.checkout_path, f) for f in files])
msb@chromium.orge28e4982009-09-25 20:51:45 +00001023
smutae7ea312016-07-18 11:59:41 -07001024 def GetUsableRev(self, rev, options):
agable41e3a6c2016-10-20 11:36:56 -07001025 """Finds a useful revision for this repository."""
smutae7ea312016-07-18 11:59:41 -07001026 sha1 = None
1027 if not os.path.isdir(self.checkout_path):
1028 raise NoUsableRevError(
agablea98a6cd2016-11-15 14:30:10 -08001029 'This is not a git repo, so we cannot get a usable rev.')
agable41e3a6c2016-10-20 11:36:56 -07001030
1031 if scm.GIT.IsValidRevision(cwd=self.checkout_path, rev=rev):
1032 sha1 = rev
smutae7ea312016-07-18 11:59:41 -07001033 else:
agable41e3a6c2016-10-20 11:36:56 -07001034 # May exist in origin, but we don't have it yet, so fetch and look
1035 # again.
1036 self._Fetch(options)
smutae7ea312016-07-18 11:59:41 -07001037 if scm.GIT.IsValidRevision(cwd=self.checkout_path, rev=rev):
1038 sha1 = rev
smutae7ea312016-07-18 11:59:41 -07001039
1040 if not sha1:
1041 raise NoUsableRevError(
agablea98a6cd2016-11-15 14:30:10 -08001042 'Hash %s does not appear to be a valid hash in this repo.' % rev)
smutae7ea312016-07-18 11:59:41 -07001043
1044 return sha1
1045
primiano@chromium.org1c127382015-02-17 11:15:40 +00001046 def GetGitBackupDirPath(self):
1047 """Returns the path where the .git folder for the current project can be
1048 staged/restored. Use case: subproject moved from DEPS <-> outer project."""
1049 return os.path.join(self._root_dir,
1050 'old_' + self.relpath.replace(os.sep, '_')) + '.git'
1051
Edward Lesmes07a68342021-04-20 23:39:30 +00001052 def _GetMirror(self, url, options, revision=None, revision_ref=None):
szager@chromium.orgb0a13a22014-06-18 00:52:25 +00001053 """Get a git_cache.Mirror object for the argument url."""
Robert Iannuccia19649b2018-06-29 16:31:45 +00001054 if not self.cache_dir:
szager@chromium.orgb0a13a22014-06-18 00:52:25 +00001055 return None
hinoka@google.comb1b54572014-04-16 22:29:23 +00001056 mirror_kwargs = {
1057 'print_func': self.filter,
Edward Lesmes07a68342021-04-20 23:39:30 +00001058 'refs': [],
1059 'commits': [],
hinoka@google.comb1b54572014-04-16 22:29:23 +00001060 }
hinoka@google.comb1b54572014-04-16 22:29:23 +00001061 if hasattr(options, 'with_branch_heads') and options.with_branch_heads:
1062 mirror_kwargs['refs'].append('refs/branch-heads/*')
Edward Lemurbdbe07f2019-03-28 22:14:45 +00001063 elif revision_ref and revision_ref.startswith('refs/branch-heads/'):
1064 mirror_kwargs['refs'].append(revision_ref)
szager@chromium.org8d3348f2014-08-19 22:49:16 +00001065 if hasattr(options, 'with_tags') and options.with_tags:
1066 mirror_kwargs['refs'].append('refs/tags/*')
Edward Lemurbdbe07f2019-03-28 22:14:45 +00001067 elif revision_ref and revision_ref.startswith('refs/tags/'):
1068 mirror_kwargs['refs'].append(revision_ref)
Edward Lesmes07a68342021-04-20 23:39:30 +00001069 if revision and not revision.startswith('refs/'):
1070 mirror_kwargs['commits'].append(revision)
szager@chromium.orgb0a13a22014-06-18 00:52:25 +00001071 return git_cache.Mirror(url, **mirror_kwargs)
1072
John Budorick882c91e2018-07-12 22:11:41 +00001073 def _UpdateMirrorIfNotContains(self, mirror, options, rev_type, revision):
Andrii Shyshkalov46a672b2017-11-24 18:04:43 -08001074 """Update a git mirror by fetching the latest commits from the remote,
1075 unless mirror already contains revision whose type is sha1 hash.
1076 """
John Budorick882c91e2018-07-12 22:11:41 +00001077 if rev_type == 'hash' and mirror.contains_revision(revision):
Andrii Shyshkalov46a672b2017-11-24 18:04:43 -08001078 if options.verbose:
1079 self.Print('skipping mirror update, it has rev=%s already' % revision,
1080 timestamp=False)
1081 return
1082
szager@chromium.org3ec84f62014-08-22 21:00:22 +00001083 if getattr(options, 'shallow', False):
Victor Vianna392ce7e2022-06-07 21:47:51 +00001084 depth = 10000
hinoka@chromium.org46b87412014-05-15 00:42:05 +00001085 else:
1086 depth = None
e.hakkinen@samsung.come8bc1aa2015-04-08 08:00:37 +00001087 mirror.populate(verbose=options.verbose,
1088 bootstrap=not getattr(options, 'no_bootstrap', False),
Vadim Shtayura08049e22017-10-11 00:14:52 +00001089 depth=depth,
Vadim Shtayura08049e22017-10-11 00:14:52 +00001090 lock_timeout=getattr(options, 'lock_timeout', 0))
iannucci@chromium.org53456aa2013-07-03 19:38:34 +00001091
John Budorick882c91e2018-07-12 22:11:41 +00001092 def _Clone(self, revision, url, options):
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001093 """Clone a git repository from the given URL.
1094
msb@chromium.org786fb682010-06-02 15:16:23 +00001095 Once we've cloned the repo, we checkout a working branch if the specified
1096 revision is a branch head. If it is a tag or a specific commit, then we
1097 leave HEAD detached as it makes future updates simpler -- in this case the
1098 user should first create a new branch or switch to an existing branch before
1099 making changes in the repo."""
Joanna Wang1a977bd2022-06-02 21:51:17 +00001100 in_cog_workspace = self._IsCog()
1101
1102 if self.print_outbuf:
1103 print_stdout = True
1104 filter_fn = None
1105 else:
1106 print_stdout = False
1107 filter_fn = self.filter
1108
maruel@chromium.orgf5d37bf2010-09-02 00:50:34 +00001109 if not options.verbose:
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001110 # git clone doesn't seem to insert a newline properly before printing
1111 # to stdout
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001112 self.Print('')
Joanna Wang1a977bd2022-06-02 21:51:17 +00001113
nsylvain@chromium.org328c3c72011-06-01 20:50:27 +00001114 # If the parent directory does not exist, Git clone on Windows will not
1115 # create it, so we need to do it manually.
1116 parent_dir = os.path.dirname(self.checkout_path)
ilevy@chromium.org3534aa52013-07-20 01:58:08 +00001117 gclient_utils.safe_makedirs(parent_dir)
primiano@chromium.org5439ea52014-08-06 17:18:18 +00001118
Joanna Wang1a977bd2022-06-02 21:51:17 +00001119 if in_cog_workspace:
1120 clone_cmd = ['citc', 'clone-repo', url, self.checkout_path]
1121 clone_cmd.append(
1122 gclient_utils.ExtractRefName(self.remote, revision) or revision)
1123 try:
1124 self._Run(clone_cmd,
1125 options,
1126 cwd=self._root_dir,
1127 retry=True,
1128 print_stdout=print_stdout,
1129 filter_fn=filter_fn)
Joanna Wang1a977bd2022-06-02 21:51:17 +00001130 except:
1131 traceback.print_exc(file=self.out_fh)
1132 raise
1133 self._SetFetchConfig(options)
1134 else:
1135 cfg = gclient_utils.DefaultIndexPackConfig(url)
1136 clone_cmd = cfg + ['clone', '--no-checkout', '--progress']
1137 if self.cache_dir:
1138 clone_cmd.append('--shared')
1139 if options.verbose:
1140 clone_cmd.append('--verbose')
1141 clone_cmd.append(url)
primiano@chromium.org5439ea52014-08-06 17:18:18 +00001142
Joanna Wang1a977bd2022-06-02 21:51:17 +00001143 template_dir = None
1144 if hasattr(options, 'no_history') and options.no_history:
1145 if gclient_utils.IsGitSha(revision):
1146 # In the case of a subproject, the pinned sha is not necessarily the
1147 # head of the remote branch (so we can't just use --depth=N). Instead,
1148 # we tell git to fetch all the remote objects from SHA..HEAD by means
1149 # of a template git dir which has a 'shallow' file pointing to the
1150 # sha.
1151 template_dir = tempfile.mkdtemp(prefix='_gclient_gittmp_%s' %
1152 os.path.basename(self.checkout_path),
1153 dir=parent_dir)
1154 self._Run(['init', '--bare', template_dir],
1155 options,
1156 cwd=self._root_dir)
1157 with open(os.path.join(template_dir, 'shallow'),
1158 'w') as template_file:
1159 template_file.write(revision)
1160 clone_cmd.append('--template=' + template_dir)
1161 else:
1162 # Otherwise, we're just interested in the HEAD. Just use --depth.
1163 clone_cmd.append('--depth=1')
1164
1165 tmp_dir = tempfile.mkdtemp(prefix='_gclient_%s_' %
1166 os.path.basename(self.checkout_path),
1167 dir=parent_dir)
ilevy@chromium.org3534aa52013-07-20 01:58:08 +00001168 clone_cmd.append(tmp_dir)
Joanna Wang1a977bd2022-06-02 21:51:17 +00001169
1170 try:
1171 self._Run(clone_cmd,
1172 options,
1173 cwd=self._root_dir,
1174 retry=True,
1175 print_stdout=print_stdout,
1176 filter_fn=filter_fn)
Joanna Wang47fd5672022-08-05 20:53:31 +00001177 logging.debug('Cloned into temporary dir, moving to checkout_path')
Joanna Wang1a977bd2022-06-02 21:51:17 +00001178 gclient_utils.safe_makedirs(self.checkout_path)
1179 gclient_utils.safe_rename(os.path.join(tmp_dir, '.git'),
1180 os.path.join(self.checkout_path, '.git'))
1181 except:
1182 traceback.print_exc(file=self.out_fh)
1183 raise
1184 finally:
1185 if os.listdir(tmp_dir):
1186 self.Print('_____ removing non-empty tmp dir %s' % tmp_dir)
Josip Sokcevicebccac72022-06-24 21:44:49 +00001187 gclient_utils.rmtree(tmp_dir)
Joanna Wang1a977bd2022-06-02 21:51:17 +00001188 if template_dir:
1189 gclient_utils.rmtree(template_dir)
1190
1191 self._SetFetchConfig(options)
1192 self._Fetch(options, prune=options.force)
1193 revision = self._AutoFetchRef(options, revision)
1194 remote_ref = scm.GIT.RefToRemoteRef(revision, self.remote)
1195 self._Checkout(options, ''.join(remote_ref or revision), quiet=True)
1196
romain.pokrzywka@gmail.com483a0ba2014-05-30 00:06:07 +00001197 if self._GetCurrentBranch() is None:
msb@chromium.org786fb682010-06-02 15:16:23 +00001198 # Squelch git's very verbose detached HEAD warning and use our own
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001199 self.Print(
Joanna Wang1a977bd2022-06-02 21:51:17 +00001200 ('Checked out %s to a detached HEAD. Before making any commits\n'
1201 'in this repo, you should use \'git checkout <branch>\' to switch \n'
1202 'to an existing branch or use \'git checkout %s -b <branch>\' to\n'
1203 'create a new branch for your work.') % (revision, self.remote))
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001204
szager@chromium.org6cd41b62014-04-21 23:55:22 +00001205 def _AskForData(self, prompt, options):
bauerb@chromium.org30c46d62014-01-23 12:11:56 +00001206 if options.jobs > 1:
szager@chromium.org6cd41b62014-04-21 23:55:22 +00001207 self.Print(prompt)
bauerb@chromium.org30c46d62014-01-23 12:11:56 +00001208 raise gclient_utils.Error("Background task requires input. Rerun "
1209 "gclient with --jobs=1 so that\n"
1210 "interaction is possible.")
Edward Lesmesae3586b2020-03-23 21:21:14 +00001211 return gclient_utils.AskForData(prompt)
bauerb@chromium.org30c46d62014-01-23 12:11:56 +00001212
1213
maruel@chromium.orgf5d37bf2010-09-02 00:50:34 +00001214 def _AttemptRebase(self, upstream, files, options, newbase=None,
bauerb@chromium.org30c46d62014-01-23 12:11:56 +00001215 branch=None, printed_path=False, merge=False):
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001216 """Attempt to rebase onto either upstream or, if specified, newbase."""
iannucci@chromium.org396e1a62013-07-03 19:41:04 +00001217 if files is not None:
Aaron Gablef4068aa2017-12-12 15:14:09 -08001218 files.extend(self._GetDiffFilenames(upstream))
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001219 revision = upstream
1220 if newbase:
1221 revision = newbase
bauerb@chromium.org30c46d62014-01-23 12:11:56 +00001222 action = 'merge' if merge else 'rebase'
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001223 if not printed_path:
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001224 self.Print('_____ %s : Attempting %s onto %s...' % (
bauerb@chromium.org30c46d62014-01-23 12:11:56 +00001225 self.relpath, action, revision))
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001226 printed_path = True
1227 else:
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001228 self.Print('Attempting %s onto %s...' % (action, revision))
bauerb@chromium.org30c46d62014-01-23 12:11:56 +00001229
1230 if merge:
1231 merge_output = self._Capture(['merge', revision])
1232 if options.verbose:
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001233 self.Print(merge_output)
bauerb@chromium.org30c46d62014-01-23 12:11:56 +00001234 return
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001235
1236 # Build the rebase command here using the args
1237 # git rebase [options] [--onto <newbase>] <upstream> [<branch>]
1238 rebase_cmd = ['rebase']
maruel@chromium.orgf5d37bf2010-09-02 00:50:34 +00001239 if options.verbose:
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001240 rebase_cmd.append('--verbose')
1241 if newbase:
1242 rebase_cmd.extend(['--onto', newbase])
1243 rebase_cmd.append(upstream)
1244 if branch:
1245 rebase_cmd.append(branch)
1246
1247 try:
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +00001248 rebase_output = scm.GIT.Capture(rebase_cmd, cwd=self.checkout_path)
Raul Tambreb946b232019-03-26 14:48:46 +00001249 except subprocess2.CalledProcessError as e:
Edward Lemur979fa782019-08-13 22:44:05 +00001250 if (re.match(br'cannot rebase: you have unstaged changes', e.stderr) or
1251 re.match(br'cannot rebase: your index contains uncommitted changes',
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +00001252 e.stderr)):
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001253 while True:
bauerb@chromium.org30c46d62014-01-23 12:11:56 +00001254 rebase_action = self._AskForData(
maruel@chromium.org90541732011-04-01 17:54:18 +00001255 'Cannot rebase because of unstaged changes.\n'
1256 '\'git reset --hard HEAD\' ?\n'
1257 'WARNING: destroys any uncommitted work in your current branch!'
bratell@opera.com18fa4542013-05-21 13:30:46 +00001258 ' (y)es / (q)uit / (s)how : ', options)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001259 if re.match(r'yes|y', rebase_action, re.I):
Robert Iannuccic41d8b92017-02-16 17:07:37 -08001260 self._Scrub('HEAD', options)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001261 # Should this be recursive?
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +00001262 rebase_output = scm.GIT.Capture(rebase_cmd, cwd=self.checkout_path)
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001263 break
Aravind Vasudevanc5f0cbb2022-01-24 23:56:57 +00001264
1265 if re.match(r'quit|q', rebase_action, re.I):
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001266 raise gclient_utils.Error("Please merge or rebase manually\n"
1267 "cd %s && git " % self.checkout_path
1268 + "%s" % ' '.join(rebase_cmd))
Aravind Vasudevanc5f0cbb2022-01-24 23:56:57 +00001269
1270 if re.match(r'show|s', rebase_action, re.I):
Edward Lemur979fa782019-08-13 22:44:05 +00001271 self.Print('%s' % e.stderr.decode('utf-8').strip())
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001272 continue
Aravind Vasudevanc5f0cbb2022-01-24 23:56:57 +00001273
1274 gclient_utils.Error("Input not recognized")
1275 continue
Edward Lemur979fa782019-08-13 22:44:05 +00001276 elif re.search(br'^CONFLICT', e.stdout, re.M):
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001277 raise gclient_utils.Error("Conflict while rebasing this branch.\n"
1278 "Fix the conflict and run gclient again.\n"
1279 "See 'man git-rebase' for details.\n")
1280 else:
Edward Lemur979fa782019-08-13 22:44:05 +00001281 self.Print(e.stdout.decode('utf-8').strip())
1282 self.Print('Rebase produced error output:\n%s' %
1283 e.stderr.decode('utf-8').strip())
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001284 raise gclient_utils.Error("Unrecognized error, please merge or rebase "
1285 "manually.\ncd %s && git " %
1286 self.checkout_path
1287 + "%s" % ' '.join(rebase_cmd))
1288
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001289 self.Print(rebase_output.strip())
maruel@chromium.orgf5d37bf2010-09-02 00:50:34 +00001290 if not options.verbose:
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001291 # Make the output a little prettier. It's nice to have some
1292 # whitespace between projects when syncing.
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001293 self.Print('')
nasser@codeaurora.orgd90ba3f2010-02-23 14:42:57 +00001294
maruel@chromium.org6e29d572010-06-04 17:32:20 +00001295 @staticmethod
1296 def _CheckMinVersion(min_version):
maruel@chromium.orgd0f854a2010-03-11 19:35:53 +00001297 (ok, current_version) = scm.GIT.AssertVersion(min_version)
1298 if not ok:
1299 raise gclient_utils.Error('git version %s < minimum required %s' %
1300 (current_version, min_version))
msb@chromium.org923a0372009-12-11 20:42:43 +00001301
John Budorick882c91e2018-07-12 22:11:41 +00001302 def _EnsureValidHeadObjectOrCheckout(self, revision, options, url):
tandrii@chromium.orgc438c142015-08-24 22:55:55 +00001303 # Special case handling if all 3 conditions are met:
1304 # * the mirros have recently changed, but deps destination remains same,
1305 # * the git histories of mirrors are conflicting.
1306 # * git cache is used
1307 # This manifests itself in current checkout having invalid HEAD commit on
1308 # most git operations. Since git cache is used, just deleted the .git
1309 # folder, and re-create it by cloning.
1310 try:
1311 self._Capture(['rev-list', '-n', '1', 'HEAD'])
1312 except subprocess2.CalledProcessError as e:
Edward Lemur979fa782019-08-13 22:44:05 +00001313 if (b'fatal: bad object HEAD' in e.stderr
tandrii@chromium.orgc438c142015-08-24 22:55:55 +00001314 and self.cache_dir and self.cache_dir in url):
1315 self.Print((
1316 'Likely due to DEPS change with git cache_dir, '
1317 'the current commit points to no longer existing object.\n'
1318 '%s' % e)
1319 )
1320 self._DeleteOrMove(options.force)
John Budorick882c91e2018-07-12 22:11:41 +00001321 self._Clone(revision, url, options)
tandrii@chromium.orgc438c142015-08-24 22:55:55 +00001322 else:
1323 raise
1324
msb@chromium.org786fb682010-06-02 15:16:23 +00001325 def _IsRebasing(self):
1326 # Check for any of REBASE-i/REBASE-m/REBASE/AM. Unfortunately git doesn't
1327 # have a plumbing command to determine whether a rebase is in progress, so
1328 # for now emualate (more-or-less) git-rebase.sh / git-completion.bash
1329 g = os.path.join(self.checkout_path, '.git')
1330 return (
1331 os.path.isdir(os.path.join(g, "rebase-merge")) or
1332 os.path.isdir(os.path.join(g, "rebase-apply")))
1333
Robert Iannuccic41d8b92017-02-16 17:07:37 -08001334 def _CheckClean(self, revision, fixup=False):
iannucci@chromium.orgd9b318c2015-12-04 20:03:08 +00001335 lockfile = os.path.join(self.checkout_path, ".git", "index.lock")
1336 if os.path.exists(lockfile):
1337 raise gclient_utils.Error(
agable83faed02016-10-24 14:37:10 -07001338 '\n____ %s at %s\n'
iannucci@chromium.orgd9b318c2015-12-04 20:03:08 +00001339 '\tYour repo is locked, possibly due to a concurrent git process.\n'
1340 '\tIf no git executable is running, then clean up %r and try again.\n'
agable83faed02016-10-24 14:37:10 -07001341 % (self.relpath, revision, lockfile))
iannucci@chromium.orgd9b318c2015-12-04 20:03:08 +00001342
msb@chromium.org786fb682010-06-02 15:16:23 +00001343 # Make sure the tree is clean; see git-rebase.sh for reference
1344 try:
1345 scm.GIT.Capture(['update-index', '--ignore-submodules', '--refresh'],
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +00001346 cwd=self.checkout_path)
maruel@chromium.orgbffad372011-09-08 17:54:22 +00001347 except subprocess2.CalledProcessError:
agable83faed02016-10-24 14:37:10 -07001348 raise gclient_utils.Error('\n____ %s at %s\n'
maruel@chromium.org6e29d572010-06-04 17:32:20 +00001349 '\tYou have unstaged changes.\n'
1350 '\tPlease commit, stash, or reset.\n'
agable83faed02016-10-24 14:37:10 -07001351 % (self.relpath, revision))
msb@chromium.org786fb682010-06-02 15:16:23 +00001352 try:
1353 scm.GIT.Capture(['diff-index', '--cached', '--name-status', '-r',
smut@google.com27c9c8a2014-09-11 19:57:55 +00001354 '--ignore-submodules', 'HEAD', '--'],
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +00001355 cwd=self.checkout_path)
maruel@chromium.orgbffad372011-09-08 17:54:22 +00001356 except subprocess2.CalledProcessError:
agable83faed02016-10-24 14:37:10 -07001357 raise gclient_utils.Error('\n____ %s at %s\n'
maruel@chromium.org6e29d572010-06-04 17:32:20 +00001358 '\tYour index contains uncommitted changes\n'
1359 '\tPlease commit, stash, or reset.\n'
agable83faed02016-10-24 14:37:10 -07001360 % (self.relpath, revision))
msb@chromium.org786fb682010-06-02 15:16:23 +00001361
agable83faed02016-10-24 14:37:10 -07001362 def _CheckDetachedHead(self, revision, _options):
msb@chromium.org786fb682010-06-02 15:16:23 +00001363 # HEAD is detached. Make sure it is safe to move away from (i.e., it is
1364 # reference by a commit). If not, error out -- most likely a rebase is
1365 # in progress, try to detect so we can give a better error.
1366 try:
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +00001367 scm.GIT.Capture(['name-rev', '--no-undefined', 'HEAD'],
1368 cwd=self.checkout_path)
maruel@chromium.orgbffad372011-09-08 17:54:22 +00001369 except subprocess2.CalledProcessError:
msb@chromium.org786fb682010-06-02 15:16:23 +00001370 # Commit is not contained by any rev. See if the user is rebasing:
1371 if self._IsRebasing():
1372 # Punt to the user
agable83faed02016-10-24 14:37:10 -07001373 raise gclient_utils.Error('\n____ %s at %s\n'
msb@chromium.org786fb682010-06-02 15:16:23 +00001374 '\tAlready in a conflict, i.e. (no branch).\n'
1375 '\tFix the conflict and run gclient again.\n'
1376 '\tOr to abort run:\n\t\tgit-rebase --abort\n'
1377 '\tSee man git-rebase for details.\n'
agable83faed02016-10-24 14:37:10 -07001378 % (self.relpath, revision))
msb@chromium.org786fb682010-06-02 15:16:23 +00001379 # Let's just save off the commit so we can proceed.
maruel@chromium.org6cafa132010-09-07 14:17:26 +00001380 name = ('saved-by-gclient-' +
1381 self._Capture(['rev-parse', '--short', 'HEAD']))
mmoss@chromium.org77bd7362013-09-25 23:46:14 +00001382 self._Capture(['branch', '-f', name])
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001383 self.Print('_____ found an unreferenced commit and saved it as \'%s\'' %
maruel@chromium.orgf5d37bf2010-09-02 00:50:34 +00001384 name)
msb@chromium.org786fb682010-06-02 15:16:23 +00001385
msb@chromium.org5bde4852009-12-14 16:47:12 +00001386 def _GetCurrentBranch(self):
msb@chromium.org786fb682010-06-02 15:16:23 +00001387 # Returns name of current branch or None for detached HEAD
maruel@chromium.org6cafa132010-09-07 14:17:26 +00001388 branch = self._Capture(['rev-parse', '--abbrev-ref=strict', 'HEAD'])
msb@chromium.org786fb682010-06-02 15:16:23 +00001389 if branch == 'HEAD':
msb@chromium.org5bde4852009-12-14 16:47:12 +00001390 return None
1391 return branch
1392
borenet@google.comc3e09d22014-04-10 13:58:18 +00001393 def _Capture(self, args, **kwargs):
Mike Frysinger286fb162019-09-30 03:14:10 +00001394 set_git_dir = 'cwd' not in kwargs
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001395 kwargs.setdefault('cwd', self.checkout_path)
1396 kwargs.setdefault('stderr', subprocess2.PIPE)
Robert Iannuccia7a9ceb2017-02-16 17:38:06 -08001397 strip = kwargs.pop('strip', True)
szager@chromium.org6d8115d2014-04-23 20:59:23 +00001398 env = scm.GIT.ApplyEnvVars(kwargs)
Mike Frysinger286fb162019-09-30 03:14:10 +00001399 # If an explicit cwd isn't set, then default to the .git/ subdir so we get
1400 # stricter behavior. This can be useful in cases of slight corruption --
1401 # we don't accidentally go corrupting parent git checks too. See
1402 # https://crbug.com/1000825 for an example.
1403 if set_git_dir:
Dirk Prankedb1e79c2019-10-23 01:46:32 +00001404 git_dir = os.path.abspath(os.path.join(self.checkout_path, '.git'))
Dirk Prankedb1e79c2019-10-23 01:46:32 +00001405 # Depending on how the .gclient file was defined, self.checkout_path
1406 # might be set to a unicode string, not a regular string; on Windows
1407 # Python2, we can't set env vars to be unicode strings, so we
1408 # forcibly cast the value to a string before setting it.
1409 env.setdefault('GIT_DIR', str(git_dir))
Raul Tambrecd862e32019-05-10 21:19:00 +00001410 ret = subprocess2.check_output(
1411 ['git'] + args, env=env, **kwargs).decode('utf-8')
Robert Iannuccia7a9ceb2017-02-16 17:38:06 -08001412 if strip:
1413 ret = ret.strip()
Erik Chene16ffff2019-10-14 20:35:53 +00001414 self.Print('Finished running: %s %s' % ('git', ' '.join(args)))
Robert Iannuccia7a9ceb2017-02-16 17:38:06 -08001415 return ret
maruel@chromium.org6cafa132010-09-07 14:17:26 +00001416
dnj@chromium.orgbb424c02014-06-23 22:42:51 +00001417 def _Checkout(self, options, ref, force=False, quiet=None):
1418 """Performs a 'git-checkout' operation.
1419
1420 Args:
1421 options: The configured option set
1422 ref: (str) The branch/commit to checkout
Quinten Yearsley925cedb2020-04-13 17:49:39 +00001423 quiet: (bool/None) Whether or not the checkout should pass '--quiet'; if
dnj@chromium.orgbb424c02014-06-23 22:42:51 +00001424 'None', the behavior is inferred from 'options.verbose'.
1425 Returns: (str) The output of the checkout operation
1426 """
1427 if quiet is None:
1428 quiet = (not options.verbose)
1429 checkout_args = ['checkout']
1430 if force:
1431 checkout_args.append('--force')
1432 if quiet:
1433 checkout_args.append('--quiet')
1434 checkout_args.append(ref)
1435 return self._Capture(checkout_args)
1436
Paweł Hajdan, Jr63b8c2a2017-09-05 17:59:08 +02001437 def _Fetch(self, options, remote=None, prune=False, quiet=False,
1438 refspec=None):
dnj@chromium.org680f2172014-06-25 00:39:32 +00001439 cfg = gclient_utils.DefaultIndexPackConfig(self.url)
Edward Lemur9a5e3bd2019-04-02 23:37:45 +00001440 # When updating, the ref is modified to be a remote ref .
1441 # (e.g. refs/heads/NAME becomes refs/remotes/REMOTE/NAME).
1442 # Try to reverse that mapping.
1443 original_ref = scm.GIT.RemoteRefToRef(refspec, self.remote)
1444 if original_ref:
1445 refspec = original_ref + ':' + refspec
1446 # When a mirror is configured, it only fetches
1447 # refs/{heads,branch-heads,tags}/*.
1448 # If asked to fetch other refs, we must fetch those directly from the
1449 # repository, and not from the mirror.
1450 if not original_ref.startswith(
1451 ('refs/heads/', 'refs/branch-heads/', 'refs/tags/')):
1452 remote, _ = gclient_utils.SplitUrlRevision(self.url)
dnj@chromium.org680f2172014-06-25 00:39:32 +00001453 fetch_cmd = cfg + [
1454 'fetch',
1455 remote or self.remote,
1456 ]
Paweł Hajdan, Jr63b8c2a2017-09-05 17:59:08 +02001457 if refspec:
1458 fetch_cmd.append(refspec)
dnj@chromium.org680f2172014-06-25 00:39:32 +00001459
1460 if prune:
1461 fetch_cmd.append('--prune')
1462 if options.verbose:
1463 fetch_cmd.append('--verbose')
danakjd5c0b562019-11-08 17:27:47 +00001464 if not hasattr(options, 'with_tags') or not options.with_tags:
1465 fetch_cmd.append('--no-tags')
dnj@chromium.org680f2172014-06-25 00:39:32 +00001466 elif quiet:
1467 fetch_cmd.append('--quiet')
tandrii64103db2016-10-11 05:30:05 -07001468 self._Run(fetch_cmd, options, show_header=options.verbose, retry=True)
dnj@chromium.org680f2172014-06-25 00:39:32 +00001469
Edward Lemur579c9862018-07-13 23:17:51 +00001470 def _SetFetchConfig(self, options):
szager@chromium.org8d3348f2014-08-19 22:49:16 +00001471 """Adds, and optionally fetches, "branch-heads" and "tags" refspecs
1472 if requested."""
Edward Lemur2f38df62018-07-14 02:13:21 +00001473 if options.force or options.reset:
Edward Lemur579c9862018-07-13 23:17:51 +00001474 try:
1475 self._Run(['config', '--unset-all', 'remote.%s.fetch' % self.remote],
1476 options)
1477 self._Run(['config', 'remote.%s.fetch' % self.remote,
1478 '+refs/heads/*:refs/remotes/%s/*' % self.remote], options)
1479 except subprocess2.CalledProcessError as e:
1480 # If exit code was 5, it means we attempted to unset a config that
1481 # didn't exist. Ignore it.
1482 if e.returncode != 5:
1483 raise
mmoss@chromium.orge409df62013-04-16 17:28:57 +00001484 if hasattr(options, 'with_branch_heads') and options.with_branch_heads:
maruel@chromium.org1a60dca2013-11-26 14:06:26 +00001485 config_cmd = ['config', 'remote.%s.fetch' % self.remote,
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +00001486 '+refs/branch-heads/*:refs/remotes/branch-heads/*',
1487 '^\\+refs/branch-heads/\\*:.*$']
1488 self._Run(config_cmd, options)
szager@chromium.org8d3348f2014-08-19 22:49:16 +00001489 if hasattr(options, 'with_tags') and options.with_tags:
1490 config_cmd = ['config', 'remote.%s.fetch' % self.remote,
1491 '+refs/tags/*:refs/tags/*',
1492 '^\\+refs/tags/\\*:.*$']
1493 self._Run(config_cmd, options)
mmoss@chromium.orge409df62013-04-16 17:28:57 +00001494
John Budorick882c91e2018-07-12 22:11:41 +00001495 def _AutoFetchRef(self, options, revision):
Paweł Hajdan, Jr63b8c2a2017-09-05 17:59:08 +02001496 """Attempts to fetch |revision| if not available in local repo.
1497
1498 Returns possibly updated revision."""
Edward Lemure0ba7b82020-03-11 20:31:32 +00001499 if not scm.GIT.IsValidRevision(self.checkout_path, revision):
Paweł Hajdan, Jr63b8c2a2017-09-05 17:59:08 +02001500 self._Fetch(options, refspec=revision)
1501 revision = self._Capture(['rev-parse', 'FETCH_HEAD'])
1502 return revision
1503
Edward Lemur24146be2019-08-01 21:44:52 +00001504 def _Run(self, args, options, **kwargs):
Quinten Yearsleyb2cc4a92016-12-15 13:53:26 -08001505 # Disable 'unused options' warning | pylint: disable=unused-argument
sbc@chromium.org2cd0b8e2014-09-22 21:17:59 +00001506 kwargs.setdefault('cwd', self.checkout_path)
Edward Lemur24146be2019-08-01 21:44:52 +00001507 kwargs.setdefault('filter_fn', self.filter)
1508 kwargs.setdefault('show_header', True)
szager@chromium.org6d8115d2014-04-23 20:59:23 +00001509 env = scm.GIT.ApplyEnvVars(kwargs)
Nico Weberc49c88a2020-07-08 17:36:02 +00001510
agable@chromium.org772efaf2014-04-01 02:35:44 +00001511 cmd = ['git'] + args
Edward Lemur24146be2019-08-01 21:44:52 +00001512 gclient_utils.CheckCallAndFilter(cmd, env=env, **kwargs)
John Budorick0f7b2002018-01-19 15:46:17 -08001513
1514
1515class CipdPackage(object):
1516 """A representation of a single CIPD package."""
1517
John Budorickd3ba72b2018-03-20 12:27:42 -07001518 def __init__(self, name, version, authority_for_subdir):
John Budorick0f7b2002018-01-19 15:46:17 -08001519 self._authority_for_subdir = authority_for_subdir
1520 self._name = name
1521 self._version = version
1522
1523 @property
John Budorick0f7b2002018-01-19 15:46:17 -08001524 def authority_for_subdir(self):
1525 """Whether this package has authority to act on behalf of its subdir.
1526
1527 Some operations should only be performed once per subdirectory. A package
1528 that has authority for its subdirectory is the only package that should
1529 perform such operations.
1530
1531 Returns:
1532 bool; whether this package has subdir authority.
1533 """
1534 return self._authority_for_subdir
1535
1536 @property
1537 def name(self):
1538 return self._name
1539
1540 @property
1541 def version(self):
1542 return self._version
1543
1544
1545class CipdRoot(object):
1546 """A representation of a single CIPD root."""
1547 def __init__(self, root_dir, service_url):
1548 self._all_packages = set()
1549 self._mutator_lock = threading.Lock()
1550 self._packages_by_subdir = collections.defaultdict(list)
1551 self._root_dir = root_dir
1552 self._service_url = service_url
1553
1554 def add_package(self, subdir, package, version):
1555 """Adds a package to this CIPD root.
1556
1557 As far as clients are concerned, this grants both root and subdir authority
1558 to packages arbitrarily. (The implementation grants root authority to the
1559 first package added and subdir authority to the first package added for that
1560 subdir, but clients should not depend on or expect that behavior.)
1561
1562 Args:
1563 subdir: str; relative path to where the package should be installed from
1564 the cipd root directory.
1565 package: str; the cipd package name.
1566 version: str; the cipd package version.
1567 Returns:
1568 CipdPackage; the package that was created and added to this root.
1569 """
1570 with self._mutator_lock:
1571 cipd_package = CipdPackage(
1572 package, version,
John Budorick0f7b2002018-01-19 15:46:17 -08001573 not self._packages_by_subdir[subdir])
1574 self._all_packages.add(cipd_package)
1575 self._packages_by_subdir[subdir].append(cipd_package)
1576 return cipd_package
1577
1578 def packages(self, subdir):
1579 """Get the list of configured packages for the given subdir."""
1580 return list(self._packages_by_subdir[subdir])
1581
1582 def clobber(self):
1583 """Remove the .cipd directory.
1584
1585 This is useful for forcing ensure to redownload and reinitialize all
1586 packages.
1587 """
1588 with self._mutator_lock:
John Budorickd3ba72b2018-03-20 12:27:42 -07001589 cipd_cache_dir = os.path.join(self.root_dir, '.cipd')
John Budorick0f7b2002018-01-19 15:46:17 -08001590 try:
1591 gclient_utils.rmtree(os.path.join(cipd_cache_dir))
1592 except OSError:
1593 if os.path.exists(cipd_cache_dir):
1594 raise
1595
1596 @contextlib.contextmanager
1597 def _create_ensure_file(self):
1598 try:
Stephanie Kim700aee72022-06-01 19:58:30 +00001599 contents = '$ParanoidMode CheckPresence\n'
1600 # TODO(crbug/1329641): Remove once cipd packages have been updated
1601 # to always be created in copy mode.
1602 contents += '$OverrideInstallMode copy\n\n'
Edward Lesmes05934952019-12-19 20:38:09 +00001603 for subdir, packages in sorted(self._packages_by_subdir.items()):
1604 contents += '@Subdir %s\n' % subdir
1605 for package in sorted(packages, key=lambda p: p.name):
1606 contents += '%s %s\n' % (package.name, package.version)
1607 contents += '\n'
John Budorick0f7b2002018-01-19 15:46:17 -08001608 ensure_file = None
1609 with tempfile.NamedTemporaryFile(
Edward Lesmes05934952019-12-19 20:38:09 +00001610 suffix='.ensure', delete=False, mode='wb') as ensure_file:
1611 ensure_file.write(contents.encode('utf-8', 'replace'))
John Budorick0f7b2002018-01-19 15:46:17 -08001612 yield ensure_file.name
1613 finally:
1614 if ensure_file is not None and os.path.exists(ensure_file.name):
1615 os.remove(ensure_file.name)
1616
1617 def ensure(self):
1618 """Run `cipd ensure`."""
1619 with self._mutator_lock:
1620 with self._create_ensure_file() as ensure_file:
1621 cmd = [
1622 'cipd', 'ensure',
1623 '-log-level', 'error',
1624 '-root', self.root_dir,
1625 '-ensure-file', ensure_file,
1626 ]
Edward Lemur24146be2019-08-01 21:44:52 +00001627 gclient_utils.CheckCallAndFilter(
1628 cmd, print_stdout=True, show_header=True)
John Budorick0f7b2002018-01-19 15:46:17 -08001629
John Budorickd3ba72b2018-03-20 12:27:42 -07001630 def run(self, command):
1631 if command == 'update':
1632 self.ensure()
1633 elif command == 'revert':
1634 self.clobber()
1635 self.ensure()
1636
John Budorick0f7b2002018-01-19 15:46:17 -08001637 def created_package(self, package):
1638 """Checks whether this root created the given package.
1639
1640 Args:
1641 package: CipdPackage; the package to check.
1642 Returns:
1643 bool; whether this root created the given package.
1644 """
1645 return package in self._all_packages
1646
1647 @property
1648 def root_dir(self):
1649 return self._root_dir
1650
1651 @property
1652 def service_url(self):
1653 return self._service_url
1654
1655
1656class CipdWrapper(SCMWrapper):
1657 """Wrapper for CIPD.
1658
1659 Currently only supports chrome-infra-packages.appspot.com.
1660 """
John Budorick3929e9e2018-02-04 18:18:07 -08001661 name = 'cipd'
John Budorick0f7b2002018-01-19 15:46:17 -08001662
1663 def __init__(self, url=None, root_dir=None, relpath=None, out_fh=None,
1664 out_cb=None, root=None, package=None):
1665 super(CipdWrapper, self).__init__(
1666 url=url, root_dir=root_dir, relpath=relpath, out_fh=out_fh,
1667 out_cb=out_cb)
1668 assert root.created_package(package)
1669 self._package = package
1670 self._root = root
1671
1672 #override
1673 def GetCacheMirror(self):
1674 return None
1675
1676 #override
1677 def GetActualRemoteURL(self, options):
1678 return self._root.service_url
1679
1680 #override
1681 def DoesRemoteURLMatch(self, options):
1682 del options
1683 return True
1684
1685 def revert(self, options, args, file_list):
John Budorickd3ba72b2018-03-20 12:27:42 -07001686 """Does nothing.
1687
1688 CIPD packages should be reverted at the root by running
1689 `CipdRoot.run('revert')`.
1690 """
John Budorick0f7b2002018-01-19 15:46:17 -08001691
1692 def diff(self, options, args, file_list):
1693 """CIPD has no notion of diffing."""
John Budorick0f7b2002018-01-19 15:46:17 -08001694
1695 def pack(self, options, args, file_list):
1696 """CIPD has no notion of diffing."""
John Budorick0f7b2002018-01-19 15:46:17 -08001697
1698 def revinfo(self, options, args, file_list):
1699 """Grab the instance ID."""
1700 try:
1701 tmpdir = tempfile.mkdtemp()
1702 describe_json_path = os.path.join(tmpdir, 'describe.json')
1703 cmd = [
1704 'cipd', 'describe',
1705 self._package.name,
1706 '-log-level', 'error',
1707 '-version', self._package.version,
1708 '-json-output', describe_json_path
1709 ]
Edward Lemur24146be2019-08-01 21:44:52 +00001710 gclient_utils.CheckCallAndFilter(cmd)
John Budorick0f7b2002018-01-19 15:46:17 -08001711 with open(describe_json_path) as f:
1712 describe_json = json.load(f)
1713 return describe_json.get('result', {}).get('pin', {}).get('instance_id')
1714 finally:
1715 gclient_utils.rmtree(tmpdir)
1716
1717 def status(self, options, args, file_list):
1718 pass
1719
1720 def update(self, options, args, file_list):
John Budorickd3ba72b2018-03-20 12:27:42 -07001721 """Does nothing.
1722
1723 CIPD packages should be updated at the root by running
1724 `CipdRoot.run('update')`.
1725 """