blob: 0da2118b9c317f60789a3776f6e326164ac03088 [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001# Copyright (C) 2008 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
Shawn O. Pearce438ee1c2008-11-03 09:59:36 -080015import errno
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070016import filecmp
Wink Saville4c426ef2015-06-03 08:05:17 -070017import glob
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070018import os
LaMont Jonesd82be3e2022-04-05 19:30:46 +000019import platform
Shawn O. Pearcec325dc32011-10-03 08:30:24 -070020import random
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070021import re
22import shutil
23import stat
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -070024import subprocess
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070025import sys
Julien Campergue335f5ef2013-10-16 11:02:35 +020026import tarfile
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +080027import tempfile
Shawn O. Pearcec325dc32011-10-03 08:30:24 -070028import time
LaMont Jones1eddca82022-09-01 15:15:04 +000029from typing import NamedTuple
Mike Frysingeracf63b22019-06-13 02:24:21 -040030import urllib.parse
Shawn O. Pearcedf5ee522011-10-11 14:05:21 -070031
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070032from color import Coloring
LaMont Jones0de4fc32022-04-21 17:18:35 +000033import fetch
Dave Borowitzb42b4742012-10-31 12:27:27 -070034from git_command import GitCommand, git_require
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -070035from git_config import GitConfig, IsId, GetSchemeFromUrl, GetUrlCookieFile, \
36 ID_RE
LaMont Jonesff6b1da2022-06-01 21:03:34 +000037import git_superproject
LaMont Jones55ee3042022-04-06 17:10:21 +000038from git_trace2_event_log import EventLog
Remy Bohmer16c13282020-09-10 10:38:04 +020039from error import GitError, UploadError, DownloadError
Mike Frysingere6a202f2019-08-02 15:57:57 -040040from error import ManifestInvalidRevisionError, ManifestInvalidPathError
LaMont Jones409407a2022-04-05 21:21:56 +000041from error import NoManifestException, ManifestParseError
Renaud Paquayd5cec5e2016-11-01 11:24:03 -070042import platform_utils
Mike Frysinger70d861f2019-08-26 15:22:36 -040043import progress
Mike Frysinger8a11f6f2019-08-27 00:26:15 -040044from repo_trace import IsTrace, Trace
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070045
Mike Frysinger21b7fbe2020-02-26 23:53:36 -050046from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M, R_WORKTREE_M
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070047
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -070048
LaMont Jones1eddca82022-09-01 15:15:04 +000049class SyncNetworkHalfResult(NamedTuple):
50 """Sync_NetworkHalf return value."""
51 # True if successful.
52 success: bool
53 # Did we query the remote? False when optimized_fetch is True and we have the
54 # commit already present.
55 remote_fetched: bool
56
George Engelbrecht9bc283e2020-04-02 12:36:09 -060057# Maximum sleep time allowed during retries.
58MAXIMUM_RETRY_SLEEP_SEC = 3600.0
59# +-10% random jitter is added to each Fetches retry sleep duration.
60RETRY_JITTER_PERCENT = 0.1
61
Mike Frysinger1d00a7e2021-12-21 00:40:31 -050062# Whether to use alternates.
63# TODO(vapier): Remove knob once behavior is verified.
64_ALTERNATES = os.environ.get('REPO_USE_ALTERNATES') == '1'
George Engelbrecht9bc283e2020-04-02 12:36:09 -060065
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -070066def _lwrite(path, content):
67 lock = '%s.lock' % path
68
Remy Bohmer169b0212020-11-21 10:57:52 +010069 # Maintain Unix line endings on all OS's to match git behavior.
70 with open(lock, 'w', newline='\n') as fd:
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -070071 fd.write(content)
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -070072
73 try:
Renaud Paquayad1abcb2016-11-01 11:34:55 -070074 platform_utils.rename(lock, path)
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -070075 except OSError:
Renaud Paquay010fed72016-11-11 14:25:29 -080076 platform_utils.remove(lock)
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -070077 raise
78
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -070079
Shawn O. Pearce48244782009-04-16 08:25:57 -070080def _error(fmt, *args):
81 msg = fmt % args
Sarah Owenscecd1d82012-11-01 22:59:27 -070082 print('error: %s' % msg, file=sys.stderr)
Shawn O. Pearce48244782009-04-16 08:25:57 -070083
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -070084
David Pursehousef33929d2015-08-24 14:39:14 +090085def _warn(fmt, *args):
86 msg = fmt % args
87 print('warn: %s' % msg, file=sys.stderr)
88
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -070089
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070090def not_rev(r):
91 return '^' + r
92
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -070093
Shawn O. Pearceb54a3922009-01-05 16:18:58 -080094def sq(r):
95 return "'" + r.replace("'", "'\''") + "'"
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -080096
David Pursehouse819827a2020-02-12 15:20:19 +090097
Jonathan Nieder93719792015-03-17 11:29:58 -070098_project_hook_list = None
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -070099
100
Jonathan Nieder93719792015-03-17 11:29:58 -0700101def _ProjectHooks():
102 """List the hooks present in the 'hooks' directory.
103
104 These hooks are project hooks and are copied to the '.git/hooks' directory
105 of all subprojects.
106
107 This function caches the list of hooks (based on the contents of the
108 'repo/hooks' directory) on the first call.
109
110 Returns:
111 A list of absolute paths to all of the files in the hooks directory.
112 """
113 global _project_hook_list
114 if _project_hook_list is None:
Renaud Paquay227ad2e2016-11-01 14:37:13 -0700115 d = platform_utils.realpath(os.path.abspath(os.path.dirname(__file__)))
Jonathan Nieder93719792015-03-17 11:29:58 -0700116 d = os.path.join(d, 'hooks')
Renaud Paquaybed8b622018-09-27 10:46:58 -0700117 _project_hook_list = [os.path.join(d, x) for x in platform_utils.listdir(d)]
Jonathan Nieder93719792015-03-17 11:29:58 -0700118 return _project_hook_list
119
120
Shawn O. Pearce632768b2008-10-23 11:58:52 -0700121class DownloadedChange(object):
122 _commit_cache = None
123
124 def __init__(self, project, base, change_id, ps_id, commit):
125 self.project = project
126 self.base = base
127 self.change_id = change_id
128 self.ps_id = ps_id
129 self.commit = commit
130
131 @property
132 def commits(self):
133 if self._commit_cache is None:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700134 self._commit_cache = self.project.bare_git.rev_list('--abbrev=8',
135 '--abbrev-commit',
136 '--pretty=oneline',
137 '--reverse',
138 '--date-order',
139 not_rev(self.base),
140 self.commit,
141 '--')
Shawn O. Pearce632768b2008-10-23 11:58:52 -0700142 return self._commit_cache
143
144
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700145class ReviewableBranch(object):
146 _commit_cache = None
Mike Frysinger6da17752019-09-11 18:43:17 -0400147 _base_exists = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700148
149 def __init__(self, project, branch, base):
150 self.project = project
151 self.branch = branch
152 self.base = base
153
154 @property
155 def name(self):
156 return self.branch.name
157
158 @property
159 def commits(self):
160 if self._commit_cache is None:
Mike Frysinger6da17752019-09-11 18:43:17 -0400161 args = ('--abbrev=8', '--abbrev-commit', '--pretty=oneline', '--reverse',
162 '--date-order', not_rev(self.base), R_HEADS + self.name, '--')
163 try:
164 self._commit_cache = self.project.bare_git.rev_list(*args)
165 except GitError:
166 # We weren't able to probe the commits for this branch. Was it tracking
167 # a branch that no longer exists? If so, return no commits. Otherwise,
168 # rethrow the error as we don't know what's going on.
169 if self.base_exists:
170 raise
171
172 self._commit_cache = []
173
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700174 return self._commit_cache
175
176 @property
Shawn O. Pearcec99883f2008-11-11 17:12:43 -0800177 def unabbrev_commits(self):
178 r = dict()
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700179 for commit in self.project.bare_git.rev_list(not_rev(self.base),
180 R_HEADS + self.name,
181 '--'):
Shawn O. Pearcec99883f2008-11-11 17:12:43 -0800182 r[commit[0:8]] = commit
183 return r
184
185 @property
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700186 def date(self):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700187 return self.project.bare_git.log('--pretty=format:%cd',
188 '-n', '1',
189 R_HEADS + self.name,
190 '--')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700191
Mike Frysinger6da17752019-09-11 18:43:17 -0400192 @property
193 def base_exists(self):
194 """Whether the branch we're tracking exists.
195
196 Normally it should, but sometimes branches we track can get deleted.
197 """
198 if self._base_exists is None:
199 try:
200 self.project.bare_git.rev_parse('--verify', not_rev(self.base))
201 # If we're still here, the base branch exists.
202 self._base_exists = True
203 except GitError:
204 # If we failed to verify, the base branch doesn't exist.
205 self._base_exists = False
206
207 return self._base_exists
208
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700209 def UploadForReview(self, people,
Mike Frysinger819cc812020-02-19 02:27:22 -0500210 dryrun=False,
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700211 auto_topic=False,
Mike Frysinger84685ba2020-02-19 02:22:22 -0500212 hashtags=(),
Mike Frysingerfc1b18a2020-02-24 15:38:07 -0500213 labels=(),
Changcheng Xiao87984c62017-08-02 16:55:03 +0200214 private=False,
Vadim Bendeburybd8f6582018-10-31 13:48:01 -0700215 notify=None,
Changcheng Xiao87984c62017-08-02 16:55:03 +0200216 wip=False,
William Escandeac76fd32022-08-02 16:05:37 -0700217 ready=False,
Łukasz Gardońbed59ce2017-08-08 10:18:11 +0200218 dest_branch=None,
Masaya Suzuki305a2d02017-11-13 10:48:34 -0800219 validate_certs=True,
220 push_options=None):
Mike Frysinger819cc812020-02-19 02:27:22 -0500221 self.project.UploadForReview(branch=self.name,
222 people=people,
223 dryrun=dryrun,
Brian Harring435370c2012-07-28 15:37:04 -0700224 auto_topic=auto_topic,
Mike Frysinger84685ba2020-02-19 02:22:22 -0500225 hashtags=hashtags,
Mike Frysingerfc1b18a2020-02-24 15:38:07 -0500226 labels=labels,
Changcheng Xiao87984c62017-08-02 16:55:03 +0200227 private=private,
Vadim Bendeburybd8f6582018-10-31 13:48:01 -0700228 notify=notify,
Changcheng Xiao87984c62017-08-02 16:55:03 +0200229 wip=wip,
William Escandeac76fd32022-08-02 16:05:37 -0700230 ready=ready,
Łukasz Gardońbed59ce2017-08-08 10:18:11 +0200231 dest_branch=dest_branch,
Masaya Suzuki305a2d02017-11-13 10:48:34 -0800232 validate_certs=validate_certs,
233 push_options=push_options)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700234
Ficus Kirkpatrickbc7ef672009-05-04 12:45:11 -0700235 def GetPublishedRefs(self):
236 refs = {}
237 output = self.project.bare_git.ls_remote(
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700238 self.branch.remote.SshReviewUrl(self.project.UserEmail),
239 'refs/changes/*')
Ficus Kirkpatrickbc7ef672009-05-04 12:45:11 -0700240 for line in output.split('\n'):
241 try:
242 (sha, ref) = line.split()
243 refs[sha] = ref
244 except ValueError:
245 pass
246
247 return refs
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700248
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700249
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700250class StatusColoring(Coloring):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700251
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700252 def __init__(self, config):
Mike Frysinger5d9c4972021-02-19 13:34:09 -0500253 super().__init__(config, 'status')
Anthony King7bdac712014-07-16 12:56:40 +0100254 self.project = self.printer('header', attr='bold')
255 self.branch = self.printer('header', attr='bold')
256 self.nobranch = self.printer('nobranch', fg='red')
257 self.important = self.printer('important', fg='red')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700258
Anthony King7bdac712014-07-16 12:56:40 +0100259 self.added = self.printer('added', fg='green')
260 self.changed = self.printer('changed', fg='red')
261 self.untracked = self.printer('untracked', fg='red')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700262
263
264class DiffColoring(Coloring):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700265
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700266 def __init__(self, config):
Mike Frysinger5d9c4972021-02-19 13:34:09 -0500267 super().__init__(config, 'diff')
Anthony King7bdac712014-07-16 12:56:40 +0100268 self.project = self.printer('header', attr='bold')
Mike Frysinger0a9265e2019-09-30 23:59:27 -0400269 self.fail = self.printer('fail', fg='red')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700270
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700271
Jack Neus6ea0cae2021-07-20 20:52:33 +0000272class Annotation(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700273
James W. Mills24c13082012-04-12 15:04:13 -0500274 def __init__(self, name, value, keep):
275 self.name = name
276 self.value = value
277 self.keep = keep
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700278
Jack Neus6ea0cae2021-07-20 20:52:33 +0000279 def __eq__(self, other):
280 if not isinstance(other, Annotation):
281 return False
282 return self.__dict__ == other.__dict__
283
284 def __lt__(self, other):
285 # This exists just so that lists of Annotation objects can be sorted, for
286 # use in comparisons.
287 if not isinstance(other, Annotation):
288 raise ValueError('comparison is not between two Annotation objects')
289 if self.name == other.name:
290 if self.value == other.value:
291 return self.keep < other.keep
292 return self.value < other.value
293 return self.name < other.name
294
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700295
Mike Frysingere6a202f2019-08-02 15:57:57 -0400296def _SafeExpandPath(base, subpath, skipfinal=False):
297 """Make sure |subpath| is completely safe under |base|.
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700298
Mike Frysingere6a202f2019-08-02 15:57:57 -0400299 We make sure no intermediate symlinks are traversed, and that the final path
300 is not a special file (e.g. not a socket or fifo).
301
302 NB: We rely on a number of paths already being filtered out while parsing the
303 manifest. See the validation logic in manifest_xml.py for more details.
304 """
Mike Frysingerd9254592020-02-19 22:36:26 -0500305 # Split up the path by its components. We can't use os.path.sep exclusively
306 # as some platforms (like Windows) will convert / to \ and that bypasses all
307 # our constructed logic here. Especially since manifest authors only use
308 # / in their paths.
309 resep = re.compile(r'[/%s]' % re.escape(os.path.sep))
310 components = resep.split(subpath)
Mike Frysingere6a202f2019-08-02 15:57:57 -0400311 if skipfinal:
312 # Whether the caller handles the final component itself.
313 finalpart = components.pop()
314
315 path = base
316 for part in components:
317 if part in {'.', '..'}:
318 raise ManifestInvalidPathError(
319 '%s: "%s" not allowed in paths' % (subpath, part))
320
321 path = os.path.join(path, part)
322 if platform_utils.islink(path):
323 raise ManifestInvalidPathError(
324 '%s: traversing symlinks not allow' % (path,))
325
326 if os.path.exists(path):
327 if not os.path.isfile(path) and not platform_utils.isdir(path):
328 raise ManifestInvalidPathError(
329 '%s: only regular files & directories allowed' % (path,))
330
331 if skipfinal:
332 path = os.path.join(path, finalpart)
333
334 return path
335
336
337class _CopyFile(object):
338 """Container for <copyfile> manifest element."""
339
340 def __init__(self, git_worktree, src, topdir, dest):
341 """Register a <copyfile> request.
342
343 Args:
344 git_worktree: Absolute path to the git project checkout.
345 src: Relative path under |git_worktree| of file to read.
346 topdir: Absolute path to the top of the repo client checkout.
347 dest: Relative path under |topdir| of file to write.
348 """
349 self.git_worktree = git_worktree
350 self.topdir = topdir
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700351 self.src = src
352 self.dest = dest
353
354 def _Copy(self):
Mike Frysingere6a202f2019-08-02 15:57:57 -0400355 src = _SafeExpandPath(self.git_worktree, self.src)
356 dest = _SafeExpandPath(self.topdir, self.dest)
357
358 if platform_utils.isdir(src):
359 raise ManifestInvalidPathError(
360 '%s: copying from directory not supported' % (self.src,))
361 if platform_utils.isdir(dest):
362 raise ManifestInvalidPathError(
363 '%s: copying to directory not allowed' % (self.dest,))
364
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700365 # copy file if it does not exist or is out of date
366 if not os.path.exists(dest) or not filecmp.cmp(src, dest):
367 try:
368 # remove existing file first, since it might be read-only
369 if os.path.exists(dest):
Renaud Paquay010fed72016-11-11 14:25:29 -0800370 platform_utils.remove(dest)
Matthew Buckett2daf6672009-07-11 09:43:47 -0400371 else:
Mickaël Salaün2f6ab7f2012-09-30 00:37:55 +0200372 dest_dir = os.path.dirname(dest)
Renaud Paquaybed8b622018-09-27 10:46:58 -0700373 if not platform_utils.isdir(dest_dir):
Mickaël Salaün2f6ab7f2012-09-30 00:37:55 +0200374 os.makedirs(dest_dir)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700375 shutil.copy(src, dest)
376 # make the file read-only
377 mode = os.stat(dest)[stat.ST_MODE]
378 mode = mode & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
379 os.chmod(dest, mode)
380 except IOError:
Shawn O. Pearce48244782009-04-16 08:25:57 -0700381 _error('Cannot copy file %s to %s', src, dest)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700382
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700383
Anthony King7bdac712014-07-16 12:56:40 +0100384class _LinkFile(object):
Mike Frysingere6a202f2019-08-02 15:57:57 -0400385 """Container for <linkfile> manifest element."""
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700386
Mike Frysingere6a202f2019-08-02 15:57:57 -0400387 def __init__(self, git_worktree, src, topdir, dest):
388 """Register a <linkfile> request.
389
390 Args:
391 git_worktree: Absolute path to the git project checkout.
392 src: Target of symlink relative to path under |git_worktree|.
393 topdir: Absolute path to the top of the repo client checkout.
394 dest: Relative path under |topdir| of symlink to create.
395 """
Wink Saville4c426ef2015-06-03 08:05:17 -0700396 self.git_worktree = git_worktree
Mike Frysingere6a202f2019-08-02 15:57:57 -0400397 self.topdir = topdir
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500398 self.src = src
399 self.dest = dest
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500400
Wink Saville4c426ef2015-06-03 08:05:17 -0700401 def __linkIt(self, relSrc, absDest):
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500402 # link file if it does not exist or is out of date
Renaud Paquay227ad2e2016-11-01 14:37:13 -0700403 if not platform_utils.islink(absDest) or (platform_utils.readlink(absDest) != relSrc):
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500404 try:
405 # remove existing file first, since it might be read-only
Dan Willemsene1e0bd12015-11-18 16:49:38 -0800406 if os.path.lexists(absDest):
Renaud Paquay010fed72016-11-11 14:25:29 -0800407 platform_utils.remove(absDest)
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500408 else:
Wink Saville4c426ef2015-06-03 08:05:17 -0700409 dest_dir = os.path.dirname(absDest)
Renaud Paquaybed8b622018-09-27 10:46:58 -0700410 if not platform_utils.isdir(dest_dir):
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500411 os.makedirs(dest_dir)
Renaud Paquayd5cec5e2016-11-01 11:24:03 -0700412 platform_utils.symlink(relSrc, absDest)
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500413 except IOError:
Wink Saville4c426ef2015-06-03 08:05:17 -0700414 _error('Cannot link file %s to %s', relSrc, absDest)
415
416 def _Link(self):
Mike Frysingere6a202f2019-08-02 15:57:57 -0400417 """Link the self.src & self.dest paths.
418
419 Handles wild cards on the src linking all of the files in the source in to
420 the destination directory.
Wink Saville4c426ef2015-06-03 08:05:17 -0700421 """
Mike Frysinger07392ed2020-02-10 21:35:48 -0500422 # Some people use src="." to create stable links to projects. Lets allow
423 # that but reject all other uses of "." to keep things simple.
424 if self.src == '.':
425 src = self.git_worktree
426 else:
427 src = _SafeExpandPath(self.git_worktree, self.src)
Mike Frysingere6a202f2019-08-02 15:57:57 -0400428
Angel Petkovdbfbcb12020-05-02 23:16:20 +0300429 if not glob.has_magic(src):
430 # Entity does not contain a wild card so just a simple one to one link operation.
Mike Frysingere6a202f2019-08-02 15:57:57 -0400431 dest = _SafeExpandPath(self.topdir, self.dest, skipfinal=True)
432 # dest & src are absolute paths at this point. Make sure the target of
433 # the symlink is relative in the context of the repo client checkout.
434 relpath = os.path.relpath(src, os.path.dirname(dest))
435 self.__linkIt(relpath, dest)
Wink Saville4c426ef2015-06-03 08:05:17 -0700436 else:
Mike Frysingere6a202f2019-08-02 15:57:57 -0400437 dest = _SafeExpandPath(self.topdir, self.dest)
Angel Petkovdbfbcb12020-05-02 23:16:20 +0300438 # Entity contains a wild card.
Mike Frysingere6a202f2019-08-02 15:57:57 -0400439 if os.path.exists(dest) and not platform_utils.isdir(dest):
440 _error('Link error: src with wildcard, %s must be a directory', dest)
Wink Saville4c426ef2015-06-03 08:05:17 -0700441 else:
Mike Frysingere6a202f2019-08-02 15:57:57 -0400442 for absSrcFile in glob.glob(src):
Wink Saville4c426ef2015-06-03 08:05:17 -0700443 # Create a releative path from source dir to destination dir
444 absSrcDir = os.path.dirname(absSrcFile)
Mike Frysingere6a202f2019-08-02 15:57:57 -0400445 relSrcDir = os.path.relpath(absSrcDir, dest)
Wink Saville4c426ef2015-06-03 08:05:17 -0700446
447 # Get the source file name
448 srcFile = os.path.basename(absSrcFile)
449
450 # Now form the final full paths to srcFile. They will be
451 # absolute for the desintaiton and relative for the srouce.
Mike Frysingere6a202f2019-08-02 15:57:57 -0400452 absDest = os.path.join(dest, srcFile)
Wink Saville4c426ef2015-06-03 08:05:17 -0700453 relSrc = os.path.join(relSrcDir, srcFile)
454 self.__linkIt(relSrc, absDest)
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500455
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700456
Shawn O. Pearced1f70d92009-05-19 14:58:02 -0700457class RemoteSpec(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700458
Shawn O. Pearced1f70d92009-05-19 14:58:02 -0700459 def __init__(self,
460 name,
Anthony King7bdac712014-07-16 12:56:40 +0100461 url=None,
Steve Raed6480452016-08-10 15:00:00 -0700462 pushUrl=None,
Anthony King7bdac712014-07-16 12:56:40 +0100463 review=None,
Dan Willemsen96c2d652016-04-06 16:03:54 -0700464 revision=None,
David Rileye0684ad2017-04-05 00:02:59 -0700465 orig_name=None,
466 fetchUrl=None):
Shawn O. Pearced1f70d92009-05-19 14:58:02 -0700467 self.name = name
468 self.url = url
Steve Raed6480452016-08-10 15:00:00 -0700469 self.pushUrl = pushUrl
Shawn O. Pearced1f70d92009-05-19 14:58:02 -0700470 self.review = review
Anthony King36ea2fb2014-05-06 11:54:01 +0100471 self.revision = revision
Dan Willemsen96c2d652016-04-06 16:03:54 -0700472 self.orig_name = orig_name
David Rileye0684ad2017-04-05 00:02:59 -0700473 self.fetchUrl = fetchUrl
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700474
Ian Kasprzak0286e312021-02-05 10:06:18 -0800475
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700476class Project(object):
Kevin Degi384b3c52014-10-16 16:02:58 -0600477 # These objects can be shared between several working trees.
LaMont Jones68d69632022-06-07 18:24:20 +0000478 @property
479 def shareable_dirs(self):
480 """Return the shareable directories"""
481 if self.UseAlternates:
482 return ['hooks', 'rr-cache']
483 else:
484 return ['hooks', 'objects', 'rr-cache']
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700485
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700486 def __init__(self,
487 manifest,
488 name,
489 remote,
490 gitdir,
David James8d201162013-10-11 17:03:19 -0700491 objdir,
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700492 worktree,
493 relpath,
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700494 revisionExpr,
Mike Pontillod3153822012-02-28 11:53:24 -0800495 revisionId,
Anthony King7bdac712014-07-16 12:56:40 +0100496 rebase=True,
497 groups=None,
498 sync_c=False,
499 sync_s=False,
YOUNG HO CHAa32c92c2018-02-14 16:57:31 +0900500 sync_tags=True,
Anthony King7bdac712014-07-16 12:56:40 +0100501 clone_depth=None,
502 upstream=None,
503 parent=None,
Mike Frysinger979d5bd2020-02-09 02:28:34 -0500504 use_git_worktrees=False,
Anthony King7bdac712014-07-16 12:56:40 +0100505 is_derived=False,
David Pursehouseb1553542014-09-04 21:28:09 +0900506 dest_branch=None,
Simran Basib9a1b732015-08-20 12:19:28 -0700507 optimized_fetch=False,
George Engelbrecht9bc283e2020-04-02 12:36:09 -0600508 retry_fetches=0,
Simran Basib9a1b732015-08-20 12:19:28 -0700509 old_revision=None):
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800510 """Init a Project object.
511
512 Args:
513 manifest: The XmlManifest object.
514 name: The `name` attribute of manifest.xml's project element.
515 remote: RemoteSpec object specifying its remote's properties.
516 gitdir: Absolute path of git directory.
David James8d201162013-10-11 17:03:19 -0700517 objdir: Absolute path of directory to store git objects.
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800518 worktree: Absolute path of git working tree.
519 relpath: Relative path of git working tree to repo's top directory.
520 revisionExpr: The `revision` attribute of manifest.xml's project element.
521 revisionId: git commit id for checking out.
522 rebase: The `rebase` attribute of manifest.xml's project element.
523 groups: The `groups` attribute of manifest.xml's project element.
524 sync_c: The `sync-c` attribute of manifest.xml's project element.
525 sync_s: The `sync-s` attribute of manifest.xml's project element.
YOUNG HO CHAa32c92c2018-02-14 16:57:31 +0900526 sync_tags: The `sync-tags` attribute of manifest.xml's project element.
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800527 upstream: The `upstream` attribute of manifest.xml's project element.
528 parent: The parent Project object.
Mike Frysinger979d5bd2020-02-09 02:28:34 -0500529 use_git_worktrees: Whether to use `git worktree` for this project.
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800530 is_derived: False if the project was explicitly defined in the manifest;
531 True if the project is a discovered submodule.
Bryan Jacobsf609f912013-05-06 13:36:24 -0400532 dest_branch: The branch to which to push changes for review by default.
David Pursehouseb1553542014-09-04 21:28:09 +0900533 optimized_fetch: If True, when a project is set to a sha1 revision, only
534 fetch from the remote if the sha1 is not present locally.
George Engelbrecht9bc283e2020-04-02 12:36:09 -0600535 retry_fetches: Retry remote fetches n times upon receiving transient error
536 with exponential backoff and jitter.
Simran Basib9a1b732015-08-20 12:19:28 -0700537 old_revision: saved git commit id for open GITC projects.
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800538 """
Mike Frysinger8c1e9cb2020-09-06 14:53:18 -0400539 self.client = self.manifest = manifest
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700540 self.name = name
541 self.remote = remote
Michael Kelly37c21c22020-06-13 02:10:40 -0700542 self.UpdatePaths(relpath, worktree, gitdir, objdir)
Michael Kelly2f3c3312020-07-21 19:40:38 -0700543 self.SetRevision(revisionExpr, revisionId=revisionId)
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700544
Mike Pontillod3153822012-02-28 11:53:24 -0800545 self.rebase = rebase
Colin Cross5acde752012-03-28 20:15:45 -0700546 self.groups = groups
Anatol Pomazau79770d22012-04-20 14:41:59 -0700547 self.sync_c = sync_c
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800548 self.sync_s = sync_s
YOUNG HO CHAa32c92c2018-02-14 16:57:31 +0900549 self.sync_tags = sync_tags
David Pursehouseede7f122012-11-27 22:25:30 +0900550 self.clone_depth = clone_depth
Brian Harring14a66742012-09-28 20:21:57 -0700551 self.upstream = upstream
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800552 self.parent = parent
Mike Frysinger979d5bd2020-02-09 02:28:34 -0500553 # NB: Do not use this setting in __init__ to change behavior so that the
554 # manifest.git checkout can inspect & change it after instantiating. See
555 # the XmlManifest init code for more info.
556 self.use_git_worktrees = use_git_worktrees
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800557 self.is_derived = is_derived
David Pursehouseb1553542014-09-04 21:28:09 +0900558 self.optimized_fetch = optimized_fetch
George Engelbrecht9bc283e2020-04-02 12:36:09 -0600559 self.retry_fetches = max(0, retry_fetches)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800560 self.subprojects = []
Mike Pontillod3153822012-02-28 11:53:24 -0800561
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700562 self.snapshots = {}
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700563 self.copyfiles = []
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500564 self.linkfiles = []
James W. Mills24c13082012-04-12 15:04:13 -0500565 self.annotations = []
Bryan Jacobsf609f912013-05-06 13:36:24 -0400566 self.dest_branch = dest_branch
Simran Basib9a1b732015-08-20 12:19:28 -0700567 self.old_revision = old_revision
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700568
Doug Anderson37282b42011-03-04 11:54:18 -0800569 # This will be filled in if a project is later identified to be the
570 # project containing repo hooks.
571 self.enabled_repo_hooks = []
572
LaMont Jonescc879a92021-11-18 22:40:18 +0000573 def RelPath(self, local=True):
574 """Return the path for the project relative to a manifest.
575
576 Args:
577 local: a boolean, if True, the path is relative to the local
578 (sub)manifest. If false, the path is relative to the
579 outermost manifest.
580 """
581 if local:
582 return self.relpath
583 return os.path.join(self.manifest.path_prefix, self.relpath)
584
Michael Kelly2f3c3312020-07-21 19:40:38 -0700585 def SetRevision(self, revisionExpr, revisionId=None):
586 """Set revisionId based on revision expression and id"""
587 self.revisionExpr = revisionExpr
588 if revisionId is None and revisionExpr and IsId(revisionExpr):
589 self.revisionId = self.revisionExpr
590 else:
591 self.revisionId = revisionId
592
Michael Kelly37c21c22020-06-13 02:10:40 -0700593 def UpdatePaths(self, relpath, worktree, gitdir, objdir):
594 """Update paths used by this project"""
595 self.gitdir = gitdir.replace('\\', '/')
596 self.objdir = objdir.replace('\\', '/')
597 if worktree:
598 self.worktree = os.path.normpath(worktree).replace('\\', '/')
599 else:
600 self.worktree = None
601 self.relpath = relpath
602
603 self.config = GitConfig.ForRepository(gitdir=self.gitdir,
604 defaults=self.manifest.globalConfig)
605
606 if self.worktree:
607 self.work_git = self._GitGetByExec(self, bare=False, gitdir=self.gitdir)
608 else:
609 self.work_git = None
610 self.bare_git = self._GitGetByExec(self, bare=True, gitdir=self.gitdir)
611 self.bare_ref = GitRefs(self.gitdir)
612 self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=self.objdir)
613
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700614 @property
LaMont Jones68d69632022-06-07 18:24:20 +0000615 def UseAlternates(self):
616 """Whether git alternates are in use.
617
618 This will be removed once migration to alternates is complete.
619 """
620 return _ALTERNATES or self.manifest.is_multimanifest
621
622 @property
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800623 def Derived(self):
624 return self.is_derived
625
626 @property
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700627 def Exists(self):
Renaud Paquaybed8b622018-09-27 10:46:58 -0700628 return platform_utils.isdir(self.gitdir) and platform_utils.isdir(self.objdir)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700629
630 @property
631 def CurrentBranch(self):
632 """Obtain the name of the currently checked out branch.
Mike Frysingerc8290ad2019-10-01 01:07:11 -0400633
634 The branch name omits the 'refs/heads/' prefix.
635 None is returned if the project is on a detached HEAD, or if the work_git is
636 otheriwse inaccessible (e.g. an incomplete sync).
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700637 """
Mike Frysingerc8290ad2019-10-01 01:07:11 -0400638 try:
639 b = self.work_git.GetHead()
640 except NoManifestException:
641 # If the local checkout is in a bad state, don't barf. Let the callers
642 # process this like the head is unreadable.
643 return None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700644 if b.startswith(R_HEADS):
645 return b[len(R_HEADS):]
646 return None
647
Shawn O. Pearce3d2cdd02009-04-18 15:26:10 -0700648 def IsRebaseInProgress(self):
Mike Frysinger4b0eb5a2020-02-23 23:22:34 -0500649 return (os.path.exists(self.work_git.GetDotgitPath('rebase-apply')) or
650 os.path.exists(self.work_git.GetDotgitPath('rebase-merge')) or
651 os.path.exists(os.path.join(self.worktree, '.dotest')))
Julius Gustavsson0cb1b3f2010-06-17 17:55:02 +0200652
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700653 def IsDirty(self, consider_untracked=True):
654 """Is the working directory modified in some way?
655 """
656 self.work_git.update_index('-q',
657 '--unmerged',
658 '--ignore-missing',
659 '--refresh')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900660 if self.work_git.DiffZ('diff-index', '-M', '--cached', HEAD):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700661 return True
662 if self.work_git.DiffZ('diff-files'):
663 return True
Martin Geisler9fb64ae2022-07-08 10:50:10 +0200664 if consider_untracked and self.UntrackedFiles():
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700665 return True
666 return False
667
668 _userident_name = None
669 _userident_email = None
670
671 @property
672 def UserName(self):
673 """Obtain the user's personal name.
674 """
675 if self._userident_name is None:
676 self._LoadUserIdentity()
677 return self._userident_name
678
679 @property
680 def UserEmail(self):
681 """Obtain the user's email address. This is very likely
682 to be their Gerrit login.
683 """
684 if self._userident_email is None:
685 self._LoadUserIdentity()
686 return self._userident_email
687
688 def _LoadUserIdentity(self):
David Pursehousec1b86a22012-11-14 11:36:51 +0900689 u = self.bare_git.var('GIT_COMMITTER_IDENT')
690 m = re.compile("^(.*) <([^>]*)> ").match(u)
691 if m:
692 self._userident_name = m.group(1)
693 self._userident_email = m.group(2)
694 else:
695 self._userident_name = ''
696 self._userident_email = ''
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700697
Mike Frysingerdede5642022-07-10 04:56:04 -0400698 def GetRemote(self, name=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700699 """Get the configuration for a single remote.
Mike Frysingerdede5642022-07-10 04:56:04 -0400700
701 Defaults to the current project's remote.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700702 """
Mike Frysingerdede5642022-07-10 04:56:04 -0400703 if name is None:
704 name = self.remote.name
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700705 return self.config.GetRemote(name)
706
707 def GetBranch(self, name):
708 """Get the configuration for a single branch.
709 """
710 return self.config.GetBranch(name)
711
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700712 def GetBranches(self):
713 """Get all existing local branches.
714 """
715 current = self.CurrentBranch
David Pursehouse8a68ff92012-09-24 12:15:13 +0900716 all_refs = self._allrefs
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700717 heads = {}
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700718
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530719 for name, ref_id in all_refs.items():
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700720 if name.startswith(R_HEADS):
721 name = name[len(R_HEADS):]
722 b = self.GetBranch(name)
723 b.current = name == current
724 b.published = None
David Pursehouse8a68ff92012-09-24 12:15:13 +0900725 b.revision = ref_id
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700726 heads[name] = b
727
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530728 for name, ref_id in all_refs.items():
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700729 if name.startswith(R_PUB):
730 name = name[len(R_PUB):]
731 b = heads.get(name)
732 if b:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900733 b.published = ref_id
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700734
735 return heads
736
Colin Cross5acde752012-03-28 20:15:45 -0700737 def MatchesGroups(self, manifest_groups):
738 """Returns true if the manifest groups specified at init should cause
739 this project to be synced.
740 Prefixing a manifest group with "-" inverts the meaning of a group.
Conley Owensbb1b5f52012-08-13 13:11:18 -0700741 All projects are implicitly labelled with "all".
Conley Owens971de8e2012-04-16 10:36:08 -0700742
743 labels are resolved in order. In the example case of
Conley Owensbb1b5f52012-08-13 13:11:18 -0700744 project_groups: "all,group1,group2"
Conley Owens971de8e2012-04-16 10:36:08 -0700745 manifest_groups: "-group1,group2"
746 the project will be matched.
David Holmer0a1c6a12012-11-14 19:19:00 -0500747
748 The special manifest group "default" will match any project that
749 does not have the special project group "notdefault"
Colin Cross5acde752012-03-28 20:15:45 -0700750 """
LaMont Jones501733c2022-04-20 16:42:32 +0000751 default_groups = self.manifest.default_groups or ['default']
752 expanded_manifest_groups = manifest_groups or default_groups
Conley Owensbb1b5f52012-08-13 13:11:18 -0700753 expanded_project_groups = ['all'] + (self.groups or [])
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700754 if 'notdefault' not in expanded_project_groups:
David Holmer0a1c6a12012-11-14 19:19:00 -0500755 expanded_project_groups += ['default']
Conley Owensbb1b5f52012-08-13 13:11:18 -0700756
Conley Owens971de8e2012-04-16 10:36:08 -0700757 matched = False
Conley Owensbb1b5f52012-08-13 13:11:18 -0700758 for group in expanded_manifest_groups:
759 if group.startswith('-') and group[1:] in expanded_project_groups:
Conley Owens971de8e2012-04-16 10:36:08 -0700760 matched = False
Conley Owensbb1b5f52012-08-13 13:11:18 -0700761 elif group in expanded_project_groups:
Conley Owens971de8e2012-04-16 10:36:08 -0700762 matched = True
Colin Cross5acde752012-03-28 20:15:45 -0700763
Conley Owens971de8e2012-04-16 10:36:08 -0700764 return matched
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700765
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700766# Status Display ##
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700767 def UncommitedFiles(self, get_all=True):
768 """Returns a list of strings, uncommitted files in the git tree.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700769
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700770 Args:
771 get_all: a boolean, if True - get information about all different
772 uncommitted files. If False - return as soon as any kind of
773 uncommitted files is detected.
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500774 """
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700775 details = []
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500776 self.work_git.update_index('-q',
777 '--unmerged',
778 '--ignore-missing',
779 '--refresh')
780 if self.IsRebaseInProgress():
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700781 details.append("rebase in progress")
782 if not get_all:
783 return details
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500784
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700785 changes = self.work_git.DiffZ('diff-index', '--cached', HEAD).keys()
786 if changes:
787 details.extend(changes)
788 if not get_all:
789 return details
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500790
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700791 changes = self.work_git.DiffZ('diff-files').keys()
792 if changes:
793 details.extend(changes)
794 if not get_all:
795 return details
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500796
Martin Geisler9fb64ae2022-07-08 10:50:10 +0200797 changes = self.UntrackedFiles()
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700798 if changes:
799 details.extend(changes)
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500800
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700801 return details
802
Martin Geisler9fb64ae2022-07-08 10:50:10 +0200803 def UntrackedFiles(self):
804 """Returns a list of strings, untracked files in the git tree."""
805 return self.work_git.LsOthers()
806
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700807 def HasChanges(self):
808 """Returns true if there are uncommitted changes.
809 """
Martin Geisler8db78c72022-07-08 11:05:24 +0200810 return bool(self.UncommitedFiles(get_all=False))
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500811
LaMont Jones8501d462022-06-22 19:21:15 +0000812 def PrintWorkTreeStatus(self, output_redir=None, quiet=False, local=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700813 """Prints the status of the repository to stdout.
Terence Haddock4655e812011-03-31 12:33:34 +0200814
815 Args:
Rostislav Krasny0bcc2d22020-01-25 14:49:14 +0200816 output_redir: If specified, redirect the output to this object.
Andrew Wheeler4d5bb682012-02-27 13:52:22 -0600817 quiet: If True then only print the project name. Do not print
818 the modified files, branch name, etc.
LaMont Jones8501d462022-06-22 19:21:15 +0000819 local: a boolean, if True, the path is relative to the local
820 (sub)manifest. If false, the path is relative to the
821 outermost manifest.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700822 """
Renaud Paquaybed8b622018-09-27 10:46:58 -0700823 if not platform_utils.isdir(self.worktree):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700824 if output_redir is None:
Terence Haddock4655e812011-03-31 12:33:34 +0200825 output_redir = sys.stdout
Sarah Owenscecd1d82012-11-01 22:59:27 -0700826 print(file=output_redir)
LaMont Jones8501d462022-06-22 19:21:15 +0000827 print('project %s/' % self.RelPath(local), file=output_redir)
Sarah Owenscecd1d82012-11-01 22:59:27 -0700828 print(' missing (run "repo sync")', file=output_redir)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700829 return
830
831 self.work_git.update_index('-q',
832 '--unmerged',
833 '--ignore-missing',
834 '--refresh')
Shawn O. Pearce3d2cdd02009-04-18 15:26:10 -0700835 rb = self.IsRebaseInProgress()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700836 di = self.work_git.DiffZ('diff-index', '-M', '--cached', HEAD)
837 df = self.work_git.DiffZ('diff-files')
838 do = self.work_git.LsOthers()
Ali Utku Selen76abcc12012-01-25 10:51:12 +0100839 if not rb and not di and not df and not do and not self.CurrentBranch:
Shawn O. Pearce161f4452009-04-10 17:41:44 -0700840 return 'CLEAN'
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700841
842 out = StatusColoring(self.config)
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700843 if output_redir is not None:
Terence Haddock4655e812011-03-31 12:33:34 +0200844 out.redirect(output_redir)
LaMont Jones8501d462022-06-22 19:21:15 +0000845 out.project('project %-40s', self.RelPath(local) + '/ ')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700846
Andrew Wheeler4d5bb682012-02-27 13:52:22 -0600847 if quiet:
848 out.nl()
849 return 'DIRTY'
850
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700851 branch = self.CurrentBranch
852 if branch is None:
853 out.nobranch('(*** NO BRANCH ***)')
854 else:
855 out.branch('branch %s', branch)
856 out.nl()
857
Shawn O. Pearce3d2cdd02009-04-18 15:26:10 -0700858 if rb:
859 out.important('prior sync failed; rebase still in progress')
860 out.nl()
861
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700862 paths = list()
863 paths.extend(di.keys())
864 paths.extend(df.keys())
865 paths.extend(do)
866
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530867 for p in sorted(set(paths)):
David Pursehouse5c6eeac2012-10-11 16:44:48 +0900868 try:
869 i = di[p]
870 except KeyError:
871 i = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700872
David Pursehouse5c6eeac2012-10-11 16:44:48 +0900873 try:
874 f = df[p]
875 except KeyError:
876 f = None
Julius Gustavsson0cb1b3f2010-06-17 17:55:02 +0200877
David Pursehouse5c6eeac2012-10-11 16:44:48 +0900878 if i:
879 i_status = i.status.upper()
880 else:
881 i_status = '-'
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700882
David Pursehouse5c6eeac2012-10-11 16:44:48 +0900883 if f:
884 f_status = f.status.lower()
885 else:
886 f_status = '-'
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700887
888 if i and i.src_path:
Shawn O. Pearcefe086752009-03-03 13:49:48 -0800889 line = ' %s%s\t%s => %s (%s%%)' % (i_status, f_status,
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700890 i.src_path, p, i.level)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700891 else:
892 line = ' %s%s\t%s' % (i_status, f_status, p)
893
894 if i and not f:
895 out.added('%s', line)
896 elif (i and f) or (not i and f):
897 out.changed('%s', line)
898 elif not i and not f:
899 out.untracked('%s', line)
900 else:
901 out.write('%s', line)
902 out.nl()
Terence Haddock4655e812011-03-31 12:33:34 +0200903
Shawn O. Pearce161f4452009-04-10 17:41:44 -0700904 return 'DIRTY'
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700905
LaMont Jones8501d462022-06-22 19:21:15 +0000906 def PrintWorkTreeDiff(self, absolute_paths=False, output_redir=None,
907 local=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700908 """Prints the status of the repository to stdout.
909 """
910 out = DiffColoring(self.config)
Mike Frysinger69b4a9c2021-02-16 18:18:01 -0500911 if output_redir:
912 out.redirect(output_redir)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700913 cmd = ['diff']
914 if out.is_on:
915 cmd.append('--color')
916 cmd.append(HEAD)
pelyad67872d2012-03-28 14:49:58 +0300917 if absolute_paths:
LaMont Jones8501d462022-06-22 19:21:15 +0000918 cmd.append('--src-prefix=a/%s/' % self.RelPath(local))
919 cmd.append('--dst-prefix=b/%s/' % self.RelPath(local))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700920 cmd.append('--')
Mike Frysinger0a9265e2019-09-30 23:59:27 -0400921 try:
922 p = GitCommand(self,
923 cmd,
924 capture_stdout=True,
925 capture_stderr=True)
Mike Frysinger84230002021-02-16 17:08:35 -0500926 p.Wait()
Mike Frysinger0a9265e2019-09-30 23:59:27 -0400927 except GitError as e:
928 out.nl()
LaMont Jones8501d462022-06-22 19:21:15 +0000929 out.project('project %s/' % self.RelPath(local))
Mike Frysinger0a9265e2019-09-30 23:59:27 -0400930 out.nl()
931 out.fail('%s', str(e))
932 out.nl()
933 return False
Mike Frysinger84230002021-02-16 17:08:35 -0500934 if p.stdout:
935 out.nl()
LaMont Jones8501d462022-06-22 19:21:15 +0000936 out.project('project %s/' % self.RelPath(local))
Mike Frysinger84230002021-02-16 17:08:35 -0500937 out.nl()
Mike Frysinger9888acc2021-03-09 11:31:14 -0500938 out.write('%s', p.stdout)
Mike Frysinger0a9265e2019-09-30 23:59:27 -0400939 return p.Wait() == 0
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700940
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700941# Publish / Upload ##
David Pursehouse8a68ff92012-09-24 12:15:13 +0900942 def WasPublished(self, branch, all_refs=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700943 """Was the branch published (uploaded) for code review?
944 If so, returns the SHA-1 hash of the last published
945 state for the branch.
946 """
Shawn O. Pearcefbcde472009-04-17 20:58:02 -0700947 key = R_PUB + branch
David Pursehouse8a68ff92012-09-24 12:15:13 +0900948 if all_refs is None:
Shawn O. Pearcefbcde472009-04-17 20:58:02 -0700949 try:
950 return self.bare_git.rev_parse(key)
951 except GitError:
952 return None
953 else:
954 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900955 return all_refs[key]
Shawn O. Pearcefbcde472009-04-17 20:58:02 -0700956 except KeyError:
957 return None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700958
David Pursehouse8a68ff92012-09-24 12:15:13 +0900959 def CleanPublishedCache(self, all_refs=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700960 """Prunes any stale published refs.
961 """
David Pursehouse8a68ff92012-09-24 12:15:13 +0900962 if all_refs is None:
963 all_refs = self._allrefs
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700964 heads = set()
965 canrm = {}
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530966 for name, ref_id in all_refs.items():
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700967 if name.startswith(R_HEADS):
968 heads.add(name)
969 elif name.startswith(R_PUB):
David Pursehouse8a68ff92012-09-24 12:15:13 +0900970 canrm[name] = ref_id
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700971
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530972 for name, ref_id in canrm.items():
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700973 n = name[len(R_PUB):]
974 if R_HEADS + n not in heads:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900975 self.bare_git.DeleteRef(name, ref_id)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700976
Mandeep Singh Bainesd6c93a22011-05-26 10:34:11 -0700977 def GetUploadableBranches(self, selected_branch=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700978 """List any branches which can be uploaded for review.
979 """
980 heads = {}
981 pubed = {}
982
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530983 for name, ref_id in self._allrefs.items():
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700984 if name.startswith(R_HEADS):
David Pursehouse8a68ff92012-09-24 12:15:13 +0900985 heads[name[len(R_HEADS):]] = ref_id
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700986 elif name.startswith(R_PUB):
David Pursehouse8a68ff92012-09-24 12:15:13 +0900987 pubed[name[len(R_PUB):]] = ref_id
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700988
989 ready = []
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530990 for branch, ref_id in heads.items():
David Pursehouse8a68ff92012-09-24 12:15:13 +0900991 if branch in pubed and pubed[branch] == ref_id:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700992 continue
Mandeep Singh Bainesd6c93a22011-05-26 10:34:11 -0700993 if selected_branch and branch != selected_branch:
994 continue
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700995
Shawn O. Pearce35f25962008-11-11 17:03:13 -0800996 rb = self.GetUploadableBranch(branch)
997 if rb:
998 ready.append(rb)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700999 return ready
1000
Shawn O. Pearce35f25962008-11-11 17:03:13 -08001001 def GetUploadableBranch(self, branch_name):
1002 """Get a single uploadable branch, or None.
1003 """
1004 branch = self.GetBranch(branch_name)
1005 base = branch.LocalMerge
1006 if branch.LocalMerge:
1007 rb = ReviewableBranch(self, branch, base)
1008 if rb.commits:
1009 return rb
1010 return None
1011
Shawn O. Pearcea5ece0e2010-07-15 16:52:42 -07001012 def UploadForReview(self, branch=None,
Anthony King7bdac712014-07-16 12:56:40 +01001013 people=([], []),
Mike Frysinger819cc812020-02-19 02:27:22 -05001014 dryrun=False,
Brian Harring435370c2012-07-28 15:37:04 -07001015 auto_topic=False,
Mike Frysinger84685ba2020-02-19 02:22:22 -05001016 hashtags=(),
Mike Frysingerfc1b18a2020-02-24 15:38:07 -05001017 labels=(),
Changcheng Xiao87984c62017-08-02 16:55:03 +02001018 private=False,
Vadim Bendeburybd8f6582018-10-31 13:48:01 -07001019 notify=None,
Changcheng Xiao87984c62017-08-02 16:55:03 +02001020 wip=False,
William Escandeac76fd32022-08-02 16:05:37 -07001021 ready=False,
Łukasz Gardońbed59ce2017-08-08 10:18:11 +02001022 dest_branch=None,
Masaya Suzuki305a2d02017-11-13 10:48:34 -08001023 validate_certs=True,
1024 push_options=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001025 """Uploads the named branch for code review.
1026 """
1027 if branch is None:
1028 branch = self.CurrentBranch
1029 if branch is None:
1030 raise GitError('not currently on a branch')
1031
1032 branch = self.GetBranch(branch)
1033 if not branch.LocalMerge:
1034 raise GitError('branch %s does not track a remote' % branch.name)
1035 if not branch.remote.review:
1036 raise GitError('remote %s has no review url' % branch.remote.name)
1037
Mike Frysinger3a0a1452022-05-20 12:52:33 -04001038 # Basic validity check on label syntax.
1039 for label in labels:
1040 if not re.match(r'^.+[+-][0-9]+$', label):
1041 raise UploadError(
1042 f'invalid label syntax "{label}": labels use forms like '
1043 'CodeReview+1 or Verified-1')
1044
Bryan Jacobsf609f912013-05-06 13:36:24 -04001045 if dest_branch is None:
1046 dest_branch = self.dest_branch
1047 if dest_branch is None:
1048 dest_branch = branch.merge
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001049 if not dest_branch.startswith(R_HEADS):
1050 dest_branch = R_HEADS + dest_branch
1051
Shawn O. Pearce339ba9f2008-11-06 09:52:51 -08001052 if not branch.remote.projectname:
1053 branch.remote.projectname = self.name
1054 branch.remote.Save()
1055
Łukasz Gardońbed59ce2017-08-08 10:18:11 +02001056 url = branch.remote.ReviewUrl(self.UserEmail, validate_certs)
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001057 if url is None:
1058 raise UploadError('review not configured')
1059 cmd = ['push']
Mike Frysinger819cc812020-02-19 02:27:22 -05001060 if dryrun:
1061 cmd.append('-n')
Shawn O. Pearceb54a3922009-01-05 16:18:58 -08001062
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001063 if url.startswith('ssh://'):
Jonathan Nieder713c5872018-11-05 13:21:52 -08001064 cmd.append('--receive-pack=gerrit receive-pack')
Shawn O. Pearcea5ece0e2010-07-15 16:52:42 -07001065
Masaya Suzuki305a2d02017-11-13 10:48:34 -08001066 for push_option in (push_options or []):
1067 cmd.append('-o')
1068 cmd.append(push_option)
1069
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001070 cmd.append(url)
Shawn O. Pearceb54a3922009-01-05 16:18:58 -08001071
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001072 if dest_branch.startswith(R_HEADS):
1073 dest_branch = dest_branch[len(R_HEADS):]
Brian Harring435370c2012-07-28 15:37:04 -07001074
Mike Frysingerb0fbc7f2020-02-25 02:58:04 -05001075 ref_spec = '%s:refs/for/%s' % (R_HEADS + branch.name, dest_branch)
David Pursehousef25a3702018-11-14 19:01:22 -08001076 opts = []
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001077 if auto_topic:
David Pursehousef25a3702018-11-14 19:01:22 -08001078 opts += ['topic=' + branch.name]
Mike Frysinger84685ba2020-02-19 02:22:22 -05001079 opts += ['t=%s' % p for p in hashtags]
Mike Frysinger3a0a1452022-05-20 12:52:33 -04001080 # NB: No need to encode labels as they've been validated above.
Mike Frysingerfc1b18a2020-02-24 15:38:07 -05001081 opts += ['l=%s' % p for p in labels]
Changcheng Xiao87984c62017-08-02 16:55:03 +02001082
David Pursehousef25a3702018-11-14 19:01:22 -08001083 opts += ['r=%s' % p for p in people[0]]
Jonathan Nieder713c5872018-11-05 13:21:52 -08001084 opts += ['cc=%s' % p for p in people[1]]
Vadim Bendeburybd8f6582018-10-31 13:48:01 -07001085 if notify:
1086 opts += ['notify=' + notify]
Jonathan Nieder713c5872018-11-05 13:21:52 -08001087 if private:
1088 opts += ['private']
1089 if wip:
1090 opts += ['wip']
William Escandeac76fd32022-08-02 16:05:37 -07001091 if ready:
1092 opts += ['ready']
Jonathan Nieder713c5872018-11-05 13:21:52 -08001093 if opts:
1094 ref_spec = ref_spec + '%' + ','.join(opts)
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001095 cmd.append(ref_spec)
1096
Anthony King7bdac712014-07-16 12:56:40 +01001097 if GitCommand(self, cmd, bare=True).Wait() != 0:
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001098 raise UploadError('Upload failed')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001099
Mike Frysingerd7f86832020-11-19 19:18:46 -05001100 if not dryrun:
1101 msg = "posted to %s for %s" % (branch.remote.review, dest_branch)
1102 self.bare_git.UpdateRef(R_PUB + branch.name,
1103 R_HEADS + branch.name,
1104 message=msg)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001105
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001106# Sync ##
Julien Campergue335f5ef2013-10-16 11:02:35 +02001107 def _ExtractArchive(self, tarpath, path=None):
1108 """Extract the given tar on its current location
1109
1110 Args:
1111 - tarpath: The path to the actual tar file
1112
1113 """
1114 try:
1115 with tarfile.open(tarpath, 'r') as tar:
1116 tar.extractall(path=path)
1117 return True
1118 except (IOError, tarfile.TarError) as e:
David Pursehousef33929d2015-08-24 14:39:14 +09001119 _error("Cannot extract archive %s: %s", tarpath, str(e))
Julien Campergue335f5ef2013-10-16 11:02:35 +02001120 return False
1121
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -07001122 def Sync_NetworkHalf(self,
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001123 quiet=False,
Mike Frysinger521d01b2020-02-17 01:51:49 -05001124 verbose=False,
Mike Frysinger7b586f22021-02-23 18:38:39 -05001125 output_redir=None,
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001126 is_new=None,
Mike Frysinger73561142021-05-03 01:10:09 -04001127 current_branch_only=None,
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001128 force_sync=False,
1129 clone_bundle=True,
Mike Frysingerd68ed632021-05-03 01:21:35 -04001130 tags=None,
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001131 archive=False,
1132 optimized_fetch=False,
George Engelbrecht9bc283e2020-04-02 12:36:09 -06001133 retry_fetches=0,
Martin Kellye4e94d22017-03-21 16:05:12 -07001134 prune=False,
Xin Li745be2e2019-06-03 11:24:30 -07001135 submodules=False,
Mike Frysinger19e409c2021-05-05 19:44:35 -04001136 ssh_proxy=None,
Raman Tennetif32f2432021-04-12 20:57:25 -07001137 clone_filter=None,
Raman Tenneticd89ec12021-04-22 09:18:14 -07001138 partial_clone_exclude=set()):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001139 """Perform only the network IO portion of the sync process.
1140 Local working directory/branch state is not affected.
1141 """
Julien Campergue335f5ef2013-10-16 11:02:35 +02001142 if archive and not isinstance(self, MetaProject):
1143 if self.remote.url.startswith(('http://', 'https://')):
David Pursehousef33929d2015-08-24 14:39:14 +09001144 _error("%s: Cannot fetch archives from http/https remotes.", self.name)
LaMont Jones1eddca82022-09-01 15:15:04 +00001145 return SyncNetworkHalfResult(False, False)
Julien Campergue335f5ef2013-10-16 11:02:35 +02001146
1147 name = self.relpath.replace('\\', '/')
1148 name = name.replace('/', '_')
1149 tarpath = '%s.tar' % name
1150 topdir = self.manifest.topdir
1151
1152 try:
1153 self._FetchArchive(tarpath, cwd=topdir)
1154 except GitError as e:
David Pursehousef33929d2015-08-24 14:39:14 +09001155 _error('%s', e)
LaMont Jones1eddca82022-09-01 15:15:04 +00001156 return SyncNetworkHalfResult(False, False)
Julien Campergue335f5ef2013-10-16 11:02:35 +02001157
1158 # From now on, we only need absolute tarpath
1159 tarpath = os.path.join(topdir, tarpath)
1160
1161 if not self._ExtractArchive(tarpath, path=topdir):
LaMont Jones1eddca82022-09-01 15:15:04 +00001162 return SyncNetworkHalfResult(False, True)
Julien Campergue335f5ef2013-10-16 11:02:35 +02001163 try:
Renaud Paquay010fed72016-11-11 14:25:29 -08001164 platform_utils.remove(tarpath)
Julien Campergue335f5ef2013-10-16 11:02:35 +02001165 except OSError as e:
David Pursehousef33929d2015-08-24 14:39:14 +09001166 _warn("Cannot remove archive %s: %s", tarpath, str(e))
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001167 self._CopyAndLinkFiles()
LaMont Jones1eddca82022-09-01 15:15:04 +00001168 return SyncNetworkHalfResult(True, True)
Mike Frysinger76844ba2021-02-28 17:08:55 -05001169
1170 # If the shared object dir already exists, don't try to rebootstrap with a
1171 # clone bundle download. We should have the majority of objects already.
1172 if clone_bundle and os.path.exists(self.objdir):
1173 clone_bundle = False
1174
Raman Tennetif32f2432021-04-12 20:57:25 -07001175 if self.name in partial_clone_exclude:
1176 clone_bundle = True
1177 clone_filter = None
1178
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07001179 if is_new is None:
1180 is_new = not self.Exists
Shawn O. Pearce88443382010-10-08 10:02:09 +02001181 if is_new:
David Pursehousee8ace262020-02-13 12:41:15 +09001182 self._InitGitDir(force_sync=force_sync, quiet=quiet)
Jimmie Westera0444582012-10-24 13:44:42 +02001183 else:
David Pursehousee8ace262020-02-13 12:41:15 +09001184 self._UpdateHooks(quiet=quiet)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001185 self._InitRemote()
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001186
LaMont Jones68d69632022-06-07 18:24:20 +00001187 if self.UseAlternates:
Mike Frysinger1d00a7e2021-12-21 00:40:31 -05001188 # If gitdir/objects is a symlink, migrate it from the old layout.
1189 gitdir_objects = os.path.join(self.gitdir, 'objects')
1190 if platform_utils.islink(gitdir_objects):
1191 platform_utils.remove(gitdir_objects, missing_ok=True)
1192 gitdir_alt = os.path.join(self.gitdir, 'objects/info/alternates')
1193 if not os.path.exists(gitdir_alt):
1194 os.makedirs(os.path.dirname(gitdir_alt), exist_ok=True)
1195 _lwrite(gitdir_alt, os.path.join(
1196 os.path.relpath(self.objdir, gitdir_objects), 'objects') + '\n')
1197
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001198 if is_new:
Mike Frysinger152032c2021-12-20 21:17:43 -05001199 alt = os.path.join(self.objdir, 'objects/info/alternates')
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001200 try:
Mike Frysinger3164d402019-11-11 05:40:22 -05001201 with open(alt) as fd:
Samuel Hollandbaa00092018-01-22 10:57:29 -06001202 # This works for both absolute and relative alternate directories.
1203 alt_dir = os.path.join(self.objdir, 'objects', fd.readline().rstrip())
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001204 except IOError:
1205 alt_dir = None
1206 else:
1207 alt_dir = None
1208
Mike Frysingere50b6a72020-02-19 01:45:48 -05001209 if (clone_bundle
1210 and alt_dir is None
1211 and self._ApplyCloneBundle(initial=is_new, quiet=quiet, verbose=verbose)):
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001212 is_new = False
1213
Mike Frysinger73561142021-05-03 01:10:09 -04001214 if current_branch_only is None:
Shawn O. Pearce6ba6ba02012-05-24 09:46:50 -07001215 if self.sync_c:
1216 current_branch_only = True
1217 elif not self.manifest._loaded:
1218 # Manifest cannot check defaults until it syncs.
1219 current_branch_only = False
1220 elif self.manifest.default.sync_c:
1221 current_branch_only = True
1222
Mike Frysingerd68ed632021-05-03 01:21:35 -04001223 if tags is None:
1224 tags = self.sync_tags
YOUNG HO CHAa32c92c2018-02-14 16:57:31 +09001225
Aymen Bouaziz6c594462016-10-25 18:03:51 +02001226 if self.clone_depth:
1227 depth = self.clone_depth
1228 else:
LaMont Jonesd82be3e2022-04-05 19:30:46 +00001229 depth = self.manifest.manifestProject.depth
Aymen Bouaziz6c594462016-10-25 18:03:51 +02001230
Mike Frysinger521d01b2020-02-17 01:51:49 -05001231 # See if we can skip the network fetch entirely.
LaMont Jones1eddca82022-09-01 15:15:04 +00001232 remote_fetched = False
Mike Frysinger521d01b2020-02-17 01:51:49 -05001233 if not (optimized_fetch and
1234 (ID_RE.match(self.revisionExpr) and
1235 self._CheckForImmutableRevision())):
LaMont Jones1eddca82022-09-01 15:15:04 +00001236 remote_fetched = True
Mike Frysinger521d01b2020-02-17 01:51:49 -05001237 if not self._RemoteFetch(
Mike Frysinger7b586f22021-02-23 18:38:39 -05001238 initial=is_new,
1239 quiet=quiet, verbose=verbose, output_redir=output_redir,
1240 alt_dir=alt_dir, current_branch_only=current_branch_only,
Mike Frysingerc58ec4d2020-02-17 14:36:08 -05001241 tags=tags, prune=prune, depth=depth,
David Pursehouse3cceda52020-02-18 14:11:39 +09001242 submodules=submodules, force_sync=force_sync,
Mike Frysinger19e409c2021-05-05 19:44:35 -04001243 ssh_proxy=ssh_proxy,
George Engelbrecht9bc283e2020-04-02 12:36:09 -06001244 clone_filter=clone_filter, retry_fetches=retry_fetches):
LaMont Jones1eddca82022-09-01 15:15:04 +00001245 return SyncNetworkHalfResult(False, remote_fetched)
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08001246
Nikolai Merinov09f0abb2018-10-19 15:07:05 +05001247 mp = self.manifest.manifestProject
LaMont Jonesd82be3e2022-04-05 19:30:46 +00001248 dissociate = mp.dissociate
Nikolai Merinov09f0abb2018-10-19 15:07:05 +05001249 if dissociate:
Mike Frysinger152032c2021-12-20 21:17:43 -05001250 alternates_file = os.path.join(self.objdir, 'objects/info/alternates')
Nikolai Merinov09f0abb2018-10-19 15:07:05 +05001251 if os.path.exists(alternates_file):
1252 cmd = ['repack', '-a', '-d']
Mike Frysinger7b586f22021-02-23 18:38:39 -05001253 p = GitCommand(self, cmd, bare=True, capture_stdout=bool(output_redir),
1254 merge_output=bool(output_redir))
1255 if p.stdout and output_redir:
Mike Frysinger3c093122021-03-03 11:17:27 -05001256 output_redir.write(p.stdout)
Mike Frysinger7b586f22021-02-23 18:38:39 -05001257 if p.Wait() != 0:
LaMont Jones1eddca82022-09-01 15:15:04 +00001258 return SyncNetworkHalfResult(False, remote_fetched)
Nikolai Merinov09f0abb2018-10-19 15:07:05 +05001259 platform_utils.remove(alternates_file)
1260
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08001261 if self.worktree:
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08001262 self._InitMRef()
1263 else:
1264 self._InitMirrorHead()
Mike Frysinger9d96f582021-09-28 11:27:24 -04001265 platform_utils.remove(os.path.join(self.gitdir, 'FETCH_HEAD'),
1266 missing_ok=True)
LaMont Jones1eddca82022-09-01 15:15:04 +00001267 return SyncNetworkHalfResult(True, remote_fetched)
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08001268
1269 def PostRepoUpgrade(self):
1270 self._InitHooks()
1271
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001272 def _CopyAndLinkFiles(self):
Mike Frysinger8c1e9cb2020-09-06 14:53:18 -04001273 if self.client.isGitcClient:
Simran Basib9a1b732015-08-20 12:19:28 -07001274 return
David Pursehouse8a68ff92012-09-24 12:15:13 +09001275 for copyfile in self.copyfiles:
1276 copyfile._Copy()
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001277 for linkfile in self.linkfiles:
1278 linkfile._Link()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001279
Julien Camperguedd654222014-01-09 16:21:37 +01001280 def GetCommitRevisionId(self):
1281 """Get revisionId of a commit.
1282
1283 Use this method instead of GetRevisionId to get the id of the commit rather
1284 than the id of the current git object (for example, a tag)
1285
1286 """
1287 if not self.revisionExpr.startswith(R_TAGS):
1288 return self.GetRevisionId(self._allrefs)
1289
1290 try:
1291 return self.bare_git.rev_list(self.revisionExpr, '-1')[0]
1292 except GitError:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001293 raise ManifestInvalidRevisionError('revision %s in %s not found' %
1294 (self.revisionExpr, self.name))
Julien Camperguedd654222014-01-09 16:21:37 +01001295
David Pursehouse8a68ff92012-09-24 12:15:13 +09001296 def GetRevisionId(self, all_refs=None):
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001297 if self.revisionId:
1298 return self.revisionId
1299
Mike Frysingerdede5642022-07-10 04:56:04 -04001300 rem = self.GetRemote()
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001301 rev = rem.ToLocal(self.revisionExpr)
1302
David Pursehouse8a68ff92012-09-24 12:15:13 +09001303 if all_refs is not None and rev in all_refs:
1304 return all_refs[rev]
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001305
1306 try:
1307 return self.bare_git.rev_parse('--verify', '%s^0' % rev)
1308 except GitError:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001309 raise ManifestInvalidRevisionError('revision %s in %s not found' %
1310 (self.revisionExpr, self.name))
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001311
Raman Tenneti6a872c92021-01-14 19:17:50 -08001312 def SetRevisionId(self, revisionId):
Xin Li0e776a52021-06-29 21:42:34 +00001313 if self.revisionExpr:
Xin Liaabf79d2021-04-29 01:50:38 -07001314 self.upstream = self.revisionExpr
1315
Raman Tenneti6a872c92021-01-14 19:17:50 -08001316 self.revisionId = revisionId
1317
Martin Kellye4e94d22017-03-21 16:05:12 -07001318 def Sync_LocalHalf(self, syncbuf, force_sync=False, submodules=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001319 """Perform only the local IO portion of the sync process.
1320 Network access is not required.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001321 """
Mike Frysingerb610b852019-11-11 05:10:03 -05001322 if not os.path.exists(self.gitdir):
1323 syncbuf.fail(self,
1324 'Cannot checkout %s due to missing network sync; Run '
1325 '`repo sync -n %s` first.' %
1326 (self.name, self.name))
1327 return
1328
Martin Kellye4e94d22017-03-21 16:05:12 -07001329 self._InitWorkTree(force_sync=force_sync, submodules=submodules)
David Pursehouse8a68ff92012-09-24 12:15:13 +09001330 all_refs = self.bare_ref.all
1331 self.CleanPublishedCache(all_refs)
1332 revid = self.GetRevisionId(all_refs)
Skyler Kaufman835cd682011-03-08 12:14:41 -08001333
Mike Frysinger0458faa2021-03-10 23:35:44 -05001334 # Special case the root of the repo client checkout. Make sure it doesn't
1335 # contain files being checked out to dirs we don't allow.
1336 if self.relpath == '.':
1337 PROTECTED_PATHS = {'.repo'}
1338 paths = set(self.work_git.ls_tree('-z', '--name-only', '--', revid).split('\0'))
1339 bad_paths = paths & PROTECTED_PATHS
1340 if bad_paths:
1341 syncbuf.fail(self,
1342 'Refusing to checkout project that writes to protected '
1343 'paths: %s' % (', '.join(bad_paths),))
1344 return
1345
David Pursehouse1d947b32012-10-25 12:23:11 +09001346 def _doff():
1347 self._FastForward(revid)
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001348 self._CopyAndLinkFiles()
David Pursehouse1d947b32012-10-25 12:23:11 +09001349
Martin Kellye4e94d22017-03-21 16:05:12 -07001350 def _dosubmodules():
1351 self._SyncSubmodules(quiet=True)
1352
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001353 head = self.work_git.GetHead()
1354 if head.startswith(R_HEADS):
1355 branch = head[len(R_HEADS):]
1356 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09001357 head = all_refs[head]
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001358 except KeyError:
1359 head = None
1360 else:
1361 branch = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001362
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001363 if branch is None or syncbuf.detach_head:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001364 # Currently on a detached HEAD. The user is assumed to
1365 # not have any local modifications worth worrying about.
1366 #
Shawn O. Pearce3d2cdd02009-04-18 15:26:10 -07001367 if self.IsRebaseInProgress():
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001368 syncbuf.fail(self, _PriorSyncFailedError())
1369 return
1370
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001371 if head == revid:
1372 # No changes; don't do anything further.
Florian Vallee7cf1b362012-06-07 17:11:42 +02001373 # Except if the head needs to be detached
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001374 #
Florian Vallee7cf1b362012-06-07 17:11:42 +02001375 if not syncbuf.detach_head:
Dan Willemsen029eaf32015-09-03 12:52:28 -07001376 # The copy/linkfile config may have changed.
1377 self._CopyAndLinkFiles()
Florian Vallee7cf1b362012-06-07 17:11:42 +02001378 return
1379 else:
1380 lost = self._revlist(not_rev(revid), HEAD)
1381 if lost:
1382 syncbuf.info(self, "discarding %d commits", len(lost))
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001383
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001384 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001385 self._Checkout(revid, quiet=True)
Martin Kellye4e94d22017-03-21 16:05:12 -07001386 if submodules:
1387 self._SyncSubmodules(quiet=True)
Sarah Owensa5be53f2012-09-09 15:37:57 -07001388 except GitError as e:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001389 syncbuf.fail(self, e)
1390 return
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001391 self._CopyAndLinkFiles()
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001392 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001393
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001394 if head == revid:
1395 # No changes; don't do anything further.
1396 #
Dan Willemsen029eaf32015-09-03 12:52:28 -07001397 # The copy/linkfile config may have changed.
1398 self._CopyAndLinkFiles()
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001399 return
1400
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001401 branch = self.GetBranch(branch)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001402
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001403 if not branch.LocalMerge:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001404 # The current branch has no tracking configuration.
Anatol Pomazau2a32f6a2011-08-30 10:52:33 -07001405 # Jump off it to a detached HEAD.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001406 #
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001407 syncbuf.info(self,
1408 "leaving %s; does not track upstream",
1409 branch.name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001410 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001411 self._Checkout(revid, quiet=True)
Martin Kellye4e94d22017-03-21 16:05:12 -07001412 if submodules:
1413 self._SyncSubmodules(quiet=True)
Sarah Owensa5be53f2012-09-09 15:37:57 -07001414 except GitError as e:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001415 syncbuf.fail(self, e)
1416 return
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001417 self._CopyAndLinkFiles()
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001418 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001419
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001420 upstream_gain = self._revlist(not_rev(HEAD), revid)
Mike Frysinger2ba5a1e2019-09-23 17:42:23 -04001421
1422 # See if we can perform a fast forward merge. This can happen if our
1423 # branch isn't in the exact same state as we last published.
1424 try:
1425 self.work_git.merge_base('--is-ancestor', HEAD, revid)
1426 # Skip the published logic.
1427 pub = False
1428 except GitError:
1429 pub = self.WasPublished(branch.name, all_refs)
1430
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001431 if pub:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001432 not_merged = self._revlist(not_rev(revid), pub)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001433 if not_merged:
1434 if upstream_gain:
1435 # The user has published this branch and some of those
1436 # commits are not yet merged upstream. We do not want
1437 # to rewrite the published commits so we punt.
1438 #
Daniel Sandler4c50dee2010-03-02 15:38:03 -05001439 syncbuf.fail(self,
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001440 "branch %s is published (but not merged) and is now "
1441 "%d commits behind" % (branch.name, len(upstream_gain)))
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001442 return
Shawn O. Pearce05f66b62009-04-21 08:26:32 -07001443 elif pub == head:
1444 # All published commits are merged, and thus we are a
1445 # strict subset. We can fast-forward safely.
Shawn O. Pearcea54c5272008-10-30 11:03:00 -07001446 #
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001447 syncbuf.later1(self, _doff)
Martin Kellye4e94d22017-03-21 16:05:12 -07001448 if submodules:
1449 syncbuf.later1(self, _dosubmodules)
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001450 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001451
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001452 # Examine the local commits not in the remote. Find the
1453 # last one attributed to this user, if any.
1454 #
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001455 local_changes = self._revlist(not_rev(revid), HEAD, format='%H %ce')
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001456 last_mine = None
1457 cnt_mine = 0
1458 for commit in local_changes:
Mike Frysinger600f4922019-08-03 02:14:28 -04001459 commit_id, committer_email = commit.split(' ', 1)
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001460 if committer_email == self.UserEmail:
1461 last_mine = commit_id
1462 cnt_mine += 1
1463
Shawn O. Pearceda88ff42009-06-03 11:09:12 -07001464 if not upstream_gain and cnt_mine == len(local_changes):
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001465 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001466
1467 if self.IsDirty(consider_untracked=False):
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001468 syncbuf.fail(self, _DirtyError())
1469 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001470
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001471 # If the upstream switched on us, warn the user.
1472 #
1473 if branch.merge != self.revisionExpr:
1474 if branch.merge and self.revisionExpr:
1475 syncbuf.info(self,
1476 'manifest switched %s...%s',
1477 branch.merge,
1478 self.revisionExpr)
1479 elif branch.merge:
1480 syncbuf.info(self,
1481 'manifest no longer tracks %s',
1482 branch.merge)
1483
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001484 if cnt_mine < len(local_changes):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001485 # Upstream rebased. Not everything in HEAD
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001486 # was created by this user.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001487 #
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001488 syncbuf.info(self,
1489 "discarding %d commits removed from upstream",
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001490 len(local_changes) - cnt_mine)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001491
Mike Frysingerdede5642022-07-10 04:56:04 -04001492 branch.remote = self.GetRemote()
Anatol Pomazaucd7c5de2012-03-20 13:45:00 -07001493 if not ID_RE.match(self.revisionExpr):
1494 # in case of manifest sync the revisionExpr might be a SHA1
1495 branch.merge = self.revisionExpr
Conley Owens04f2f0e2014-10-01 17:22:46 -07001496 if not branch.merge.startswith('refs/'):
1497 branch.merge = R_HEADS + branch.merge
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001498 branch.Save()
1499
Mike Pontillod3153822012-02-28 11:53:24 -08001500 if cnt_mine > 0 and self.rebase:
Martin Kellye4e94d22017-03-21 16:05:12 -07001501 def _docopyandlink():
1502 self._CopyAndLinkFiles()
1503
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001504 def _dorebase():
Anthony King7bdac712014-07-16 12:56:40 +01001505 self._Rebase(upstream='%s^1' % last_mine, onto=revid)
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001506 syncbuf.later2(self, _dorebase)
Martin Kellye4e94d22017-03-21 16:05:12 -07001507 if submodules:
1508 syncbuf.later2(self, _dosubmodules)
1509 syncbuf.later2(self, _docopyandlink)
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001510 elif local_changes:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001511 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001512 self._ResetHard(revid)
Martin Kellye4e94d22017-03-21 16:05:12 -07001513 if submodules:
1514 self._SyncSubmodules(quiet=True)
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001515 self._CopyAndLinkFiles()
Sarah Owensa5be53f2012-09-09 15:37:57 -07001516 except GitError as e:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001517 syncbuf.fail(self, e)
1518 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001519 else:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001520 syncbuf.later1(self, _doff)
Martin Kellye4e94d22017-03-21 16:05:12 -07001521 if submodules:
1522 syncbuf.later1(self, _dosubmodules)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001523
Mike Frysingere6a202f2019-08-02 15:57:57 -04001524 def AddCopyFile(self, src, dest, topdir):
1525 """Mark |src| for copying to |dest| (relative to |topdir|).
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001526
Mike Frysingere6a202f2019-08-02 15:57:57 -04001527 No filesystem changes occur here. Actual copying happens later on.
1528
1529 Paths should have basic validation run on them before being queued.
1530 Further checking will be handled when the actual copy happens.
1531 """
1532 self.copyfiles.append(_CopyFile(self.worktree, src, topdir, dest))
1533
1534 def AddLinkFile(self, src, dest, topdir):
1535 """Mark |dest| to create a symlink (relative to |topdir|) pointing to |src|.
1536
1537 No filesystem changes occur here. Actual linking happens later on.
1538
1539 Paths should have basic validation run on them before being queued.
1540 Further checking will be handled when the actual link happens.
1541 """
1542 self.linkfiles.append(_LinkFile(self.worktree, src, topdir, dest))
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001543
James W. Mills24c13082012-04-12 15:04:13 -05001544 def AddAnnotation(self, name, value, keep):
Jack Neus6ea0cae2021-07-20 20:52:33 +00001545 self.annotations.append(Annotation(name, value, keep))
James W. Mills24c13082012-04-12 15:04:13 -05001546
Shawn O. Pearce632768b2008-10-23 11:58:52 -07001547 def DownloadPatchSet(self, change_id, patch_id):
1548 """Download a single patch set of a single change to FETCH_HEAD.
1549 """
Mike Frysingerdede5642022-07-10 04:56:04 -04001550 remote = self.GetRemote()
Shawn O. Pearce632768b2008-10-23 11:58:52 -07001551
1552 cmd = ['fetch', remote.name]
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001553 cmd.append('refs/changes/%2.2d/%d/%d'
Shawn O. Pearce632768b2008-10-23 11:58:52 -07001554 % (change_id % 100, change_id, patch_id))
Shawn O. Pearce632768b2008-10-23 11:58:52 -07001555 if GitCommand(self, cmd, bare=True).Wait() != 0:
1556 return None
1557 return DownloadedChange(self,
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001558 self.GetRevisionId(),
Shawn O. Pearce632768b2008-10-23 11:58:52 -07001559 change_id,
1560 patch_id,
1561 self.bare_git.rev_parse('FETCH_HEAD'))
1562
Mike Frysingerc0d18662020-02-19 19:19:18 -05001563 def DeleteWorktree(self, quiet=False, force=False):
1564 """Delete the source checkout and any other housekeeping tasks.
1565
1566 This currently leaves behind the internal .repo/ cache state. This helps
1567 when switching branches or manifest changes get reverted as we don't have
1568 to redownload all the git objects. But we should do some GC at some point.
1569
1570 Args:
1571 quiet: Whether to hide normal messages.
1572 force: Always delete tree even if dirty.
1573
1574 Returns:
1575 True if the worktree was completely cleaned out.
1576 """
1577 if self.IsDirty():
1578 if force:
1579 print('warning: %s: Removing dirty project: uncommitted changes lost.' %
LaMont Jones8501d462022-06-22 19:21:15 +00001580 (self.RelPath(local=False),), file=sys.stderr)
Mike Frysingerc0d18662020-02-19 19:19:18 -05001581 else:
1582 print('error: %s: Cannot remove project: uncommitted changes are '
LaMont Jones8501d462022-06-22 19:21:15 +00001583 'present.\n' % (self.RelPath(local=False),), file=sys.stderr)
Mike Frysingerc0d18662020-02-19 19:19:18 -05001584 return False
1585
1586 if not quiet:
LaMont Jones8501d462022-06-22 19:21:15 +00001587 print('%s: Deleting obsolete checkout.' % (self.RelPath(local=False),))
Mike Frysingerc0d18662020-02-19 19:19:18 -05001588
1589 # Unlock and delink from the main worktree. We don't use git's worktree
1590 # remove because it will recursively delete projects -- we handle that
1591 # ourselves below. https://crbug.com/git/48
1592 if self.use_git_worktrees:
1593 needle = platform_utils.realpath(self.gitdir)
1594 # Find the git worktree commondir under .repo/worktrees/.
1595 output = self.bare_git.worktree('list', '--porcelain').splitlines()[0]
1596 assert output.startswith('worktree '), output
1597 commondir = output[9:]
1598 # Walk each of the git worktrees to see where they point.
1599 configs = os.path.join(commondir, 'worktrees')
1600 for name in os.listdir(configs):
1601 gitdir = os.path.join(configs, name, 'gitdir')
1602 with open(gitdir) as fp:
1603 relpath = fp.read().strip()
1604 # Resolve the checkout path and see if it matches this project.
1605 fullpath = platform_utils.realpath(os.path.join(configs, name, relpath))
1606 if fullpath == needle:
1607 platform_utils.rmtree(os.path.join(configs, name))
1608
1609 # Delete the .git directory first, so we're less likely to have a partially
1610 # working git repository around. There shouldn't be any git projects here,
1611 # so rmtree works.
1612
1613 # Try to remove plain files first in case of git worktrees. If this fails
1614 # for any reason, we'll fall back to rmtree, and that'll display errors if
1615 # it can't remove things either.
1616 try:
1617 platform_utils.remove(self.gitdir)
1618 except OSError:
1619 pass
1620 try:
1621 platform_utils.rmtree(self.gitdir)
1622 except OSError as e:
1623 if e.errno != errno.ENOENT:
1624 print('error: %s: %s' % (self.gitdir, e), file=sys.stderr)
1625 print('error: %s: Failed to delete obsolete checkout; remove manually, '
LaMont Jones8501d462022-06-22 19:21:15 +00001626 'then run `repo sync -l`.' % (self.RelPath(local=False),),
1627 file=sys.stderr)
Mike Frysingerc0d18662020-02-19 19:19:18 -05001628 return False
1629
1630 # Delete everything under the worktree, except for directories that contain
1631 # another git project.
1632 dirs_to_remove = []
1633 failed = False
1634 for root, dirs, files in platform_utils.walk(self.worktree):
1635 for f in files:
1636 path = os.path.join(root, f)
1637 try:
1638 platform_utils.remove(path)
1639 except OSError as e:
1640 if e.errno != errno.ENOENT:
1641 print('error: %s: Failed to remove: %s' % (path, e), file=sys.stderr)
1642 failed = True
1643 dirs[:] = [d for d in dirs
1644 if not os.path.lexists(os.path.join(root, d, '.git'))]
1645 dirs_to_remove += [os.path.join(root, d) for d in dirs
1646 if os.path.join(root, d) not in dirs_to_remove]
1647 for d in reversed(dirs_to_remove):
1648 if platform_utils.islink(d):
1649 try:
1650 platform_utils.remove(d)
1651 except OSError as e:
1652 if e.errno != errno.ENOENT:
1653 print('error: %s: Failed to remove: %s' % (d, e), file=sys.stderr)
1654 failed = True
1655 elif not platform_utils.listdir(d):
1656 try:
1657 platform_utils.rmdir(d)
1658 except OSError as e:
1659 if e.errno != errno.ENOENT:
1660 print('error: %s: Failed to remove: %s' % (d, e), file=sys.stderr)
1661 failed = True
1662 if failed:
LaMont Jones8501d462022-06-22 19:21:15 +00001663 print('error: %s: Failed to delete obsolete checkout.' % (self.RelPath(local=False),),
Mike Frysingerc0d18662020-02-19 19:19:18 -05001664 file=sys.stderr)
1665 print(' Remove manually, then run `repo sync -l`.', file=sys.stderr)
1666 return False
1667
1668 # Try deleting parent dirs if they are empty.
1669 path = self.worktree
1670 while path != self.manifest.topdir:
1671 try:
1672 platform_utils.rmdir(path)
1673 except OSError as e:
1674 if e.errno != errno.ENOENT:
1675 break
1676 path = os.path.dirname(path)
1677
1678 return True
1679
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001680# Branch Management ##
Theodore Dubois60fdc5c2019-07-30 12:14:25 -07001681 def StartBranch(self, name, branch_merge='', revision=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001682 """Create a new branch off the manifest's revision.
1683 """
Simran Basib9a1b732015-08-20 12:19:28 -07001684 if not branch_merge:
1685 branch_merge = self.revisionExpr
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001686 head = self.work_git.GetHead()
1687 if head == (R_HEADS + name):
1688 return True
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001689
David Pursehouse8a68ff92012-09-24 12:15:13 +09001690 all_refs = self.bare_ref.all
Anthony King7bdac712014-07-16 12:56:40 +01001691 if R_HEADS + name in all_refs:
Mike Frysingerad1b7bd2022-08-18 09:17:09 -04001692 return GitCommand(self, ['checkout', '-q', name, '--']).Wait() == 0
Shawn O. Pearce0a389e92009-04-10 16:21:18 -07001693
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001694 branch = self.GetBranch(name)
Mike Frysingerdede5642022-07-10 04:56:04 -04001695 branch.remote = self.GetRemote()
Simran Basib9a1b732015-08-20 12:19:28 -07001696 branch.merge = branch_merge
1697 if not branch.merge.startswith('refs/') and not ID_RE.match(branch_merge):
1698 branch.merge = R_HEADS + branch_merge
Theodore Dubois60fdc5c2019-07-30 12:14:25 -07001699
1700 if revision is None:
1701 revid = self.GetRevisionId(all_refs)
1702 else:
1703 revid = self.work_git.rev_parse(revision)
Shawn O. Pearce0a389e92009-04-10 16:21:18 -07001704
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001705 if head.startswith(R_HEADS):
1706 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09001707 head = all_refs[head]
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001708 except KeyError:
1709 head = None
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001710 if revid and head and revid == head:
Mike Frysinger746e7f62020-02-19 15:49:02 -05001711 ref = R_HEADS + name
1712 self.work_git.update_ref(ref, revid)
1713 self.work_git.symbolic_ref(HEAD, ref)
1714 branch.Save()
1715 return True
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001716
Mike Frysingerad1b7bd2022-08-18 09:17:09 -04001717 if GitCommand(self, ['checkout', '-q', '-b', branch.name, revid]).Wait() == 0:
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001718 branch.Save()
1719 return True
1720 return False
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001721
Wink Saville02d79452009-04-10 13:01:24 -07001722 def CheckoutBranch(self, name):
1723 """Checkout a local topic branch.
Doug Anderson3ba5f952011-04-07 12:51:04 -07001724
1725 Args:
1726 name: The name of the branch to checkout.
1727
1728 Returns:
1729 True if the checkout succeeded; False if it didn't; None if the branch
1730 didn't exist.
Wink Saville02d79452009-04-10 13:01:24 -07001731 """
Shawn O. Pearce89e717d2009-04-18 15:04:41 -07001732 rev = R_HEADS + name
1733 head = self.work_git.GetHead()
1734 if head == rev:
1735 # Already on the branch
1736 #
1737 return True
Wink Saville02d79452009-04-10 13:01:24 -07001738
David Pursehouse8a68ff92012-09-24 12:15:13 +09001739 all_refs = self.bare_ref.all
Wink Saville02d79452009-04-10 13:01:24 -07001740 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09001741 revid = all_refs[rev]
Shawn O. Pearce89e717d2009-04-18 15:04:41 -07001742 except KeyError:
1743 # Branch does not exist in this project
1744 #
Doug Anderson3ba5f952011-04-07 12:51:04 -07001745 return None
Wink Saville02d79452009-04-10 13:01:24 -07001746
Shawn O. Pearce89e717d2009-04-18 15:04:41 -07001747 if head.startswith(R_HEADS):
1748 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09001749 head = all_refs[head]
Shawn O. Pearce89e717d2009-04-18 15:04:41 -07001750 except KeyError:
1751 head = None
1752
1753 if head == revid:
1754 # Same revision; just update HEAD to point to the new
1755 # target branch, but otherwise take no other action.
1756 #
Mike Frysinger9f91c432020-02-23 23:24:40 -05001757 _lwrite(self.work_git.GetDotgitPath(subpath=HEAD),
1758 'ref: %s%s\n' % (R_HEADS, name))
Shawn O. Pearce89e717d2009-04-18 15:04:41 -07001759 return True
1760
1761 return GitCommand(self,
1762 ['checkout', name, '--'],
Anthony King7bdac712014-07-16 12:56:40 +01001763 capture_stdout=True,
1764 capture_stderr=True).Wait() == 0
Wink Saville02d79452009-04-10 13:01:24 -07001765
Shawn O. Pearce9fa44db2008-11-03 11:24:59 -08001766 def AbandonBranch(self, name):
1767 """Destroy a local topic branch.
Doug Andersondafb1d62011-04-07 11:46:59 -07001768
1769 Args:
1770 name: The name of the branch to abandon.
1771
1772 Returns:
1773 True if the abandon succeeded; False if it didn't; None if the branch
1774 didn't exist.
Shawn O. Pearce9fa44db2008-11-03 11:24:59 -08001775 """
Shawn O. Pearce552ac892009-04-18 15:15:24 -07001776 rev = R_HEADS + name
David Pursehouse8a68ff92012-09-24 12:15:13 +09001777 all_refs = self.bare_ref.all
1778 if rev not in all_refs:
Doug Andersondafb1d62011-04-07 11:46:59 -07001779 # Doesn't exist
1780 return None
Shawn O. Pearce9fa44db2008-11-03 11:24:59 -08001781
Shawn O. Pearce552ac892009-04-18 15:15:24 -07001782 head = self.work_git.GetHead()
1783 if head == rev:
1784 # We can't destroy the branch while we are sitting
1785 # on it. Switch to a detached HEAD.
1786 #
David Pursehouse8a68ff92012-09-24 12:15:13 +09001787 head = all_refs[head]
Shawn O. Pearce9fa44db2008-11-03 11:24:59 -08001788
David Pursehouse8a68ff92012-09-24 12:15:13 +09001789 revid = self.GetRevisionId(all_refs)
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001790 if head == revid:
Mike Frysinger9f91c432020-02-23 23:24:40 -05001791 _lwrite(self.work_git.GetDotgitPath(subpath=HEAD), '%s\n' % revid)
Shawn O. Pearce552ac892009-04-18 15:15:24 -07001792 else:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001793 self._Checkout(revid, quiet=True)
Shawn O. Pearce552ac892009-04-18 15:15:24 -07001794
1795 return GitCommand(self,
1796 ['branch', '-D', name],
Anthony King7bdac712014-07-16 12:56:40 +01001797 capture_stdout=True,
1798 capture_stderr=True).Wait() == 0
Shawn O. Pearce9fa44db2008-11-03 11:24:59 -08001799
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001800 def PruneHeads(self):
1801 """Prune any topic branches already merged into upstream.
1802 """
1803 cb = self.CurrentBranch
1804 kill = []
Shawn O. Pearce3778f9d2009-03-02 12:30:50 -08001805 left = self._allrefs
1806 for name in left.keys():
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001807 if name.startswith(R_HEADS):
1808 name = name[len(R_HEADS):]
1809 if cb is None or name != cb:
1810 kill.append(name)
1811
Mike Frysingera3794e92021-03-11 23:24:01 -05001812 # Minor optimization: If there's nothing to prune, then don't try to read
1813 # any project state.
1814 if not kill and not cb:
1815 return []
1816
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001817 rev = self.GetRevisionId(left)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001818 if cb is not None \
1819 and not self._revlist(HEAD + '...' + rev) \
Anthony King7bdac712014-07-16 12:56:40 +01001820 and not self.IsDirty(consider_untracked=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001821 self.work_git.DetachHead(HEAD)
1822 kill.append(cb)
1823
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001824 if kill:
Shawn O. Pearce5b23f242009-04-17 18:43:33 -07001825 old = self.bare_git.GetHead()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001826
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001827 try:
1828 self.bare_git.DetachHead(rev)
1829
1830 b = ['branch', '-d']
1831 b.extend(kill)
1832 b = GitCommand(self, b, bare=True,
1833 capture_stdout=True,
1834 capture_stderr=True)
1835 b.Wait()
1836 finally:
Dan Willemsen1a799d12015-12-15 13:40:05 -08001837 if ID_RE.match(old):
1838 self.bare_git.DetachHead(old)
1839 else:
1840 self.bare_git.SetHead(old)
Shawn O. Pearce3778f9d2009-03-02 12:30:50 -08001841 left = self._allrefs
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001842
Shawn O. Pearce3778f9d2009-03-02 12:30:50 -08001843 for branch in kill:
1844 if (R_HEADS + branch) not in left:
1845 self.CleanPublishedCache()
1846 break
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001847
1848 if cb and cb not in kill:
1849 kill.append(cb)
Shawn O. Pearce7c6c64d2009-03-02 12:38:13 -08001850 kill.sort()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001851
1852 kept = []
1853 for branch in kill:
Anthony King7bdac712014-07-16 12:56:40 +01001854 if R_HEADS + branch in left:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001855 branch = self.GetBranch(branch)
1856 base = branch.LocalMerge
1857 if not base:
1858 base = rev
1859 kept.append(ReviewableBranch(self, branch, base))
1860 return kept
1861
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001862# Submodule Management ##
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001863 def GetRegisteredSubprojects(self):
1864 result = []
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001865
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001866 def rec(subprojects):
1867 if not subprojects:
1868 return
1869 result.extend(subprojects)
1870 for p in subprojects:
1871 rec(p.subprojects)
1872 rec(self.subprojects)
1873 return result
1874
1875 def _GetSubmodules(self):
1876 # Unfortunately we cannot call `git submodule status --recursive` here
1877 # because the working tree might not exist yet, and it cannot be used
1878 # without a working tree in its current implementation.
1879
1880 def get_submodules(gitdir, rev):
1881 # Parse .gitmodules for submodule sub_paths and sub_urls
1882 sub_paths, sub_urls = parse_gitmodules(gitdir, rev)
1883 if not sub_paths:
1884 return []
1885 # Run `git ls-tree` to read SHAs of submodule object, which happen to be
1886 # revision of submodule repository
1887 sub_revs = git_ls_tree(gitdir, rev, sub_paths)
1888 submodules = []
1889 for sub_path, sub_url in zip(sub_paths, sub_urls):
1890 try:
1891 sub_rev = sub_revs[sub_path]
1892 except KeyError:
1893 # Ignore non-exist submodules
1894 continue
1895 submodules.append((sub_rev, sub_path, sub_url))
1896 return submodules
1897
Sebastian Schuberth41a26832019-03-11 13:53:38 +01001898 re_path = re.compile(r'^submodule\.(.+)\.path=(.*)$')
1899 re_url = re.compile(r'^submodule\.(.+)\.url=(.*)$')
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001900
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001901 def parse_gitmodules(gitdir, rev):
1902 cmd = ['cat-file', 'blob', '%s:.gitmodules' % rev]
1903 try:
Anthony King7bdac712014-07-16 12:56:40 +01001904 p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True,
1905 bare=True, gitdir=gitdir)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001906 except GitError:
1907 return [], []
1908 if p.Wait() != 0:
1909 return [], []
1910
1911 gitmodules_lines = []
1912 fd, temp_gitmodules_path = tempfile.mkstemp()
1913 try:
Mike Frysinger163d42e2020-02-11 03:35:24 -05001914 os.write(fd, p.stdout.encode('utf-8'))
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001915 os.close(fd)
1916 cmd = ['config', '--file', temp_gitmodules_path, '--list']
Anthony King7bdac712014-07-16 12:56:40 +01001917 p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True,
1918 bare=True, gitdir=gitdir)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001919 if p.Wait() != 0:
1920 return [], []
1921 gitmodules_lines = p.stdout.split('\n')
1922 except GitError:
1923 return [], []
1924 finally:
Renaud Paquay010fed72016-11-11 14:25:29 -08001925 platform_utils.remove(temp_gitmodules_path)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001926
1927 names = set()
1928 paths = {}
1929 urls = {}
1930 for line in gitmodules_lines:
1931 if not line:
1932 continue
1933 m = re_path.match(line)
1934 if m:
1935 names.add(m.group(1))
1936 paths[m.group(1)] = m.group(2)
1937 continue
1938 m = re_url.match(line)
1939 if m:
1940 names.add(m.group(1))
1941 urls[m.group(1)] = m.group(2)
1942 continue
1943 names = sorted(names)
1944 return ([paths.get(name, '') for name in names],
1945 [urls.get(name, '') for name in names])
1946
1947 def git_ls_tree(gitdir, rev, paths):
1948 cmd = ['ls-tree', rev, '--']
1949 cmd.extend(paths)
1950 try:
Anthony King7bdac712014-07-16 12:56:40 +01001951 p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True,
1952 bare=True, gitdir=gitdir)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001953 except GitError:
1954 return []
1955 if p.Wait() != 0:
1956 return []
1957 objects = {}
1958 for line in p.stdout.split('\n'):
1959 if not line.strip():
1960 continue
1961 object_rev, object_path = line.split()[2:4]
1962 objects[object_path] = object_rev
1963 return objects
1964
1965 try:
1966 rev = self.GetRevisionId()
1967 except GitError:
1968 return []
1969 return get_submodules(self.gitdir, rev)
1970
1971 def GetDerivedSubprojects(self):
1972 result = []
1973 if not self.Exists:
1974 # If git repo does not exist yet, querying its submodules will
1975 # mess up its states; so return here.
1976 return result
1977 for rev, path, url in self._GetSubmodules():
1978 name = self.manifest.GetSubprojectName(self, path)
David James8d201162013-10-11 17:03:19 -07001979 relpath, worktree, gitdir, objdir = \
1980 self.manifest.GetSubprojectPaths(self, name, path)
1981 project = self.manifest.paths.get(relpath)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001982 if project:
1983 result.extend(project.GetDerivedSubprojects())
1984 continue
David James8d201162013-10-11 17:03:19 -07001985
Shouheng Zhang02c0ee62017-12-08 09:54:58 +08001986 if url.startswith('..'):
1987 url = urllib.parse.urljoin("%s/" % self.remote.url, url)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001988 remote = RemoteSpec(self.remote.name,
Anthony King7bdac712014-07-16 12:56:40 +01001989 url=url,
Steve Raed6480452016-08-10 15:00:00 -07001990 pushUrl=self.remote.pushUrl,
Anthony King7bdac712014-07-16 12:56:40 +01001991 review=self.remote.review,
1992 revision=self.remote.revision)
1993 subproject = Project(manifest=self.manifest,
1994 name=name,
1995 remote=remote,
1996 gitdir=gitdir,
1997 objdir=objdir,
1998 worktree=worktree,
1999 relpath=relpath,
Aymen Bouaziz2598ed02016-06-24 14:34:08 +02002000 revisionExpr=rev,
Anthony King7bdac712014-07-16 12:56:40 +01002001 revisionId=rev,
2002 rebase=self.rebase,
2003 groups=self.groups,
2004 sync_c=self.sync_c,
2005 sync_s=self.sync_s,
YOUNG HO CHAa32c92c2018-02-14 16:57:31 +09002006 sync_tags=self.sync_tags,
Anthony King7bdac712014-07-16 12:56:40 +01002007 parent=self,
2008 is_derived=True)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08002009 result.append(subproject)
2010 result.extend(subproject.GetDerivedSubprojects())
2011 return result
2012
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002013# Direct Git Commands ##
Mike Frysingerf81c72e2020-02-19 15:50:00 -05002014 def EnableRepositoryExtension(self, key, value='true', version=1):
2015 """Enable git repository extension |key| with |value|.
2016
2017 Args:
2018 key: The extension to enabled. Omit the "extensions." prefix.
2019 value: The value to use for the extension.
2020 version: The minimum git repository version needed.
2021 """
2022 # Make sure the git repo version is new enough already.
2023 found_version = self.config.GetInt('core.repositoryFormatVersion')
2024 if found_version is None:
2025 found_version = 0
2026 if found_version < version:
2027 self.config.SetString('core.repositoryFormatVersion', str(version))
2028
2029 # Enable the extension!
2030 self.config.SetString('extensions.%s' % (key,), value)
2031
Mike Frysinger50a81de2020-09-06 15:51:21 -04002032 def ResolveRemoteHead(self, name=None):
2033 """Find out what the default branch (HEAD) points to.
2034
2035 Normally this points to refs/heads/master, but projects are moving to main.
2036 Support whatever the server uses rather than hardcoding "master" ourselves.
2037 """
2038 if name is None:
2039 name = self.remote.name
2040
2041 # The output will look like (NB: tabs are separators):
2042 # ref: refs/heads/master HEAD
2043 # 5f6803b100bb3cd0f534e96e88c91373e8ed1c44 HEAD
2044 output = self.bare_git.ls_remote('-q', '--symref', '--exit-code', name, 'HEAD')
2045
2046 for line in output.splitlines():
2047 lhs, rhs = line.split('\t', 1)
2048 if rhs == 'HEAD' and lhs.startswith('ref:'):
2049 return lhs[4:].strip()
2050
2051 return None
2052
Zac Livingstone4332262017-06-16 08:56:09 -06002053 def _CheckForImmutableRevision(self):
Chris AtLee2fb64662014-01-16 21:32:33 -05002054 try:
2055 # if revision (sha or tag) is not present then following function
2056 # throws an error.
Ian Kasprzak0286e312021-02-05 10:06:18 -08002057 self.bare_git.rev_list('-1', '--missing=allow-any',
2058 '%s^0' % self.revisionExpr, '--')
Xin Li0e776a52021-06-29 21:42:34 +00002059 if self.upstream:
Mike Frysingerdede5642022-07-10 04:56:04 -04002060 rev = self.GetRemote().ToLocal(self.upstream)
Xin Li0e776a52021-06-29 21:42:34 +00002061 self.bare_git.rev_list('-1', '--missing=allow-any',
2062 '%s^0' % rev, '--')
Xin Li8e983bb2021-07-20 20:15:30 +00002063 self.bare_git.merge_base('--is-ancestor', self.revisionExpr, rev)
Chris AtLee2fb64662014-01-16 21:32:33 -05002064 return True
2065 except GitError:
2066 # There is no such persistent revision. We have to fetch it.
2067 return False
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002068
Julien Campergue335f5ef2013-10-16 11:02:35 +02002069 def _FetchArchive(self, tarpath, cwd=None):
2070 cmd = ['archive', '-v', '-o', tarpath]
2071 cmd.append('--remote=%s' % self.remote.url)
LaMont Jones8501d462022-06-22 19:21:15 +00002072 cmd.append('--prefix=%s/' % self.RelPath(local=False))
Julien Campergue335f5ef2013-10-16 11:02:35 +02002073 cmd.append(self.revisionExpr)
2074
2075 command = GitCommand(self, cmd, cwd=cwd,
2076 capture_stdout=True,
2077 capture_stderr=True)
2078
2079 if command.Wait() != 0:
2080 raise GitError('git archive %s: %s' % (self.name, command.stderr))
2081
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07002082 def _RemoteFetch(self, name=None,
2083 current_branch_only=False,
Shawn O. Pearce16614f82010-10-29 12:05:43 -07002084 initial=False,
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002085 quiet=False,
Mike Frysinger521d01b2020-02-17 01:51:49 -05002086 verbose=False,
Mike Frysinger7b586f22021-02-23 18:38:39 -05002087 output_redir=None,
Mitchel Humpherys597868b2012-10-29 10:18:34 -07002088 alt_dir=None,
Mike Frysingerc58ec4d2020-02-17 14:36:08 -05002089 tags=True,
Aymen Bouaziz6c594462016-10-25 18:03:51 +02002090 prune=False,
Martin Kellye4e94d22017-03-21 16:05:12 -07002091 depth=None,
Mike Frysingere57f1142019-03-18 21:27:54 -04002092 submodules=False,
Mike Frysinger19e409c2021-05-05 19:44:35 -04002093 ssh_proxy=None,
Xin Li745be2e2019-06-03 11:24:30 -07002094 force_sync=False,
George Engelbrecht9bc283e2020-04-02 12:36:09 -06002095 clone_filter=None,
2096 retry_fetches=2,
2097 retry_sleep_initial_sec=4.0,
2098 retry_exp_factor=2.0):
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07002099 is_sha1 = False
2100 tag_name = None
David Pursehouse9bc422f2014-04-15 10:28:56 +09002101 # The depth should not be used when fetching to a mirror because
2102 # it will result in a shallow repository that cannot be cloned or
2103 # fetched from.
Aymen Bouaziz6c594462016-10-25 18:03:51 +02002104 # The repo project should also never be synced with partial depth.
2105 if self.manifest.IsMirror or self.relpath == '.repo/repo':
2106 depth = None
David Pursehouse9bc422f2014-04-15 10:28:56 +09002107
Shawn Pearce69e04d82014-01-29 12:48:54 -08002108 if depth:
2109 current_branch_only = True
2110
Nasser Grainawi909d58b2014-09-19 12:13:04 -06002111 if ID_RE.match(self.revisionExpr) is not None:
2112 is_sha1 = True
2113
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07002114 if current_branch_only:
Nasser Grainawi909d58b2014-09-19 12:13:04 -06002115 if self.revisionExpr.startswith(R_TAGS):
Robin Schneider9c1fc5b2021-11-13 22:55:32 +01002116 # This is a tag and its commit id should never change.
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07002117 tag_name = self.revisionExpr[len(R_TAGS):]
Robin Schneider9c1fc5b2021-11-13 22:55:32 +01002118 elif self.upstream and self.upstream.startswith(R_TAGS):
2119 # This is a tag and its commit id should never change.
2120 tag_name = self.upstream[len(R_TAGS):]
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07002121
2122 if is_sha1 or tag_name is not None:
Zac Livingstone4332262017-06-16 08:56:09 -06002123 if self._CheckForImmutableRevision():
Mike Frysinger521d01b2020-02-17 01:51:49 -05002124 if verbose:
Tim Schumacher1f1596b2019-04-15 11:17:08 +02002125 print('Skipped fetching project %s (already have persistent ref)'
2126 % self.name)
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07002127 return True
Bertrand SIMONNET3000cda2014-11-25 16:19:29 -08002128 if is_sha1 and not depth:
2129 # When syncing a specific commit and --depth is not set:
2130 # * if upstream is explicitly specified and is not a sha1, fetch only
2131 # upstream as users expect only upstream to be fetch.
2132 # Note: The commit might not be in upstream in which case the sync
2133 # will fail.
2134 # * otherwise, fetch all branches to make sure we end up with the
2135 # specific commit.
Aymen Bouaziz037040f2016-06-28 12:27:23 +02002136 if self.upstream:
2137 current_branch_only = not ID_RE.match(self.upstream)
2138 else:
2139 current_branch_only = False
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07002140
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002141 if not name:
2142 name = self.remote.name
Shawn O. Pearcefb231612009-04-10 18:53:46 -07002143
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07002144 remote = self.GetRemote(name)
Mike Frysinger339f2df2021-05-06 00:44:42 -04002145 if not remote.PreConnectFetch(ssh_proxy):
2146 ssh_proxy = None
Shawn O. Pearcefb231612009-04-10 18:53:46 -07002147
Shawn O. Pearce88443382010-10-08 10:02:09 +02002148 if initial:
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002149 if alt_dir and 'objects' == os.path.basename(alt_dir):
2150 ref_dir = os.path.dirname(alt_dir)
Shawn O. Pearce88443382010-10-08 10:02:09 +02002151 packed_refs = os.path.join(self.gitdir, 'packed-refs')
Shawn O. Pearce88443382010-10-08 10:02:09 +02002152
David Pursehouse8a68ff92012-09-24 12:15:13 +09002153 all_refs = self.bare_ref.all
2154 ids = set(all_refs.values())
Shawn O. Pearce88443382010-10-08 10:02:09 +02002155 tmp = set()
2156
Chirayu Desai217ea7d2013-03-01 19:14:38 +05302157 for r, ref_id in GitRefs(ref_dir).all.items():
David Pursehouse8a68ff92012-09-24 12:15:13 +09002158 if r not in all_refs:
Shawn O. Pearce88443382010-10-08 10:02:09 +02002159 if r.startswith(R_TAGS) or remote.WritesTo(r):
David Pursehouse8a68ff92012-09-24 12:15:13 +09002160 all_refs[r] = ref_id
2161 ids.add(ref_id)
Shawn O. Pearce88443382010-10-08 10:02:09 +02002162 continue
2163
David Pursehouse8a68ff92012-09-24 12:15:13 +09002164 if ref_id in ids:
Shawn O. Pearce88443382010-10-08 10:02:09 +02002165 continue
2166
David Pursehouse8a68ff92012-09-24 12:15:13 +09002167 r = 'refs/_alt/%s' % ref_id
2168 all_refs[r] = ref_id
2169 ids.add(ref_id)
Shawn O. Pearce88443382010-10-08 10:02:09 +02002170 tmp.add(r)
2171
heping3d7bbc92017-04-12 19:51:47 +08002172 tmp_packed_lines = []
2173 old_packed_lines = []
Shawn O. Pearce88443382010-10-08 10:02:09 +02002174
Chirayu Desai217ea7d2013-03-01 19:14:38 +05302175 for r in sorted(all_refs):
David Pursehouse8a68ff92012-09-24 12:15:13 +09002176 line = '%s %s\n' % (all_refs[r], r)
heping3d7bbc92017-04-12 19:51:47 +08002177 tmp_packed_lines.append(line)
Shawn O. Pearce88443382010-10-08 10:02:09 +02002178 if r not in tmp:
heping3d7bbc92017-04-12 19:51:47 +08002179 old_packed_lines.append(line)
Shawn O. Pearce88443382010-10-08 10:02:09 +02002180
heping3d7bbc92017-04-12 19:51:47 +08002181 tmp_packed = ''.join(tmp_packed_lines)
2182 old_packed = ''.join(old_packed_lines)
Shawn O. Pearce88443382010-10-08 10:02:09 +02002183 _lwrite(packed_refs, tmp_packed)
Shawn O. Pearce88443382010-10-08 10:02:09 +02002184 else:
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002185 alt_dir = None
Shawn O. Pearce88443382010-10-08 10:02:09 +02002186
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08002187 cmd = ['fetch']
Doug Anderson30d45292011-05-04 15:01:04 -07002188
Xin Li745be2e2019-06-03 11:24:30 -07002189 if clone_filter:
2190 git_require((2, 19, 0), fail=True, msg='partial clones')
2191 cmd.append('--filter=%s' % clone_filter)
Mike Frysingerf81c72e2020-02-19 15:50:00 -05002192 self.EnableRepositoryExtension('partialclone', self.remote.name)
Xin Li745be2e2019-06-03 11:24:30 -07002193
Conley Owensf97e8382015-01-21 11:12:46 -08002194 if depth:
Doug Anderson30d45292011-05-04 15:01:04 -07002195 cmd.append('--depth=%s' % depth)
Dan Willemseneeab6862015-08-03 13:11:53 -07002196 else:
2197 # If this repo has shallow objects, then we don't know which refs have
2198 # shallow objects or not. Tell git to unshallow all fetched refs. Don't
2199 # do this with projects that don't have shallow objects, since it is less
2200 # efficient.
2201 if os.path.exists(os.path.join(self.gitdir, 'shallow')):
2202 cmd.append('--depth=2147483647')
Doug Anderson30d45292011-05-04 15:01:04 -07002203
Mike Frysinger4847e052020-02-22 00:07:35 -05002204 if not verbose:
Shawn O. Pearce16614f82010-10-29 12:05:43 -07002205 cmd.append('--quiet')
Mike Frysinger4847e052020-02-22 00:07:35 -05002206 if not quiet and sys.stdout.isatty():
2207 cmd.append('--progress')
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08002208 if not self.worktree:
2209 cmd.append('--update-head-ok')
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002210 cmd.append(name)
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07002211
Mike Frysingere57f1142019-03-18 21:27:54 -04002212 if force_sync:
2213 cmd.append('--force')
2214
David Pursehouse74cfd272015-10-14 10:50:15 +09002215 if prune:
2216 cmd.append('--prune')
2217
LaMont Jonesff6b1da2022-06-01 21:03:34 +00002218 # Always pass something for --recurse-submodules, git with GIT_DIR behaves
2219 # incorrectly when not given `--recurse-submodules=no`. (b/218891912)
LaMont Jonesadaa1d82022-02-10 17:34:36 +00002220 cmd.append(f'--recurse-submodules={"on-demand" if submodules else "no"}')
Martin Kellye4e94d22017-03-21 16:05:12 -07002221
Kuang-che Wu6856f982019-11-25 12:37:55 +08002222 spec = []
Brian Harring14a66742012-09-28 20:21:57 -07002223 if not current_branch_only:
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07002224 # Fetch whole repo
Conley Owens80b87fe2014-05-09 17:13:44 -07002225 spec.append(str((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*')))
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07002226 elif tag_name is not None:
Conley Owens80b87fe2014-05-09 17:13:44 -07002227 spec.append('tag')
2228 spec.append(tag_name)
Nasser Grainawi04e52d62014-09-30 13:34:52 -06002229
Chirayu Desaif7b64e32020-02-04 17:50:57 +05302230 if self.manifest.IsMirror and not current_branch_only:
2231 branch = None
2232 else:
2233 branch = self.revisionExpr
Kuang-che Wu6856f982019-11-25 12:37:55 +08002234 if (not self.manifest.IsMirror and is_sha1 and depth
David Pursehouseabdf7502020-02-12 14:58:39 +09002235 and git_require((1, 8, 3))):
Kuang-che Wu6856f982019-11-25 12:37:55 +08002236 # Shallow checkout of a specific commit, fetch from that commit and not
2237 # the heads only as the commit might be deeper in the history.
2238 spec.append(branch)
Xin Liaabf79d2021-04-29 01:50:38 -07002239 if self.upstream:
2240 spec.append(self.upstream)
Kuang-che Wu6856f982019-11-25 12:37:55 +08002241 else:
2242 if is_sha1:
2243 branch = self.upstream
2244 if branch is not None and branch.strip():
2245 if not branch.startswith('refs/'):
2246 branch = R_HEADS + branch
2247 spec.append(str((u'+%s:' % branch) + remote.ToLocal(branch)))
2248
2249 # If mirroring repo and we cannot deduce the tag or branch to fetch, fetch
2250 # whole repo.
2251 if self.manifest.IsMirror and not spec:
2252 spec.append(str((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*')))
2253
2254 # If using depth then we should not get all the tags since they may
2255 # be outside of the depth.
Mike Frysingerc58ec4d2020-02-17 14:36:08 -05002256 if not tags or depth:
Kuang-che Wu6856f982019-11-25 12:37:55 +08002257 cmd.append('--no-tags')
2258 else:
2259 cmd.append('--tags')
2260 spec.append(str((u'+refs/tags/*:') + remote.ToLocal('refs/tags/*')))
2261
Conley Owens80b87fe2014-05-09 17:13:44 -07002262 cmd.extend(spec)
2263
George Engelbrecht9bc283e2020-04-02 12:36:09 -06002264 # At least one retry minimum due to git remote prune.
2265 retry_fetches = max(retry_fetches, 2)
2266 retry_cur_sleep = retry_sleep_initial_sec
2267 ok = prune_tried = False
2268 for try_n in range(retry_fetches):
Mike Frysinger67d6cdf2021-12-23 17:36:09 -05002269 gitcmd = GitCommand(
2270 self, cmd, bare=True, objdir=os.path.join(self.objdir, 'objects'),
2271 ssh_proxy=ssh_proxy,
2272 merge_output=True, capture_stdout=quiet or bool(output_redir))
Mike Frysinger7b586f22021-02-23 18:38:39 -05002273 if gitcmd.stdout and not quiet and output_redir:
2274 output_redir.write(gitcmd.stdout)
John L. Villalovos126e2982015-01-29 21:58:12 -08002275 ret = gitcmd.Wait()
Brian Harring14a66742012-09-28 20:21:57 -07002276 if ret == 0:
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002277 ok = True
2278 break
George Engelbrecht9bc283e2020-04-02 12:36:09 -06002279
2280 # Retry later due to HTTP 429 Too Many Requests.
Mike Frysinger92304bf2021-02-23 03:58:43 -05002281 elif (gitcmd.stdout and
2282 'error:' in gitcmd.stdout and
2283 'HTTP 429' in gitcmd.stdout):
Mike Frysinger6823bc22021-04-15 02:06:28 -04002284 # Fallthru to sleep+retry logic at the bottom.
2285 pass
George Engelbrecht9bc283e2020-04-02 12:36:09 -06002286
Mike Frysinger6823bc22021-04-15 02:06:28 -04002287 # Try to prune remote branches once in case there are conflicts.
2288 # For example, if the remote had refs/heads/upstream, but deleted that and
2289 # now has refs/heads/upstream/foo.
2290 elif (gitcmd.stdout and
Mike Frysinger92304bf2021-02-23 03:58:43 -05002291 'error:' in gitcmd.stdout and
2292 'git remote prune' in gitcmd.stdout and
George Engelbrecht9bc283e2020-04-02 12:36:09 -06002293 not prune_tried):
2294 prune_tried = True
John L. Villalovos126e2982015-01-29 21:58:12 -08002295 prunecmd = GitCommand(self, ['remote', 'prune', name], bare=True,
John L. Villalovos9c76f672015-03-16 20:49:10 -07002296 ssh_proxy=ssh_proxy)
John L. Villalovose30f46b2015-02-25 14:27:02 -08002297 ret = prunecmd.Wait()
John L. Villalovose30f46b2015-02-25 14:27:02 -08002298 if ret:
John L. Villalovos126e2982015-01-29 21:58:12 -08002299 break
Mike Frysingerb16b9d22021-05-20 01:48:17 -04002300 print('retrying fetch after pruning remote branches', file=output_redir)
Mike Frysinger6823bc22021-04-15 02:06:28 -04002301 # Continue right away so we don't sleep as we shouldn't need to.
John L. Villalovos126e2982015-01-29 21:58:12 -08002302 continue
Brian Harring14a66742012-09-28 20:21:57 -07002303 elif current_branch_only and is_sha1 and ret == 128:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002304 # Exit code 128 means "couldn't find the ref you asked for"; if we're
2305 # in sha1 mode, we just tried sync'ing from the upstream field; it
2306 # doesn't exist, thus abort the optimization attempt and do a full sync.
Brian Harring14a66742012-09-28 20:21:57 -07002307 break
Colin Crossc4b301f2015-05-13 00:10:02 -07002308 elif ret < 0:
2309 # Git died with a signal, exit immediately
2310 break
Mike Frysinger6823bc22021-04-15 02:06:28 -04002311
2312 # Figure out how long to sleep before the next attempt, if there is one.
Mike Frysingerb16b9d22021-05-20 01:48:17 -04002313 if not verbose and gitcmd.stdout:
2314 print('\n%s:\n%s' % (self.name, gitcmd.stdout), end='', file=output_redir)
Mike Frysinger6823bc22021-04-15 02:06:28 -04002315 if try_n < retry_fetches - 1:
Mike Frysingerb16b9d22021-05-20 01:48:17 -04002316 print('%s: sleeping %s seconds before retrying' % (self.name, retry_cur_sleep),
2317 file=output_redir)
Mike Frysinger6823bc22021-04-15 02:06:28 -04002318 time.sleep(retry_cur_sleep)
2319 retry_cur_sleep = min(retry_exp_factor * retry_cur_sleep,
2320 MAXIMUM_RETRY_SLEEP_SEC)
2321 retry_cur_sleep *= (1 - random.uniform(-RETRY_JITTER_PERCENT,
2322 RETRY_JITTER_PERCENT))
Shawn O. Pearce88443382010-10-08 10:02:09 +02002323
2324 if initial:
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002325 if alt_dir:
Shawn O. Pearce88443382010-10-08 10:02:09 +02002326 if old_packed != '':
2327 _lwrite(packed_refs, old_packed)
2328 else:
Renaud Paquay010fed72016-11-11 14:25:29 -08002329 platform_utils.remove(packed_refs)
Shawn O. Pearce88443382010-10-08 10:02:09 +02002330 self.bare_git.pack_refs('--all', '--prune')
Brian Harring14a66742012-09-28 20:21:57 -07002331
Aymen Bouaziz6c594462016-10-25 18:03:51 +02002332 if is_sha1 and current_branch_only:
Brian Harring14a66742012-09-28 20:21:57 -07002333 # We just synced the upstream given branch; verify we
2334 # got what we wanted, else trigger a second run of all
2335 # refs.
Zac Livingstone4332262017-06-16 08:56:09 -06002336 if not self._CheckForImmutableRevision():
Mike Frysinger521d01b2020-02-17 01:51:49 -05002337 # Sync the current branch only with depth set to None.
2338 # We always pass depth=None down to avoid infinite recursion.
2339 return self._RemoteFetch(
Mike Frysinger7b586f22021-02-23 18:38:39 -05002340 name=name, quiet=quiet, verbose=verbose, output_redir=output_redir,
Mike Frysinger521d01b2020-02-17 01:51:49 -05002341 current_branch_only=current_branch_only and depth,
2342 initial=False, alt_dir=alt_dir,
Mike Frysinger19e409c2021-05-05 19:44:35 -04002343 depth=None, ssh_proxy=ssh_proxy, clone_filter=clone_filter)
Brian Harring14a66742012-09-28 20:21:57 -07002344
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002345 return ok
Shawn O. Pearce88443382010-10-08 10:02:09 +02002346
Mike Frysingere50b6a72020-02-19 01:45:48 -05002347 def _ApplyCloneBundle(self, initial=False, quiet=False, verbose=False):
LaMont Jonesd82be3e2022-04-05 19:30:46 +00002348 if initial and (self.manifest.manifestProject.depth or self.clone_depth):
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002349 return False
2350
Mike Frysingerdede5642022-07-10 04:56:04 -04002351 remote = self.GetRemote()
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002352 bundle_url = remote.url + '/clone.bundle'
2353 bundle_url = GitConfig.ForUser().UrlInsteadOf(bundle_url)
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002354 if GetSchemeFromUrl(bundle_url) not in ('http', 'https',
2355 'persistent-http',
2356 'persistent-https'):
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002357 return False
2358
2359 bundle_dst = os.path.join(self.gitdir, 'clone.bundle')
2360 bundle_tmp = os.path.join(self.gitdir, 'clone.bundle.tmp')
2361
2362 exist_dst = os.path.exists(bundle_dst)
2363 exist_tmp = os.path.exists(bundle_tmp)
2364
2365 if not initial and not exist_dst and not exist_tmp:
2366 return False
2367
2368 if not exist_dst:
Mike Frysingere50b6a72020-02-19 01:45:48 -05002369 exist_dst = self._FetchBundle(bundle_url, bundle_tmp, bundle_dst, quiet,
2370 verbose)
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002371 if not exist_dst:
2372 return False
2373
2374 cmd = ['fetch']
Mike Frysinger4847e052020-02-22 00:07:35 -05002375 if not verbose:
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002376 cmd.append('--quiet')
Mike Frysinger4847e052020-02-22 00:07:35 -05002377 if not quiet and sys.stdout.isatty():
2378 cmd.append('--progress')
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002379 if not self.worktree:
2380 cmd.append('--update-head-ok')
2381 cmd.append(bundle_dst)
2382 for f in remote.fetch:
2383 cmd.append(str(f))
Xin Li6e538442018-12-10 11:33:16 -08002384 cmd.append('+refs/tags/*:refs/tags/*')
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002385
Mike Frysinger67d6cdf2021-12-23 17:36:09 -05002386 ok = GitCommand(
2387 self, cmd, bare=True, objdir=os.path.join(self.objdir, 'objects')).Wait() == 0
Mike Frysinger9d96f582021-09-28 11:27:24 -04002388 platform_utils.remove(bundle_dst, missing_ok=True)
2389 platform_utils.remove(bundle_tmp, missing_ok=True)
Shawn O. Pearce88443382010-10-08 10:02:09 +02002390 return ok
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002391
Mike Frysingere50b6a72020-02-19 01:45:48 -05002392 def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet, verbose):
Mike Frysinger9d96f582021-09-28 11:27:24 -04002393 platform_utils.remove(dstPath, missing_ok=True)
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07002394
Matt Gumbel2dc810c2012-08-30 09:39:36 -07002395 cmd = ['curl', '--fail', '--output', tmpPath, '--netrc', '--location']
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002396 if quiet:
Mike Frysingere50b6a72020-02-19 01:45:48 -05002397 cmd += ['--silent', '--show-error']
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002398 if os.path.exists(tmpPath):
2399 size = os.stat(tmpPath).st_size
2400 if size >= 1024:
2401 cmd += ['--continue-at', '%d' % (size,)]
2402 else:
Renaud Paquay010fed72016-11-11 14:25:29 -08002403 platform_utils.remove(tmpPath)
Xin Li3698ab72019-06-25 16:09:43 -07002404 with GetUrlCookieFile(srcUrl, quiet) as (cookiefile, proxy):
Dave Borowitz137d0132015-01-02 11:12:54 -08002405 if cookiefile:
Mike Frysingerdc1d0e02020-02-09 16:20:06 -05002406 cmd += ['--cookie', cookiefile]
Xin Li3698ab72019-06-25 16:09:43 -07002407 if proxy:
2408 cmd += ['--proxy', proxy]
2409 elif 'http_proxy' in os.environ and 'darwin' == sys.platform:
2410 cmd += ['--proxy', os.environ['http_proxy']]
2411 if srcUrl.startswith('persistent-https'):
2412 srcUrl = 'http' + srcUrl[len('persistent-https'):]
2413 elif srcUrl.startswith('persistent-http'):
2414 srcUrl = 'http' + srcUrl[len('persistent-http'):]
Dave Borowitz137d0132015-01-02 11:12:54 -08002415 cmd += [srcUrl]
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002416
Dave Borowitz137d0132015-01-02 11:12:54 -08002417 if IsTrace():
2418 Trace('%s', ' '.join(cmd))
Mike Frysingere50b6a72020-02-19 01:45:48 -05002419 if verbose:
2420 print('%s: Downloading bundle: %s' % (self.name, srcUrl))
2421 stdout = None if verbose else subprocess.PIPE
2422 stderr = None if verbose else subprocess.STDOUT
Dave Borowitz137d0132015-01-02 11:12:54 -08002423 try:
Mike Frysingere50b6a72020-02-19 01:45:48 -05002424 proc = subprocess.Popen(cmd, stdout=stdout, stderr=stderr)
Dave Borowitz137d0132015-01-02 11:12:54 -08002425 except OSError:
2426 return False
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002427
Mike Frysingere50b6a72020-02-19 01:45:48 -05002428 (output, _) = proc.communicate()
2429 curlret = proc.returncode
Matt Gumbel2dc810c2012-08-30 09:39:36 -07002430
Dave Borowitz137d0132015-01-02 11:12:54 -08002431 if curlret == 22:
2432 # From curl man page:
2433 # 22: HTTP page not retrieved. The requested url was not found or
2434 # returned another error with the HTTP error code being 400 or above.
2435 # This return code only appears if -f, --fail is used.
Mike Frysinger4847e052020-02-22 00:07:35 -05002436 if verbose:
George Engelbrechtb6871892020-04-02 13:53:01 -06002437 print('%s: Unable to retrieve clone.bundle; ignoring.' % self.name)
2438 if output:
George Engelbrecht2fe84e12020-04-15 11:28:00 -06002439 print('Curl output:\n%s' % output)
Dave Borowitz137d0132015-01-02 11:12:54 -08002440 return False
Mike Frysingere50b6a72020-02-19 01:45:48 -05002441 elif curlret and not verbose and output:
2442 print('%s' % output, file=sys.stderr)
Matt Gumbel2dc810c2012-08-30 09:39:36 -07002443
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002444 if os.path.exists(tmpPath):
Kris Giesingc8d882a2014-12-23 13:02:32 -08002445 if curlret == 0 and self._IsValidBundle(tmpPath, quiet):
Renaud Paquayad1abcb2016-11-01 11:34:55 -07002446 platform_utils.rename(tmpPath, dstPath)
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002447 return True
2448 else:
Renaud Paquay010fed72016-11-11 14:25:29 -08002449 platform_utils.remove(tmpPath)
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002450 return False
2451 else:
2452 return False
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07002453
Kris Giesingc8d882a2014-12-23 13:02:32 -08002454 def _IsValidBundle(self, path, quiet):
Dave Borowitz91f3ba52013-06-03 12:15:23 -07002455 try:
Pierre Tardy2b7daff2019-05-16 10:28:21 +02002456 with open(path, 'rb') as f:
2457 if f.read(16) == b'# v2 git bundle\n':
Dave Borowitz91f3ba52013-06-03 12:15:23 -07002458 return True
2459 else:
Kris Giesingc8d882a2014-12-23 13:02:32 -08002460 if not quiet:
2461 print("Invalid clone.bundle file; ignoring.", file=sys.stderr)
Dave Borowitz91f3ba52013-06-03 12:15:23 -07002462 return False
2463 except OSError:
2464 return False
2465
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002466 def _Checkout(self, rev, quiet=False):
2467 cmd = ['checkout']
2468 if quiet:
2469 cmd.append('-q')
2470 cmd.append(rev)
2471 cmd.append('--')
2472 if GitCommand(self, cmd).Wait() != 0:
2473 if self._allrefs:
2474 raise GitError('%s checkout %s ' % (self.name, rev))
2475
Mike Frysinger915fda12020-03-22 12:15:20 -04002476 def _CherryPick(self, rev, ffonly=False, record_origin=False):
Pierre Tardye5a21222011-03-24 16:28:18 +01002477 cmd = ['cherry-pick']
Mike Frysingerea431762020-03-22 12:14:01 -04002478 if ffonly:
2479 cmd.append('--ff')
Mike Frysinger915fda12020-03-22 12:15:20 -04002480 if record_origin:
2481 cmd.append('-x')
Pierre Tardye5a21222011-03-24 16:28:18 +01002482 cmd.append(rev)
2483 cmd.append('--')
2484 if GitCommand(self, cmd).Wait() != 0:
2485 if self._allrefs:
2486 raise GitError('%s cherry-pick %s ' % (self.name, rev))
2487
Akshay Verma0f2e45a2018-03-24 12:27:05 +05302488 def _LsRemote(self, refs):
2489 cmd = ['ls-remote', self.remote.name, refs]
Akshay Vermacf7c0832018-03-15 21:56:30 +05302490 p = GitCommand(self, cmd, capture_stdout=True)
2491 if p.Wait() == 0:
Mike Frysinger600f4922019-08-03 02:14:28 -04002492 return p.stdout
Akshay Vermacf7c0832018-03-15 21:56:30 +05302493 return None
2494
Anthony King7bdac712014-07-16 12:56:40 +01002495 def _Revert(self, rev):
Erwan Mahea94f1622011-08-19 13:56:09 +02002496 cmd = ['revert']
2497 cmd.append('--no-edit')
2498 cmd.append(rev)
2499 cmd.append('--')
2500 if GitCommand(self, cmd).Wait() != 0:
2501 if self._allrefs:
2502 raise GitError('%s revert %s ' % (self.name, rev))
2503
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002504 def _ResetHard(self, rev, quiet=True):
2505 cmd = ['reset', '--hard']
2506 if quiet:
2507 cmd.append('-q')
2508 cmd.append(rev)
2509 if GitCommand(self, cmd).Wait() != 0:
2510 raise GitError('%s reset --hard %s ' % (self.name, rev))
2511
Martin Kellye4e94d22017-03-21 16:05:12 -07002512 def _SyncSubmodules(self, quiet=True):
2513 cmd = ['submodule', 'update', '--init', '--recursive']
2514 if quiet:
2515 cmd.append('-q')
2516 if GitCommand(self, cmd).Wait() != 0:
LaMont Jones7b9b2512021-11-03 20:48:27 +00002517 raise GitError('%s submodule update --init --recursive ' % self.name)
Martin Kellye4e94d22017-03-21 16:05:12 -07002518
Anthony King7bdac712014-07-16 12:56:40 +01002519 def _Rebase(self, upstream, onto=None):
Shawn O. Pearce19a83d82009-04-16 08:14:26 -07002520 cmd = ['rebase']
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002521 if onto is not None:
2522 cmd.extend(['--onto', onto])
2523 cmd.append(upstream)
Shawn O. Pearce19a83d82009-04-16 08:14:26 -07002524 if GitCommand(self, cmd).Wait() != 0:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002525 raise GitError('%s rebase %s ' % (self.name, upstream))
2526
Pierre Tardy3d125942012-05-04 12:18:12 +02002527 def _FastForward(self, head, ffonly=False):
Mike Frysinger2b1345b2020-02-17 01:07:11 -05002528 cmd = ['merge', '--no-stat', head]
Pierre Tardy3d125942012-05-04 12:18:12 +02002529 if ffonly:
2530 cmd.append("--ff-only")
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002531 if GitCommand(self, cmd).Wait() != 0:
2532 raise GitError('%s merge %s ' % (self.name, head))
2533
David Pursehousee8ace262020-02-13 12:41:15 +09002534 def _InitGitDir(self, mirror_git=None, force_sync=False, quiet=False):
Kevin Degi384b3c52014-10-16 16:02:58 -06002535 init_git_dir = not os.path.exists(self.gitdir)
2536 init_obj_dir = not os.path.exists(self.objdir)
Kevin Degib1a07b82015-07-27 13:33:43 -06002537 try:
2538 # Initialize the bare repository, which contains all of the objects.
2539 if init_obj_dir:
2540 os.makedirs(self.objdir)
2541 self.bare_objdir.init()
David James8d201162013-10-11 17:03:19 -07002542
Mike Frysinger333c0a42021-11-15 12:39:00 -05002543 self._UpdateHooks(quiet=quiet)
2544
Mike Frysinger21b7fbe2020-02-26 23:53:36 -05002545 if self.use_git_worktrees:
Mike Frysinger21b7fbe2020-02-26 23:53:36 -05002546 # Enable per-worktree config file support if possible. This is more a
2547 # nice-to-have feature for users rather than a hard requirement.
Adrien Bioteau65f51ad2020-07-24 14:56:20 +02002548 if git_require((2, 20, 0)):
Mike Frysinger21b7fbe2020-02-26 23:53:36 -05002549 self.EnableRepositoryExtension('worktreeConfig')
Mike Frysinger979d5bd2020-02-09 02:28:34 -05002550
Kevin Degib1a07b82015-07-27 13:33:43 -06002551 # If we have a separate directory to hold refs, initialize it as well.
2552 if self.objdir != self.gitdir:
2553 if init_git_dir:
2554 os.makedirs(self.gitdir)
2555
2556 if init_obj_dir or init_git_dir:
Mike Frysingerc72bd842021-11-14 03:58:00 -05002557 self._ReferenceGitDir(self.objdir, self.gitdir, copy_all=True)
Kevin Degiabaa7f32014-11-12 11:27:45 -07002558 try:
Mike Frysingerc72bd842021-11-14 03:58:00 -05002559 self._CheckDirReference(self.objdir, self.gitdir)
Kevin Degiabaa7f32014-11-12 11:27:45 -07002560 except GitError as e:
Kevin Degiabaa7f32014-11-12 11:27:45 -07002561 if force_sync:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002562 print("Retrying clone after deleting %s" %
2563 self.gitdir, file=sys.stderr)
Kevin Degiabaa7f32014-11-12 11:27:45 -07002564 try:
Renaud Paquay227ad2e2016-11-01 14:37:13 -07002565 platform_utils.rmtree(platform_utils.realpath(self.gitdir))
2566 if self.worktree and os.path.exists(platform_utils.realpath
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002567 (self.worktree)):
Renaud Paquay227ad2e2016-11-01 14:37:13 -07002568 platform_utils.rmtree(platform_utils.realpath(self.worktree))
David Pursehousee8ace262020-02-13 12:41:15 +09002569 return self._InitGitDir(mirror_git=mirror_git, force_sync=False,
2570 quiet=quiet)
David Pursehouse145e35b2020-02-12 15:40:47 +09002571 except Exception:
Kevin Degiabaa7f32014-11-12 11:27:45 -07002572 raise e
2573 raise e
Kevin Degib1a07b82015-07-27 13:33:43 -06002574
Kevin Degi384b3c52014-10-16 16:02:58 -06002575 if init_git_dir:
Kevin Degib1a07b82015-07-27 13:33:43 -06002576 mp = self.manifest.manifestProject
LaMont Jonesd82be3e2022-04-05 19:30:46 +00002577 ref_dir = mp.reference or ''
Kevin Degi384b3c52014-10-16 16:02:58 -06002578
LaMont Jonescc879a92021-11-18 22:40:18 +00002579 def _expanded_ref_dirs():
2580 """Iterate through the possible git reference directory paths."""
2581 name = self.name + '.git'
2582 yield mirror_git or os.path.join(ref_dir, name)
2583 for prefix in '', self.remote.name:
2584 yield os.path.join(ref_dir, '.repo', 'project-objects', prefix, name)
2585 yield os.path.join(ref_dir, '.repo', 'worktrees', prefix, name)
Shawn O. Pearce2816d4f2009-03-03 17:53:18 -08002586
LaMont Jonescc879a92021-11-18 22:40:18 +00002587 if ref_dir or mirror_git:
2588 found_ref_dir = None
2589 for path in _expanded_ref_dirs():
2590 if os.path.exists(path):
2591 found_ref_dir = path
2592 break
2593 ref_dir = found_ref_dir
Shawn O. Pearce88443382010-10-08 10:02:09 +02002594
Kevin Degib1a07b82015-07-27 13:33:43 -06002595 if ref_dir:
Samuel Hollandbaa00092018-01-22 10:57:29 -06002596 if not os.path.isabs(ref_dir):
2597 # The alternate directory is relative to the object database.
2598 ref_dir = os.path.relpath(ref_dir,
2599 os.path.join(self.objdir, 'objects'))
Mike Frysinger152032c2021-12-20 21:17:43 -05002600 _lwrite(os.path.join(self.objdir, 'objects/info/alternates'),
Kevin Degib1a07b82015-07-27 13:33:43 -06002601 os.path.join(ref_dir, 'objects') + '\n')
Shawn O. Pearce88443382010-10-08 10:02:09 +02002602
Kevin Degib1a07b82015-07-27 13:33:43 -06002603 m = self.manifest.manifestProject.config
2604 for key in ['user.name', 'user.email']:
2605 if m.Has(key, include_defaults=False):
2606 self.config.SetString(key, m.GetString(key))
XD Trol630876f2022-01-17 23:29:04 +08002607 if not self.manifest.EnableGitLfs:
2608 self.config.SetString('filter.lfs.smudge', 'git-lfs smudge --skip -- %f')
2609 self.config.SetString('filter.lfs.process', 'git-lfs filter-process --skip')
Mike Frysinger38867fb2021-02-09 23:14:41 -05002610 self.config.SetBoolean('core.bare', True if self.manifest.IsMirror else None)
Kevin Degib1a07b82015-07-27 13:33:43 -06002611 except Exception:
2612 if init_obj_dir and os.path.exists(self.objdir):
Renaud Paquaya65adf72016-11-03 10:37:53 -07002613 platform_utils.rmtree(self.objdir)
Kevin Degib1a07b82015-07-27 13:33:43 -06002614 if init_git_dir and os.path.exists(self.gitdir):
Renaud Paquaya65adf72016-11-03 10:37:53 -07002615 platform_utils.rmtree(self.gitdir)
Kevin Degib1a07b82015-07-27 13:33:43 -06002616 raise
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002617
David Pursehousee8ace262020-02-13 12:41:15 +09002618 def _UpdateHooks(self, quiet=False):
Mike Frysinger333c0a42021-11-15 12:39:00 -05002619 if os.path.exists(self.objdir):
David Pursehousee8ace262020-02-13 12:41:15 +09002620 self._InitHooks(quiet=quiet)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002621
David Pursehousee8ace262020-02-13 12:41:15 +09002622 def _InitHooks(self, quiet=False):
Mike Frysinger333c0a42021-11-15 12:39:00 -05002623 hooks = platform_utils.realpath(os.path.join(self.objdir, 'hooks'))
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08002624 if not os.path.exists(hooks):
2625 os.makedirs(hooks)
Mike Frysinger98bb7652021-12-20 21:15:59 -05002626
2627 # Delete sample hooks. They're noise.
2628 for hook in glob.glob(os.path.join(hooks, '*.sample')):
Peter Kjellerstedtb5505012022-01-21 23:09:19 +01002629 try:
2630 platform_utils.remove(hook, missing_ok=True)
2631 except PermissionError:
2632 pass
Mike Frysinger98bb7652021-12-20 21:15:59 -05002633
Jonathan Nieder93719792015-03-17 11:29:58 -07002634 for stock_hook in _ProjectHooks():
Shawn O. Pearce9452e4e2009-08-22 18:17:46 -07002635 name = os.path.basename(stock_hook)
2636
Victor Boivie65e0f352011-04-18 11:23:29 +02002637 if name in ('commit-msg',) and not self.remote.review \
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002638 and self is not self.manifest.manifestProject:
Shawn O. Pearce9452e4e2009-08-22 18:17:46 -07002639 # Don't install a Gerrit Code Review hook if this
2640 # project does not appear to use it for reviews.
2641 #
Victor Boivie65e0f352011-04-18 11:23:29 +02002642 # Since the manifest project is one of those, but also
2643 # managed through gerrit, it's excluded
Shawn O. Pearce9452e4e2009-08-22 18:17:46 -07002644 continue
2645
2646 dst = os.path.join(hooks, name)
Renaud Paquay227ad2e2016-11-01 14:37:13 -07002647 if platform_utils.islink(dst):
Shawn O. Pearce9452e4e2009-08-22 18:17:46 -07002648 continue
2649 if os.path.exists(dst):
Mike Frysinger7951e142020-02-21 00:04:15 -05002650 # If the files are the same, we'll leave it alone. We create symlinks
2651 # below by default but fallback to hardlinks if the OS blocks them.
2652 # So if we're here, it's probably because we made a hardlink below.
2653 if not filecmp.cmp(stock_hook, dst, shallow=False):
David Pursehousee8ace262020-02-13 12:41:15 +09002654 if not quiet:
2655 _warn("%s: Not replacing locally modified %s hook",
LaMont Jones8501d462022-06-22 19:21:15 +00002656 self.RelPath(local=False), name)
Mike Frysinger7951e142020-02-21 00:04:15 -05002657 continue
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08002658 try:
Renaud Paquayd5cec5e2016-11-01 11:24:03 -07002659 platform_utils.symlink(
2660 os.path.relpath(stock_hook, os.path.dirname(dst)), dst)
Sarah Owensa5be53f2012-09-09 15:37:57 -07002661 except OSError as e:
Shawn O. Pearce9452e4e2009-08-22 18:17:46 -07002662 if e.errno == errno.EPERM:
Mike Frysinger7951e142020-02-21 00:04:15 -05002663 try:
2664 os.link(stock_hook, dst)
2665 except OSError:
2666 raise GitError(self._get_symlink_error_message())
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08002667 else:
2668 raise
2669
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002670 def _InitRemote(self):
Shawn O. Pearced1f70d92009-05-19 14:58:02 -07002671 if self.remote.url:
Mike Frysingerdede5642022-07-10 04:56:04 -04002672 remote = self.GetRemote()
Shawn O. Pearced1f70d92009-05-19 14:58:02 -07002673 remote.url = self.remote.url
Steve Raed6480452016-08-10 15:00:00 -07002674 remote.pushUrl = self.remote.pushUrl
Shawn O. Pearced1f70d92009-05-19 14:58:02 -07002675 remote.review = self.remote.review
2676 remote.projectname = self.name
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002677
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08002678 if self.worktree:
2679 remote.ResetFetch(mirror=False)
2680 else:
2681 remote.ResetFetch(mirror=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002682 remote.Save()
2683
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002684 def _InitMRef(self):
Mike Frysinger790f4ce2020-12-07 22:04:55 -05002685 """Initialize the pseudo m/<manifest branch> ref."""
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002686 if self.manifest.branch:
Mike Frysinger21b7fbe2020-02-26 23:53:36 -05002687 if self.use_git_worktrees:
Mike Frysinger29626b42021-05-01 09:37:13 -04002688 # Set up the m/ space to point to the worktree-specific ref space.
2689 # We'll update the worktree-specific ref space on each checkout.
2690 ref = R_M + self.manifest.branch
2691 if not self.bare_ref.symref(ref):
2692 self.bare_git.symbolic_ref(
2693 '-m', 'redirecting to worktree scope',
2694 ref, R_WORKTREE_M + self.manifest.branch)
2695
Mike Frysinger21b7fbe2020-02-26 23:53:36 -05002696 # We can't update this ref with git worktrees until it exists.
2697 # We'll wait until the initial checkout to set it.
2698 if not os.path.exists(self.worktree):
2699 return
2700
2701 base = R_WORKTREE_M
2702 active_git = self.work_git
Remy Böhmer1469c282020-12-15 18:49:02 +01002703
2704 self._InitAnyMRef(HEAD, self.bare_git, detach=True)
Mike Frysinger21b7fbe2020-02-26 23:53:36 -05002705 else:
2706 base = R_M
2707 active_git = self.bare_git
2708
2709 self._InitAnyMRef(base + self.manifest.branch, active_git)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002710
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08002711 def _InitMirrorHead(self):
Mike Frysinger21b7fbe2020-02-26 23:53:36 -05002712 self._InitAnyMRef(HEAD, self.bare_git)
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002713
Remy Böhmer1469c282020-12-15 18:49:02 +01002714 def _InitAnyMRef(self, ref, active_git, detach=False):
Mike Frysinger790f4ce2020-12-07 22:04:55 -05002715 """Initialize |ref| in |active_git| to the value in the manifest.
2716
2717 This points |ref| to the <project> setting in the manifest.
2718
2719 Args:
2720 ref: The branch to update.
2721 active_git: The git repository to make updates in.
2722 detach: Whether to update target of symbolic refs, or overwrite the ref
2723 directly (and thus make it non-symbolic).
2724 """
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002725 cur = self.bare_ref.symref(ref)
2726
2727 if self.revisionId:
2728 if cur != '' or self.bare_ref.get(ref) != self.revisionId:
2729 msg = 'manifest set to %s' % self.revisionId
2730 dst = self.revisionId + '^0'
Mike Frysinger21b7fbe2020-02-26 23:53:36 -05002731 active_git.UpdateRef(ref, dst, message=msg, detach=True)
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002732 else:
Mike Frysingerdede5642022-07-10 04:56:04 -04002733 remote = self.GetRemote()
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002734 dst = remote.ToLocal(self.revisionExpr)
2735 if cur != dst:
2736 msg = 'manifest set to %s' % self.revisionExpr
Remy Böhmer1469c282020-12-15 18:49:02 +01002737 if detach:
2738 active_git.UpdateRef(ref, dst, message=msg, detach=True)
2739 else:
2740 active_git.symbolic_ref('-m', msg, ref, dst)
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08002741
Mike Frysingerc72bd842021-11-14 03:58:00 -05002742 def _CheckDirReference(self, srcdir, destdir):
Mike Frysinger979d5bd2020-02-09 02:28:34 -05002743 # Git worktrees don't use symlinks to share at all.
2744 if self.use_git_worktrees:
2745 return
2746
Mike Frysingerd33dce02021-12-20 18:16:33 -05002747 for name in self.shareable_dirs:
Mike Frysingered4f2112020-02-11 23:06:29 -05002748 # Try to self-heal a bit in simple cases.
2749 dst_path = os.path.join(destdir, name)
2750 src_path = os.path.join(srcdir, name)
2751
Mike Frysingered4f2112020-02-11 23:06:29 -05002752 dst = platform_utils.realpath(dst_path)
Kevin Degi384b3c52014-10-16 16:02:58 -06002753 if os.path.lexists(dst):
Mike Frysingered4f2112020-02-11 23:06:29 -05002754 src = platform_utils.realpath(src_path)
Kevin Degi384b3c52014-10-16 16:02:58 -06002755 # Fail if the links are pointing to the wrong place
2756 if src != dst:
Marc Herbertec287902016-10-27 12:58:26 -07002757 _error('%s is different in %s vs %s', name, destdir, srcdir)
Kevin Degiabaa7f32014-11-12 11:27:45 -07002758 raise GitError('--force-sync not enabled; cannot overwrite a local '
Simon Ruggierf9b76832015-07-31 17:18:34 -04002759 'work tree. If you\'re comfortable with the '
2760 'possibility of losing the work tree\'s git metadata,'
2761 ' use `repo sync --force-sync {0}` to '
LaMont Jones8501d462022-06-22 19:21:15 +00002762 'proceed.'.format(self.RelPath(local=False)))
Kevin Degi384b3c52014-10-16 16:02:58 -06002763
Mike Frysingerc72bd842021-11-14 03:58:00 -05002764 def _ReferenceGitDir(self, gitdir, dotgit, copy_all):
David James8d201162013-10-11 17:03:19 -07002765 """Update |dotgit| to reference |gitdir|, using symlinks where possible.
2766
2767 Args:
2768 gitdir: The bare git repository. Must already be initialized.
2769 dotgit: The repository you would like to initialize.
David James8d201162013-10-11 17:03:19 -07002770 copy_all: If true, copy all remaining files from |gitdir| -> |dotgit|.
2771 This saves you the effort of initializing |dotgit| yourself.
2772 """
Dan Willemsenbdb866e2016-04-05 17:22:02 -07002773 symlink_dirs = self.shareable_dirs[:]
Mike Frysingerd33dce02021-12-20 18:16:33 -05002774 to_symlink = symlink_dirs
David James8d201162013-10-11 17:03:19 -07002775
2776 to_copy = []
2777 if copy_all:
Renaud Paquaybed8b622018-09-27 10:46:58 -07002778 to_copy = platform_utils.listdir(gitdir)
David James8d201162013-10-11 17:03:19 -07002779
Renaud Paquay227ad2e2016-11-01 14:37:13 -07002780 dotgit = platform_utils.realpath(dotgit)
David James8d201162013-10-11 17:03:19 -07002781 for name in set(to_copy).union(to_symlink):
2782 try:
Renaud Paquay227ad2e2016-11-01 14:37:13 -07002783 src = platform_utils.realpath(os.path.join(gitdir, name))
Dan Willemsen2a3e1522015-07-30 20:43:33 -07002784 dst = os.path.join(dotgit, name)
David James8d201162013-10-11 17:03:19 -07002785
Kevin Degi384b3c52014-10-16 16:02:58 -06002786 if os.path.lexists(dst):
2787 continue
David James8d201162013-10-11 17:03:19 -07002788
2789 # If the source dir doesn't exist, create an empty dir.
2790 if name in symlink_dirs and not os.path.lexists(src):
2791 os.makedirs(src)
2792
Cheuk Leung8ac0c962015-07-06 21:33:00 +02002793 if name in to_symlink:
Renaud Paquayd5cec5e2016-11-01 11:24:03 -07002794 platform_utils.symlink(
2795 os.path.relpath(src, os.path.dirname(dst)), dst)
Renaud Paquay227ad2e2016-11-01 14:37:13 -07002796 elif copy_all and not platform_utils.islink(dst):
Renaud Paquaybed8b622018-09-27 10:46:58 -07002797 if platform_utils.isdir(src):
Cheuk Leung8ac0c962015-07-06 21:33:00 +02002798 shutil.copytree(src, dst)
2799 elif os.path.isfile(src):
2800 shutil.copy(src, dst)
2801
David James8d201162013-10-11 17:03:19 -07002802 except OSError as e:
2803 if e.errno == errno.EPERM:
Renaud Paquay788e9622017-01-27 11:41:12 -08002804 raise DownloadError(self._get_symlink_error_message())
David James8d201162013-10-11 17:03:19 -07002805 else:
2806 raise
2807
Mike Frysinger979d5bd2020-02-09 02:28:34 -05002808 def _InitGitWorktree(self):
2809 """Init the project using git worktrees."""
2810 self.bare_git.worktree('prune')
2811 self.bare_git.worktree('add', '-ff', '--checkout', '--detach', '--lock',
2812 self.worktree, self.GetRevisionId())
2813
2814 # Rewrite the internal state files to use relative paths between the
2815 # checkouts & worktrees.
2816 dotgit = os.path.join(self.worktree, '.git')
2817 with open(dotgit, 'r') as fp:
2818 # Figure out the checkout->worktree path.
2819 setting = fp.read()
2820 assert setting.startswith('gitdir:')
2821 git_worktree_path = setting.split(':', 1)[1].strip()
Mike Frysinger75264782020-02-21 18:55:07 -05002822 # Some platforms (e.g. Windows) won't let us update dotgit in situ because
2823 # of file permissions. Delete it and recreate it from scratch to avoid.
2824 platform_utils.remove(dotgit)
Remy Bohmer44bc9642020-11-20 21:19:10 +01002825 # Use relative path from checkout->worktree & maintain Unix line endings
2826 # on all OS's to match git behavior.
2827 with open(dotgit, 'w', newline='\n') as fp:
Mike Frysinger979d5bd2020-02-09 02:28:34 -05002828 print('gitdir:', os.path.relpath(git_worktree_path, self.worktree),
2829 file=fp)
Remy Bohmer44bc9642020-11-20 21:19:10 +01002830 # Use relative path from worktree->checkout & maintain Unix line endings
2831 # on all OS's to match git behavior.
2832 with open(os.path.join(git_worktree_path, 'gitdir'), 'w', newline='\n') as fp:
Mike Frysinger979d5bd2020-02-09 02:28:34 -05002833 print(os.path.relpath(dotgit, git_worktree_path), file=fp)
2834
Mike Frysinger21b7fbe2020-02-26 23:53:36 -05002835 self._InitMRef()
2836
Martin Kellye4e94d22017-03-21 16:05:12 -07002837 def _InitWorkTree(self, force_sync=False, submodules=False):
Mike Frysinger2a089cf2021-11-13 23:29:42 -05002838 """Setup the worktree .git path.
2839
2840 This is the user-visible path like src/foo/.git/.
2841
2842 With non-git-worktrees, this will be a symlink to the .repo/projects/ path.
2843 With git-worktrees, this will be a .git file using "gitdir: ..." syntax.
2844
2845 Older checkouts had .git/ directories. If we see that, migrate it.
2846
2847 This also handles changes in the manifest. Maybe this project was backed
2848 by "foo/bar" on the server, but now it's "new/foo/bar". We have to update
2849 the path we point to under .repo/projects/ to match.
2850 """
2851 dotgit = os.path.join(self.worktree, '.git')
2852
2853 # If using an old layout style (a directory), migrate it.
2854 if not platform_utils.islink(dotgit) and platform_utils.isdir(dotgit):
2855 self._MigrateOldWorkTreeGitDir(dotgit)
2856
2857 init_dotgit = not os.path.exists(dotgit)
2858 if self.use_git_worktrees:
2859 if init_dotgit:
Mike Frysinger979d5bd2020-02-09 02:28:34 -05002860 self._InitGitWorktree()
2861 self._CopyAndLinkFiles()
Mike Frysingerf4545122019-11-11 04:34:16 -05002862 else:
Mike Frysinger2a089cf2021-11-13 23:29:42 -05002863 if not init_dotgit:
2864 # See if the project has changed.
2865 if platform_utils.realpath(self.gitdir) != platform_utils.realpath(dotgit):
2866 platform_utils.remove(dotgit)
Mike Frysingerf4545122019-11-11 04:34:16 -05002867
Mike Frysinger2a089cf2021-11-13 23:29:42 -05002868 if init_dotgit or not os.path.exists(dotgit):
2869 os.makedirs(self.worktree, exist_ok=True)
2870 platform_utils.symlink(os.path.relpath(self.gitdir, self.worktree), dotgit)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002871
Mike Frysinger2a089cf2021-11-13 23:29:42 -05002872 if init_dotgit:
2873 _lwrite(os.path.join(dotgit, HEAD), '%s\n' % self.GetRevisionId())
Kevin Degi384b3c52014-10-16 16:02:58 -06002874
Mike Frysinger2a089cf2021-11-13 23:29:42 -05002875 # Finish checking out the worktree.
2876 cmd = ['read-tree', '--reset', '-u', '-v', HEAD]
2877 if GitCommand(self, cmd).Wait() != 0:
2878 raise GitError('Cannot initialize work tree for ' + self.name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002879
Mike Frysinger2a089cf2021-11-13 23:29:42 -05002880 if submodules:
2881 self._SyncSubmodules(quiet=True)
2882 self._CopyAndLinkFiles()
Victor Boivie0960b5b2010-11-26 13:42:13 +01002883
Mike Frysinger2a089cf2021-11-13 23:29:42 -05002884 @classmethod
2885 def _MigrateOldWorkTreeGitDir(cls, dotgit):
2886 """Migrate the old worktree .git/ dir style to a symlink.
2887
2888 This logic specifically only uses state from |dotgit| to figure out where to
2889 move content and not |self|. This way if the backing project also changed
2890 places, we only do the .git/ dir to .git symlink migration here. The path
2891 updates will happen independently.
2892 """
2893 # Figure out where in .repo/projects/ it's pointing to.
2894 if not os.path.islink(os.path.join(dotgit, 'refs')):
2895 raise GitError(f'{dotgit}: unsupported checkout state')
2896 gitdir = os.path.dirname(os.path.realpath(os.path.join(dotgit, 'refs')))
2897
2898 # Remove known symlink paths that exist in .repo/projects/.
2899 KNOWN_LINKS = {
2900 'config', 'description', 'hooks', 'info', 'logs', 'objects',
2901 'packed-refs', 'refs', 'rr-cache', 'shallow', 'svn',
2902 }
2903 # Paths that we know will be in both, but are safe to clobber in .repo/projects/.
2904 SAFE_TO_CLOBBER = {
Mike Frysinger8e912482022-01-26 04:03:34 -05002905 'COMMIT_EDITMSG', 'FETCH_HEAD', 'HEAD', 'gc.log', 'gitk.cache', 'index',
2906 'ORIG_HEAD',
Mike Frysinger2a089cf2021-11-13 23:29:42 -05002907 }
2908
Mike Frysinger89ed8ac2022-01-06 05:42:24 -05002909 # First see if we'd succeed before starting the migration.
2910 unknown_paths = []
2911 for name in platform_utils.listdir(dotgit):
2912 # Ignore all temporary/backup names. These are common with vim & emacs.
2913 if name.endswith('~') or (name[0] == '#' and name[-1] == '#'):
2914 continue
2915
2916 dotgit_path = os.path.join(dotgit, name)
2917 if name in KNOWN_LINKS:
2918 if not platform_utils.islink(dotgit_path):
2919 unknown_paths.append(f'{dotgit_path}: should be a symlink')
2920 else:
2921 gitdir_path = os.path.join(gitdir, name)
2922 if name not in SAFE_TO_CLOBBER and os.path.exists(gitdir_path):
2923 unknown_paths.append(f'{dotgit_path}: unknown file; please file a bug')
2924 if unknown_paths:
2925 raise GitError('Aborting migration: ' + '\n'.join(unknown_paths))
2926
Mike Frysinger2a089cf2021-11-13 23:29:42 -05002927 # Now walk the paths and sync the .git/ to .repo/projects/.
2928 for name in platform_utils.listdir(dotgit):
2929 dotgit_path = os.path.join(dotgit, name)
Mike Frysinger89ed8ac2022-01-06 05:42:24 -05002930
2931 # Ignore all temporary/backup names. These are common with vim & emacs.
2932 if name.endswith('~') or (name[0] == '#' and name[-1] == '#'):
2933 platform_utils.remove(dotgit_path)
2934 elif name in KNOWN_LINKS:
2935 platform_utils.remove(dotgit_path)
Mike Frysinger2a089cf2021-11-13 23:29:42 -05002936 else:
2937 gitdir_path = os.path.join(gitdir, name)
Mike Frysinger89ed8ac2022-01-06 05:42:24 -05002938 platform_utils.remove(gitdir_path, missing_ok=True)
2939 platform_utils.rename(dotgit_path, gitdir_path)
Mike Frysinger2a089cf2021-11-13 23:29:42 -05002940
2941 # Now that the dir should be empty, clear it out, and symlink it over.
2942 platform_utils.rmdir(dotgit)
2943 platform_utils.symlink(os.path.relpath(gitdir, os.path.dirname(dotgit)), dotgit)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002944
Renaud Paquay788e9622017-01-27 11:41:12 -08002945 def _get_symlink_error_message(self):
2946 if platform_utils.isWindows():
2947 return ('Unable to create symbolic link. Please re-run the command as '
2948 'Administrator, or see '
2949 'https://github.com/git-for-windows/git/wiki/Symbolic-Links '
2950 'for other options.')
2951 return 'filesystem must support symlinks'
2952
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07002953 def _revlist(self, *args, **kw):
2954 a = []
2955 a.extend(args)
2956 a.append('--')
2957 return self.work_git.rev_list(*a, **kw)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002958
2959 @property
2960 def _allrefs(self):
Shawn O. Pearced237b692009-04-17 18:49:50 -07002961 return self.bare_ref.all
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002962
Sebastian Schuberth7ecccf62016-03-29 14:11:20 +02002963 def _getLogs(self, rev1, rev2, oneline=False, color=True, pretty_format=None):
Julien Camperguedd654222014-01-09 16:21:37 +01002964 """Get logs between two revisions of this project."""
2965 comp = '..'
2966 if rev1:
2967 revs = [rev1]
2968 if rev2:
2969 revs.extend([comp, rev2])
2970 cmd = ['log', ''.join(revs)]
2971 out = DiffColoring(self.config)
2972 if out.is_on and color:
2973 cmd.append('--color')
Sebastian Schuberth7ecccf62016-03-29 14:11:20 +02002974 if pretty_format is not None:
2975 cmd.append('--pretty=format:%s' % pretty_format)
Julien Camperguedd654222014-01-09 16:21:37 +01002976 if oneline:
2977 cmd.append('--oneline')
2978
2979 try:
2980 log = GitCommand(self, cmd, capture_stdout=True, capture_stderr=True)
2981 if log.Wait() == 0:
2982 return log.stdout
2983 except GitError:
2984 # worktree may not exist if groups changed for example. In that case,
2985 # try in gitdir instead.
2986 if not os.path.exists(self.worktree):
2987 return self.bare_git.log(*cmd[1:])
2988 else:
2989 raise
2990 return None
2991
Sebastian Schuberth7ecccf62016-03-29 14:11:20 +02002992 def getAddedAndRemovedLogs(self, toProject, oneline=False, color=True,
2993 pretty_format=None):
Julien Camperguedd654222014-01-09 16:21:37 +01002994 """Get the list of logs from this revision to given revisionId"""
2995 logs = {}
2996 selfId = self.GetRevisionId(self._allrefs)
2997 toId = toProject.GetRevisionId(toProject._allrefs)
2998
Sebastian Schuberth7ecccf62016-03-29 14:11:20 +02002999 logs['added'] = self._getLogs(selfId, toId, oneline=oneline, color=color,
3000 pretty_format=pretty_format)
3001 logs['removed'] = self._getLogs(toId, selfId, oneline=oneline, color=color,
3002 pretty_format=pretty_format)
Julien Camperguedd654222014-01-09 16:21:37 +01003003 return logs
3004
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003005 class _GitGetByExec(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07003006
David James8d201162013-10-11 17:03:19 -07003007 def __init__(self, project, bare, gitdir):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003008 self._project = project
3009 self._bare = bare
David James8d201162013-10-11 17:03:19 -07003010 self._gitdir = gitdir
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003011
Kimiyuki Onaka0501b292020-08-28 10:05:27 +09003012 # __getstate__ and __setstate__ are required for pickling because __getattr__ exists.
3013 def __getstate__(self):
3014 return (self._project, self._bare, self._gitdir)
3015
3016 def __setstate__(self, state):
3017 self._project, self._bare, self._gitdir = state
3018
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003019 def LsOthers(self):
3020 p = GitCommand(self._project,
3021 ['ls-files',
3022 '-z',
3023 '--others',
3024 '--exclude-standard'],
Anthony King7bdac712014-07-16 12:56:40 +01003025 bare=False,
David James8d201162013-10-11 17:03:19 -07003026 gitdir=self._gitdir,
Anthony King7bdac712014-07-16 12:56:40 +01003027 capture_stdout=True,
3028 capture_stderr=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003029 if p.Wait() == 0:
3030 out = p.stdout
3031 if out:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07003032 # Backslash is not anomalous
David Pursehouse65b0ba52018-06-24 16:21:51 +09003033 return out[:-1].split('\0')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003034 return []
3035
3036 def DiffZ(self, name, *args):
3037 cmd = [name]
3038 cmd.append('-z')
Eli Ribble2d095da2019-05-02 18:21:42 -07003039 cmd.append('--ignore-submodules')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003040 cmd.extend(args)
3041 p = GitCommand(self._project,
3042 cmd,
David James8d201162013-10-11 17:03:19 -07003043 gitdir=self._gitdir,
Anthony King7bdac712014-07-16 12:56:40 +01003044 bare=False,
3045 capture_stdout=True,
3046 capture_stderr=True)
Mike Frysinger84230002021-02-16 17:08:35 -05003047 p.Wait()
3048 r = {}
3049 out = p.stdout
3050 if out:
3051 out = iter(out[:-1].split('\0'))
3052 while out:
3053 try:
3054 info = next(out)
3055 path = next(out)
3056 except StopIteration:
3057 break
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003058
Mike Frysinger84230002021-02-16 17:08:35 -05003059 class _Info(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07003060
Mike Frysinger84230002021-02-16 17:08:35 -05003061 def __init__(self, path, omode, nmode, oid, nid, state):
3062 self.path = path
3063 self.src_path = None
3064 self.old_mode = omode
3065 self.new_mode = nmode
3066 self.old_id = oid
3067 self.new_id = nid
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003068
Mike Frysinger84230002021-02-16 17:08:35 -05003069 if len(state) == 1:
3070 self.status = state
3071 self.level = None
3072 else:
3073 self.status = state[:1]
3074 self.level = state[1:]
3075 while self.level.startswith('0'):
3076 self.level = self.level[1:]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003077
Mike Frysinger84230002021-02-16 17:08:35 -05003078 info = info[1:].split(' ')
3079 info = _Info(path, *info)
3080 if info.status in ('R', 'C'):
3081 info.src_path = info.path
3082 info.path = next(out)
3083 r[info.path] = info
3084 return r
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003085
Mike Frysinger4b0eb5a2020-02-23 23:22:34 -05003086 def GetDotgitPath(self, subpath=None):
3087 """Return the full path to the .git dir.
3088
3089 As a convenience, append |subpath| if provided.
3090 """
Shawn O. Pearce5b23f242009-04-17 18:43:33 -07003091 if self._bare:
Mike Frysinger4b0eb5a2020-02-23 23:22:34 -05003092 dotgit = self._gitdir
Shawn O. Pearce5b23f242009-04-17 18:43:33 -07003093 else:
Mike Frysinger4b0eb5a2020-02-23 23:22:34 -05003094 dotgit = os.path.join(self._project.worktree, '.git')
3095 if os.path.isfile(dotgit):
3096 # Git worktrees use a "gitdir:" syntax to point to the scratch space.
3097 with open(dotgit) as fp:
3098 setting = fp.read()
3099 assert setting.startswith('gitdir:')
3100 gitdir = setting.split(':', 1)[1].strip()
3101 dotgit = os.path.normpath(os.path.join(self._project.worktree, gitdir))
3102
3103 return dotgit if subpath is None else os.path.join(dotgit, subpath)
3104
3105 def GetHead(self):
3106 """Return the ref that HEAD points to."""
3107 path = self.GetDotgitPath(subpath=HEAD)
Conley Owens75ee0572012-11-15 17:33:11 -08003108 try:
Mike Frysinger3164d402019-11-11 05:40:22 -05003109 with open(path) as fd:
3110 line = fd.readline()
Dan Sandler53e902a2014-03-09 13:20:02 -04003111 except IOError as e:
3112 raise NoManifestException(path, str(e))
Shawn O. Pearce76ca9f82009-04-18 14:48:03 -07003113 try:
Chirayu Desai217ea7d2013-03-01 19:14:38 +05303114 line = line.decode()
3115 except AttributeError:
3116 pass
Shawn O. Pearce5b23f242009-04-17 18:43:33 -07003117 if line.startswith('ref: '):
3118 return line[5:-1]
3119 return line[:-1]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003120
3121 def SetHead(self, ref, message=None):
3122 cmdv = []
3123 if message is not None:
3124 cmdv.extend(['-m', message])
3125 cmdv.append(HEAD)
3126 cmdv.append(ref)
3127 self.symbolic_ref(*cmdv)
3128
3129 def DetachHead(self, new, message=None):
3130 cmdv = ['--no-deref']
3131 if message is not None:
3132 cmdv.extend(['-m', message])
3133 cmdv.append(HEAD)
3134 cmdv.append(new)
3135 self.update_ref(*cmdv)
3136
3137 def UpdateRef(self, name, new, old=None,
3138 message=None,
3139 detach=False):
3140 cmdv = []
3141 if message is not None:
3142 cmdv.extend(['-m', message])
3143 if detach:
3144 cmdv.append('--no-deref')
3145 cmdv.append(name)
3146 cmdv.append(new)
3147 if old is not None:
3148 cmdv.append(old)
3149 self.update_ref(*cmdv)
3150
3151 def DeleteRef(self, name, old=None):
3152 if not old:
3153 old = self.rev_parse(name)
3154 self.update_ref('-d', name, old)
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07003155 self._project.bare_ref.deleted(name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003156
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07003157 def rev_list(self, *args, **kw):
3158 if 'format' in kw:
3159 cmdv = ['log', '--pretty=format:%s' % kw['format']]
3160 else:
3161 cmdv = ['rev-list']
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003162 cmdv.extend(args)
3163 p = GitCommand(self._project,
3164 cmdv,
Anthony King7bdac712014-07-16 12:56:40 +01003165 bare=self._bare,
David James8d201162013-10-11 17:03:19 -07003166 gitdir=self._gitdir,
Anthony King7bdac712014-07-16 12:56:40 +01003167 capture_stdout=True,
3168 capture_stderr=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003169 if p.Wait() != 0:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07003170 raise GitError('%s rev-list %s: %s' %
3171 (self._project.name, str(args), p.stderr))
Mike Frysinger81f5c592019-07-04 18:13:31 -04003172 return p.stdout.splitlines()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003173
3174 def __getattr__(self, name):
Doug Anderson37282b42011-03-04 11:54:18 -08003175 """Allow arbitrary git commands using pythonic syntax.
3176
3177 This allows you to do things like:
3178 git_obj.rev_parse('HEAD')
3179
3180 Since we don't have a 'rev_parse' method defined, the __getattr__ will
3181 run. We'll replace the '_' with a '-' and try to run a git command.
Dave Borowitz091f8932012-10-23 17:01:04 -07003182 Any other positional arguments will be passed to the git command, and the
3183 following keyword arguments are supported:
3184 config: An optional dict of git config options to be passed with '-c'.
Doug Anderson37282b42011-03-04 11:54:18 -08003185
3186 Args:
3187 name: The name of the git command to call. Any '_' characters will
3188 be replaced with '-'.
3189
3190 Returns:
3191 A callable object that will try to call git with the named command.
3192 """
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003193 name = name.replace('_', '-')
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07003194
Dave Borowitz091f8932012-10-23 17:01:04 -07003195 def runner(*args, **kwargs):
3196 cmdv = []
3197 config = kwargs.pop('config', None)
3198 for k in kwargs:
3199 raise TypeError('%s() got an unexpected keyword argument %r'
3200 % (name, k))
3201 if config is not None:
Chirayu Desai217ea7d2013-03-01 19:14:38 +05303202 for k, v in config.items():
Dave Borowitz091f8932012-10-23 17:01:04 -07003203 cmdv.append('-c')
3204 cmdv.append('%s=%s' % (k, v))
3205 cmdv.append(name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003206 cmdv.extend(args)
3207 p = GitCommand(self._project,
3208 cmdv,
Anthony King7bdac712014-07-16 12:56:40 +01003209 bare=self._bare,
David James8d201162013-10-11 17:03:19 -07003210 gitdir=self._gitdir,
Anthony King7bdac712014-07-16 12:56:40 +01003211 capture_stdout=True,
3212 capture_stderr=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003213 if p.Wait() != 0:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07003214 raise GitError('%s %s: %s' %
3215 (self._project.name, name, p.stderr))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003216 r = p.stdout
3217 if r.endswith('\n') and r.index('\n') == len(r) - 1:
3218 return r[:-1]
3219 return r
3220 return runner
3221
3222
Shawn O. Pearce350cde42009-04-16 11:21:18 -07003223class _PriorSyncFailedError(Exception):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07003224
Shawn O. Pearce350cde42009-04-16 11:21:18 -07003225 def __str__(self):
3226 return 'prior sync failed; rebase still in progress'
3227
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07003228
Shawn O. Pearce350cde42009-04-16 11:21:18 -07003229class _DirtyError(Exception):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07003230
Shawn O. Pearce350cde42009-04-16 11:21:18 -07003231 def __str__(self):
3232 return 'contains uncommitted changes'
3233
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07003234
Shawn O. Pearce350cde42009-04-16 11:21:18 -07003235class _InfoMessage(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07003236
Shawn O. Pearce350cde42009-04-16 11:21:18 -07003237 def __init__(self, project, text):
3238 self.project = project
3239 self.text = text
3240
3241 def Print(self, syncbuf):
LaMont Jones8501d462022-06-22 19:21:15 +00003242 syncbuf.out.info('%s/: %s', self.project.RelPath(local=False), self.text)
Shawn O. Pearce350cde42009-04-16 11:21:18 -07003243 syncbuf.out.nl()
3244
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07003245
Shawn O. Pearce350cde42009-04-16 11:21:18 -07003246class _Failure(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07003247
Shawn O. Pearce350cde42009-04-16 11:21:18 -07003248 def __init__(self, project, why):
3249 self.project = project
3250 self.why = why
3251
3252 def Print(self, syncbuf):
3253 syncbuf.out.fail('error: %s/: %s',
LaMont Jones8501d462022-06-22 19:21:15 +00003254 self.project.RelPath(local=False),
Shawn O. Pearce350cde42009-04-16 11:21:18 -07003255 str(self.why))
3256 syncbuf.out.nl()
3257
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07003258
Shawn O. Pearce350cde42009-04-16 11:21:18 -07003259class _Later(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07003260
Shawn O. Pearce350cde42009-04-16 11:21:18 -07003261 def __init__(self, project, action):
3262 self.project = project
3263 self.action = action
3264
3265 def Run(self, syncbuf):
3266 out = syncbuf.out
LaMont Jones8501d462022-06-22 19:21:15 +00003267 out.project('project %s/', self.project.RelPath(local=False))
Shawn O. Pearce350cde42009-04-16 11:21:18 -07003268 out.nl()
3269 try:
3270 self.action()
3271 out.nl()
3272 return True
David Pursehouse8a68ff92012-09-24 12:15:13 +09003273 except GitError:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07003274 out.nl()
3275 return False
3276
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07003277
Shawn O. Pearce350cde42009-04-16 11:21:18 -07003278class _SyncColoring(Coloring):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07003279
Shawn O. Pearce350cde42009-04-16 11:21:18 -07003280 def __init__(self, config):
Mike Frysinger5d9c4972021-02-19 13:34:09 -05003281 super().__init__(config, 'reposync')
Anthony King7bdac712014-07-16 12:56:40 +01003282 self.project = self.printer('header', attr='bold')
3283 self.info = self.printer('info')
3284 self.fail = self.printer('fail', fg='red')
Shawn O. Pearce350cde42009-04-16 11:21:18 -07003285
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07003286
Shawn O. Pearce350cde42009-04-16 11:21:18 -07003287class SyncBuffer(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07003288
Shawn O. Pearce350cde42009-04-16 11:21:18 -07003289 def __init__(self, config, detach_head=False):
3290 self._messages = []
3291 self._failures = []
3292 self._later_queue1 = []
3293 self._later_queue2 = []
3294
3295 self.out = _SyncColoring(config)
3296 self.out.redirect(sys.stderr)
3297
3298 self.detach_head = detach_head
3299 self.clean = True
David Rileye0684ad2017-04-05 00:02:59 -07003300 self.recent_clean = True
Shawn O. Pearce350cde42009-04-16 11:21:18 -07003301
3302 def info(self, project, fmt, *args):
3303 self._messages.append(_InfoMessage(project, fmt % args))
3304
3305 def fail(self, project, err=None):
3306 self._failures.append(_Failure(project, err))
David Rileye0684ad2017-04-05 00:02:59 -07003307 self._MarkUnclean()
Shawn O. Pearce350cde42009-04-16 11:21:18 -07003308
3309 def later1(self, project, what):
3310 self._later_queue1.append(_Later(project, what))
3311
3312 def later2(self, project, what):
3313 self._later_queue2.append(_Later(project, what))
3314
3315 def Finish(self):
3316 self._PrintMessages()
3317 self._RunLater()
3318 self._PrintMessages()
3319 return self.clean
3320
David Rileye0684ad2017-04-05 00:02:59 -07003321 def Recently(self):
3322 recent_clean = self.recent_clean
3323 self.recent_clean = True
3324 return recent_clean
3325
3326 def _MarkUnclean(self):
3327 self.clean = False
3328 self.recent_clean = False
3329
Shawn O. Pearce350cde42009-04-16 11:21:18 -07003330 def _RunLater(self):
3331 for q in ['_later_queue1', '_later_queue2']:
3332 if not self._RunQueue(q):
3333 return
3334
3335 def _RunQueue(self, queue):
3336 for m in getattr(self, queue):
3337 if not m.Run(self):
David Rileye0684ad2017-04-05 00:02:59 -07003338 self._MarkUnclean()
Shawn O. Pearce350cde42009-04-16 11:21:18 -07003339 return False
3340 setattr(self, queue, [])
3341 return True
3342
3343 def _PrintMessages(self):
Mike Frysinger70d861f2019-08-26 15:22:36 -04003344 if self._messages or self._failures:
3345 if os.isatty(2):
3346 self.out.write(progress.CSI_ERASE_LINE)
3347 self.out.write('\r')
3348
Shawn O. Pearce350cde42009-04-16 11:21:18 -07003349 for m in self._messages:
3350 m.Print(self)
3351 for m in self._failures:
3352 m.Print(self)
3353
3354 self._messages = []
3355 self._failures = []
3356
3357
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003358class MetaProject(Project):
LaMont Jones9b72cf22022-03-29 21:54:22 +00003359 """A special project housed under .repo."""
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07003360
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003361 def __init__(self, manifest, name, gitdir, worktree):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003362 Project.__init__(self,
Anthony King7bdac712014-07-16 12:56:40 +01003363 manifest=manifest,
3364 name=name,
3365 gitdir=gitdir,
3366 objdir=gitdir,
3367 worktree=worktree,
3368 remote=RemoteSpec('origin'),
3369 relpath='.repo/%s' % name,
3370 revisionExpr='refs/heads/master',
3371 revisionId=None,
3372 groups=None)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003373
3374 def PreSync(self):
3375 if self.Exists:
3376 cb = self.CurrentBranch
3377 if cb:
3378 base = self.GetBranch(cb).merge
3379 if base:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07003380 self.revisionExpr = base
3381 self.revisionId = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003382
Shawn O. Pearcef6906872009-04-18 10:49:00 -07003383 @property
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003384 def HasChanges(self):
LaMont Jones9b72cf22022-03-29 21:54:22 +00003385 """Has the remote received new commits not yet checked out?"""
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07003386 if not self.remote or not self.revisionExpr:
Shawn O. Pearce336f7bd2009-04-18 10:39:28 -07003387 return False
3388
David Pursehouse8a68ff92012-09-24 12:15:13 +09003389 all_refs = self.bare_ref.all
3390 revid = self.GetRevisionId(all_refs)
Shawn O. Pearce336f7bd2009-04-18 10:39:28 -07003391 head = self.work_git.GetHead()
3392 if head.startswith(R_HEADS):
3393 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09003394 head = all_refs[head]
Shawn O. Pearce336f7bd2009-04-18 10:39:28 -07003395 except KeyError:
3396 head = None
3397
3398 if revid == head:
3399 return False
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07003400 elif self._revlist(not_rev(HEAD), revid):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003401 return True
3402 return False
LaMont Jones9b72cf22022-03-29 21:54:22 +00003403
3404
3405class RepoProject(MetaProject):
3406 """The MetaProject for repo itself."""
3407
3408 @property
3409 def LastFetch(self):
3410 try:
3411 fh = os.path.join(self.gitdir, 'FETCH_HEAD')
3412 return os.path.getmtime(fh)
3413 except OSError:
3414 return 0
3415
3416class ManifestProject(MetaProject):
3417 """The MetaProject for manifests."""
3418
3419 def MetaBranchSwitch(self, submodules=False):
3420 """Prepare for manifest branch switch."""
3421
3422 # detach and delete manifest branch, allowing a new
3423 # branch to take over
3424 syncbuf = SyncBuffer(self.config, detach_head=True)
3425 self.Sync_LocalHalf(syncbuf, submodules=submodules)
3426 syncbuf.Finish()
3427
3428 return GitCommand(self,
3429 ['update-ref', '-d', 'refs/heads/default'],
3430 capture_stdout=True,
3431 capture_stderr=True).Wait() == 0
LaMont Jones9b03f152022-03-29 23:01:18 +00003432
3433 @property
LaMont Jonesd82be3e2022-04-05 19:30:46 +00003434 def standalone_manifest_url(self):
3435 """The URL of the standalone manifest, or None."""
LaMont Jones55ee3042022-04-06 17:10:21 +00003436 return self.config.GetString('manifest.standalone')
LaMont Jonesd82be3e2022-04-05 19:30:46 +00003437
3438 @property
3439 def manifest_groups(self):
3440 """The manifest groups string."""
3441 return self.config.GetString('manifest.groups')
3442
3443 @property
3444 def reference(self):
3445 """The --reference for this manifest."""
LaMont Jones4ada0432022-04-14 15:10:43 +00003446 return self.config.GetString('repo.reference')
LaMont Jonesd82be3e2022-04-05 19:30:46 +00003447
3448 @property
3449 def dissociate(self):
3450 """Whether to dissociate."""
LaMont Jones4ada0432022-04-14 15:10:43 +00003451 return self.config.GetBoolean('repo.dissociate')
LaMont Jonesd82be3e2022-04-05 19:30:46 +00003452
3453 @property
3454 def archive(self):
3455 """Whether we use archive."""
LaMont Jones4ada0432022-04-14 15:10:43 +00003456 return self.config.GetBoolean('repo.archive')
LaMont Jonesd82be3e2022-04-05 19:30:46 +00003457
3458 @property
3459 def mirror(self):
3460 """Whether we use mirror."""
LaMont Jones4ada0432022-04-14 15:10:43 +00003461 return self.config.GetBoolean('repo.mirror')
LaMont Jonesd82be3e2022-04-05 19:30:46 +00003462
3463 @property
3464 def use_worktree(self):
3465 """Whether we use worktree."""
LaMont Jones4ada0432022-04-14 15:10:43 +00003466 return self.config.GetBoolean('repo.worktree')
LaMont Jonesd82be3e2022-04-05 19:30:46 +00003467
3468 @property
3469 def clone_bundle(self):
3470 """Whether we use clone_bundle."""
LaMont Jones4ada0432022-04-14 15:10:43 +00003471 return self.config.GetBoolean('repo.clonebundle')
LaMont Jonesd82be3e2022-04-05 19:30:46 +00003472
3473 @property
3474 def submodules(self):
3475 """Whether we use submodules."""
LaMont Jones4ada0432022-04-14 15:10:43 +00003476 return self.config.GetBoolean('repo.submodules')
LaMont Jonesd82be3e2022-04-05 19:30:46 +00003477
3478 @property
3479 def git_lfs(self):
3480 """Whether we use git_lfs."""
LaMont Jones4ada0432022-04-14 15:10:43 +00003481 return self.config.GetBoolean('repo.git-lfs')
LaMont Jonesd82be3e2022-04-05 19:30:46 +00003482
3483 @property
3484 def use_superproject(self):
3485 """Whether we use superproject."""
LaMont Jones4ada0432022-04-14 15:10:43 +00003486 return self.config.GetBoolean('repo.superproject')
LaMont Jonesd82be3e2022-04-05 19:30:46 +00003487
3488 @property
3489 def partial_clone(self):
3490 """Whether this is a partial clone."""
LaMont Jones4ada0432022-04-14 15:10:43 +00003491 return self.config.GetBoolean('repo.partialclone')
LaMont Jonesd82be3e2022-04-05 19:30:46 +00003492
3493 @property
3494 def depth(self):
3495 """Partial clone depth."""
LaMont Jones4ada0432022-04-14 15:10:43 +00003496 return self.config.GetString('repo.depth')
LaMont Jonesd82be3e2022-04-05 19:30:46 +00003497
3498 @property
3499 def clone_filter(self):
3500 """The clone filter."""
LaMont Jones4ada0432022-04-14 15:10:43 +00003501 return self.config.GetString('repo.clonefilter')
LaMont Jonesd82be3e2022-04-05 19:30:46 +00003502
3503 @property
3504 def partial_clone_exclude(self):
3505 """Partial clone exclude string"""
LaMont Jones4ada0432022-04-14 15:10:43 +00003506 return self.config.GetBoolean('repo.partialcloneexclude')
3507
3508 @property
3509 def manifest_platform(self):
3510 """The --platform argument from `repo init`."""
3511 return self.config.GetString('manifest.platform')
LaMont Jonesd82be3e2022-04-05 19:30:46 +00003512
3513 @property
LaMont Jones9b03f152022-03-29 23:01:18 +00003514 def _platform_name(self):
3515 """Return the name of the platform."""
3516 return platform.system().lower()
3517
LaMont Jonesbdcba7d2022-04-11 22:50:11 +00003518 def SyncWithPossibleInit(self, submanifest, verbose=False,
3519 current_branch_only=False, tags='', git_event_log=None):
3520 """Sync a manifestProject, possibly for the first time.
3521
3522 Call Sync() with arguments from the most recent `repo init`. If this is a
3523 new sub manifest, then inherit options from the parent's manifestProject.
3524
3525 This is used by subcmds.Sync() to do an initial download of new sub
3526 manifests.
3527
3528 Args:
3529 submanifest: an XmlSubmanifest, the submanifest to re-sync.
3530 verbose: a boolean, whether to show all output, rather than only errors.
3531 current_branch_only: a boolean, whether to only fetch the current manifest
3532 branch from the server.
3533 tags: a boolean, whether to fetch tags.
3534 git_event_log: an EventLog, for git tracing.
3535 """
3536 # TODO(lamontjones): when refactoring sync (and init?) consider how to
LaMont Jonesff6b1da2022-06-01 21:03:34 +00003537 # better get the init options that we should use for new submanifests that
3538 # are added when syncing an existing workspace.
LaMont Jonesbdcba7d2022-04-11 22:50:11 +00003539 git_event_log = git_event_log or EventLog()
3540 spec = submanifest.ToSubmanifestSpec()
3541 # Use the init options from the existing manifestProject, or the parent if
3542 # it doesn't exist.
3543 #
3544 # Today, we only support changing manifest_groups on the sub-manifest, with
3545 # no supported-for-the-user way to change the other arguments from those
3546 # specified by the outermost manifest.
3547 #
3548 # TODO(lamontjones): determine which of these should come from the outermost
3549 # manifest and which should come from the parent manifest.
3550 mp = self if self.Exists else submanifest.parent.manifestProject
3551 return self.Sync(
3552 manifest_url=spec.manifestUrl,
3553 manifest_branch=spec.revision,
3554 standalone_manifest=mp.standalone_manifest_url,
3555 groups=mp.manifest_groups,
3556 platform=mp.manifest_platform,
3557 mirror=mp.mirror,
3558 dissociate=mp.dissociate,
3559 reference=mp.reference,
3560 worktree=mp.use_worktree,
3561 submodules=mp.submodules,
3562 archive=mp.archive,
3563 partial_clone=mp.partial_clone,
3564 clone_filter=mp.clone_filter,
3565 partial_clone_exclude=mp.partial_clone_exclude,
3566 clone_bundle=mp.clone_bundle,
3567 git_lfs=mp.git_lfs,
3568 use_superproject=mp.use_superproject,
3569 verbose=verbose,
3570 current_branch_only=current_branch_only,
3571 tags=tags,
3572 depth=mp.depth,
3573 git_event_log=git_event_log,
3574 manifest_name=spec.manifestName,
3575 this_manifest_only=True,
3576 outer_manifest=False,
3577 )
3578
LaMont Jones9b03f152022-03-29 23:01:18 +00003579 def Sync(self, _kwargs_only=(), manifest_url='', manifest_branch=None,
LaMont Jonesd82be3e2022-04-05 19:30:46 +00003580 standalone_manifest=False, groups='', mirror=False, reference='',
3581 dissociate=False, worktree=False, submodules=False, archive=False,
3582 partial_clone=None, depth=None, clone_filter='blob:none',
LaMont Jones9b03f152022-03-29 23:01:18 +00003583 partial_clone_exclude=None, clone_bundle=None, git_lfs=None,
3584 use_superproject=None, verbose=False, current_branch_only=False,
LaMont Jones55ee3042022-04-06 17:10:21 +00003585 git_event_log=None, platform='', manifest_name='default.xml',
3586 tags='', this_manifest_only=False, outer_manifest=True):
LaMont Jones9b03f152022-03-29 23:01:18 +00003587 """Sync the manifest and all submanifests.
3588
3589 Args:
3590 manifest_url: a string, the URL of the manifest project.
3591 manifest_branch: a string, the manifest branch to use.
3592 standalone_manifest: a boolean, whether to store the manifest as a static
3593 file.
3594 groups: a string, restricts the checkout to projects with the specified
3595 groups.
LaMont Jones9b03f152022-03-29 23:01:18 +00003596 mirror: a boolean, whether to create a mirror of the remote repository.
3597 reference: a string, location of a repo instance to use as a reference.
3598 dissociate: a boolean, whether to dissociate from reference mirrors after
3599 clone.
3600 worktree: a boolean, whether to use git-worktree to manage projects.
3601 submodules: a boolean, whether sync submodules associated with the
3602 manifest project.
3603 archive: a boolean, whether to checkout each project as an archive. See
3604 git-archive.
3605 partial_clone: a boolean, whether to perform a partial clone.
LaMont Jonesd82be3e2022-04-05 19:30:46 +00003606 depth: an int, how deep of a shallow clone to create.
LaMont Jones9b03f152022-03-29 23:01:18 +00003607 clone_filter: a string, filter to use with partial_clone.
3608 partial_clone_exclude : a string, comma-delimeted list of project namess
3609 to exclude from partial clone.
3610 clone_bundle: a boolean, whether to enable /clone.bundle on HTTP/HTTPS.
3611 git_lfs: a boolean, whether to enable git LFS support.
3612 use_superproject: a boolean, whether to use the manifest superproject to
3613 sync projects.
3614 verbose: a boolean, whether to show all output, rather than only errors.
3615 current_branch_only: a boolean, whether to only fetch the current manifest
3616 branch from the server.
LaMont Jonesd82be3e2022-04-05 19:30:46 +00003617 platform: a string, restrict the checkout to projects with the specified
3618 platform group.
LaMont Jones55ee3042022-04-06 17:10:21 +00003619 git_event_log: an EventLog, for git tracing.
LaMont Jones4ada0432022-04-14 15:10:43 +00003620 tags: a boolean, whether to fetch tags.
LaMont Jones409407a2022-04-05 21:21:56 +00003621 manifest_name: a string, the name of the manifest file to use.
3622 this_manifest_only: a boolean, whether to only operate on the current sub
3623 manifest.
3624 outer_manifest: a boolean, whether to start at the outermost manifest.
LaMont Jones9b03f152022-03-29 23:01:18 +00003625
3626 Returns:
3627 a boolean, whether the sync was successful.
3628 """
3629 assert _kwargs_only == (), 'Sync only accepts keyword arguments.'
3630
LaMont Jones501733c2022-04-20 16:42:32 +00003631 groups = groups or self.manifest.GetDefaultGroupsStr(with_platform=False)
LaMont Jonesa2ff20d2022-04-07 16:49:06 +00003632 platform = platform or 'auto'
LaMont Jones55ee3042022-04-06 17:10:21 +00003633 git_event_log = git_event_log or EventLog()
LaMont Jones409407a2022-04-05 21:21:56 +00003634 if outer_manifest and self.manifest.is_submanifest:
3635 # In a multi-manifest checkout, use the outer manifest unless we are told
3636 # not to.
3637 return self.client.outer_manifest.manifestProject.Sync(
3638 manifest_url=manifest_url,
3639 manifest_branch=manifest_branch,
3640 standalone_manifest=standalone_manifest,
3641 groups=groups,
3642 platform=platform,
3643 mirror=mirror,
3644 dissociate=dissociate,
3645 reference=reference,
3646 worktree=worktree,
3647 submodules=submodules,
3648 archive=archive,
3649 partial_clone=partial_clone,
3650 clone_filter=clone_filter,
3651 partial_clone_exclude=partial_clone_exclude,
3652 clone_bundle=clone_bundle,
3653 git_lfs=git_lfs,
3654 use_superproject=use_superproject,
3655 verbose=verbose,
3656 current_branch_only=current_branch_only,
3657 tags=tags,
3658 depth=depth,
LaMont Jones55ee3042022-04-06 17:10:21 +00003659 git_event_log=git_event_log,
LaMont Jones409407a2022-04-05 21:21:56 +00003660 manifest_name=manifest_name,
3661 this_manifest_only=this_manifest_only,
3662 outer_manifest=False)
3663
LaMont Jones9b03f152022-03-29 23:01:18 +00003664 # If repo has already been initialized, we take -u with the absence of
3665 # --standalone-manifest to mean "transition to a standard repo set up",
3666 # which necessitates starting fresh.
3667 # If --standalone-manifest is set, we always tear everything down and start
3668 # anew.
3669 if self.Exists:
3670 was_standalone_manifest = self.config.GetString('manifest.standalone')
3671 if was_standalone_manifest and not manifest_url:
3672 print('fatal: repo was initialized with a standlone manifest, '
3673 'cannot be re-initialized without --manifest-url/-u')
3674 return False
3675
3676 if standalone_manifest or (was_standalone_manifest and manifest_url):
3677 self.config.ClearCache()
3678 if self.gitdir and os.path.exists(self.gitdir):
3679 platform_utils.rmtree(self.gitdir)
3680 if self.worktree and os.path.exists(self.worktree):
3681 platform_utils.rmtree(self.worktree)
3682
3683 is_new = not self.Exists
3684 if is_new:
3685 if not manifest_url:
3686 print('fatal: manifest url is required.', file=sys.stderr)
3687 return False
3688
LaMont Jones409407a2022-04-05 21:21:56 +00003689 if verbose:
LaMont Jones9b03f152022-03-29 23:01:18 +00003690 print('Downloading manifest from %s' %
3691 (GitConfig.ForUser().UrlInsteadOf(manifest_url),),
3692 file=sys.stderr)
3693
3694 # The manifest project object doesn't keep track of the path on the
3695 # server where this git is located, so let's save that here.
3696 mirrored_manifest_git = None
3697 if reference:
3698 manifest_git_path = urllib.parse.urlparse(manifest_url).path[1:]
3699 mirrored_manifest_git = os.path.join(reference, manifest_git_path)
3700 if not mirrored_manifest_git.endswith(".git"):
3701 mirrored_manifest_git += ".git"
3702 if not os.path.exists(mirrored_manifest_git):
3703 mirrored_manifest_git = os.path.join(reference,
3704 '.repo/manifests.git')
3705
3706 self._InitGitDir(mirror_git=mirrored_manifest_git)
3707
3708 # If standalone_manifest is set, mark the project as "standalone" -- we'll
3709 # still do much of the manifests.git set up, but will avoid actual syncs to
3710 # a remote.
3711 if standalone_manifest:
3712 self.config.SetString('manifest.standalone', manifest_url)
3713 elif not manifest_url and not manifest_branch:
3714 # If -u is set and --standalone-manifest is not, then we're not in
3715 # standalone mode. Otherwise, use config to infer what we were in the last
3716 # init.
3717 standalone_manifest = bool(self.config.GetString('manifest.standalone'))
3718 if not standalone_manifest:
3719 self.config.SetString('manifest.standalone', None)
3720
3721 self._ConfigureDepth(depth)
3722
3723 # Set the remote URL before the remote branch as we might need it below.
3724 if manifest_url:
Mike Frysingerdede5642022-07-10 04:56:04 -04003725 r = self.GetRemote()
LaMont Jones9b03f152022-03-29 23:01:18 +00003726 r.url = manifest_url
3727 r.ResetFetch()
3728 r.Save()
3729
3730 if not standalone_manifest:
3731 if manifest_branch:
3732 if manifest_branch == 'HEAD':
3733 manifest_branch = self.ResolveRemoteHead()
3734 if manifest_branch is None:
3735 print('fatal: unable to resolve HEAD', file=sys.stderr)
3736 return False
3737 self.revisionExpr = manifest_branch
3738 else:
3739 if is_new:
3740 default_branch = self.ResolveRemoteHead()
3741 if default_branch is None:
3742 # If the remote doesn't have HEAD configured, default to master.
3743 default_branch = 'refs/heads/master'
3744 self.revisionExpr = default_branch
3745 else:
3746 self.PreSync()
3747
LaMont Jonesd82be3e2022-04-05 19:30:46 +00003748 groups = re.split(r'[,\s]+', groups or '')
LaMont Jones9b03f152022-03-29 23:01:18 +00003749 all_platforms = ['linux', 'darwin', 'windows']
3750 platformize = lambda x: 'platform-' + x
3751 if platform == 'auto':
LaMont Jonesd82be3e2022-04-05 19:30:46 +00003752 if not mirror and not self.mirror:
LaMont Jones9b03f152022-03-29 23:01:18 +00003753 groups.append(platformize(self._platform_name))
3754 elif platform == 'all':
3755 groups.extend(map(platformize, all_platforms))
3756 elif platform in all_platforms:
3757 groups.append(platformize(platform))
3758 elif platform != 'none':
3759 print('fatal: invalid platform flag', file=sys.stderr)
3760 return False
LaMont Jones4ada0432022-04-14 15:10:43 +00003761 self.config.SetString('manifest.platform', platform)
LaMont Jones9b03f152022-03-29 23:01:18 +00003762
3763 groups = [x for x in groups if x]
3764 groupstr = ','.join(groups)
3765 if platform == 'auto' and groupstr == self.manifest.GetDefaultGroupsStr():
3766 groupstr = None
3767 self.config.SetString('manifest.groups', groupstr)
3768
3769 if reference:
3770 self.config.SetString('repo.reference', reference)
3771
3772 if dissociate:
3773 self.config.SetBoolean('repo.dissociate', dissociate)
3774
3775 if worktree:
3776 if mirror:
3777 print('fatal: --mirror and --worktree are incompatible',
3778 file=sys.stderr)
3779 return False
3780 if submodules:
3781 print('fatal: --submodules and --worktree are incompatible',
3782 file=sys.stderr)
3783 return False
3784 self.config.SetBoolean('repo.worktree', worktree)
3785 if is_new:
3786 self.use_git_worktrees = True
3787 print('warning: --worktree is experimental!', file=sys.stderr)
3788
3789 if archive:
3790 if is_new:
3791 self.config.SetBoolean('repo.archive', archive)
3792 else:
3793 print('fatal: --archive is only supported when initializing a new '
3794 'workspace.', file=sys.stderr)
3795 print('Either delete the .repo folder in this workspace, or initialize '
3796 'in another location.', file=sys.stderr)
3797 return False
3798
3799 if mirror:
3800 if is_new:
3801 self.config.SetBoolean('repo.mirror', mirror)
3802 else:
3803 print('fatal: --mirror is only supported when initializing a new '
3804 'workspace.', file=sys.stderr)
3805 print('Either delete the .repo folder in this workspace, or initialize '
3806 'in another location.', file=sys.stderr)
3807 return False
3808
3809 if partial_clone is not None:
3810 if mirror:
3811 print('fatal: --mirror and --partial-clone are mutually exclusive',
3812 file=sys.stderr)
3813 return False
3814 self.config.SetBoolean('repo.partialclone', partial_clone)
3815 if clone_filter:
3816 self.config.SetString('repo.clonefilter', clone_filter)
LaMont Jones55ee3042022-04-06 17:10:21 +00003817 elif self.partial_clone:
LaMont Jonesd82be3e2022-04-05 19:30:46 +00003818 clone_filter = self.clone_filter
LaMont Jones9b03f152022-03-29 23:01:18 +00003819 else:
3820 clone_filter = None
3821
3822 if partial_clone_exclude is not None:
3823 self.config.SetString('repo.partialcloneexclude', partial_clone_exclude)
3824
3825 if clone_bundle is None:
3826 clone_bundle = False if partial_clone else True
3827 else:
3828 self.config.SetBoolean('repo.clonebundle', clone_bundle)
3829
3830 if submodules:
3831 self.config.SetBoolean('repo.submodules', submodules)
3832
3833 if git_lfs is not None:
3834 if git_lfs:
3835 git_require((2, 17, 0), fail=True, msg='Git LFS support')
3836
3837 self.config.SetBoolean('repo.git-lfs', git_lfs)
3838 if not is_new:
3839 print('warning: Changing --git-lfs settings will only affect new project checkouts.\n'
3840 ' Existing projects will require manual updates.\n', file=sys.stderr)
3841
3842 if use_superproject is not None:
3843 self.config.SetBoolean('repo.superproject', use_superproject)
3844
LaMont Jones0165e202022-04-27 17:34:42 +00003845 if not standalone_manifest:
3846 if not self.Sync_NetworkHalf(
3847 is_new=is_new, quiet=not verbose, verbose=verbose,
3848 clone_bundle=clone_bundle, current_branch_only=current_branch_only,
3849 tags=tags, submodules=submodules, clone_filter=clone_filter,
LaMont Jones1eddca82022-09-01 15:15:04 +00003850 partial_clone_exclude=self.manifest.PartialCloneExclude).success:
Mike Frysingerdede5642022-07-10 04:56:04 -04003851 r = self.GetRemote()
LaMont Jones0165e202022-04-27 17:34:42 +00003852 print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr)
LaMont Jonesaf8fb132022-08-31 20:21:25 +00003853
3854 # Better delete the manifest git dir if we created it; otherwise next
3855 # time (when user fixes problems) we won't go through the "is_new" logic.
3856 if is_new:
3857 platform_utils.rmtree(self.gitdir)
LaMont Jones9b03f152022-03-29 23:01:18 +00003858 return False
3859
LaMont Jones0165e202022-04-27 17:34:42 +00003860 if manifest_branch:
3861 self.MetaBranchSwitch(submodules=submodules)
3862
3863 syncbuf = SyncBuffer(self.config)
3864 self.Sync_LocalHalf(syncbuf, submodules=submodules)
3865 syncbuf.Finish()
3866
3867 if is_new or self.CurrentBranch is None:
3868 if not self.StartBranch('default'):
3869 print('fatal: cannot create default in manifest', file=sys.stderr)
3870 return False
3871
3872 if not manifest_name:
3873 print('fatal: manifest name (-m) is required.', file=sys.stderr)
3874 return False
3875
3876 elif is_new:
3877 # This is a new standalone manifest.
3878 manifest_name = 'default.xml'
3879 manifest_data = fetch.fetch_file(manifest_url, verbose=verbose)
3880 dest = os.path.join(self.worktree, manifest_name)
3881 os.makedirs(os.path.dirname(dest), exist_ok=True)
3882 with open(dest, 'wb') as f:
3883 f.write(manifest_data)
LaMont Jones409407a2022-04-05 21:21:56 +00003884
3885 try:
3886 self.manifest.Link(manifest_name)
3887 except ManifestParseError as e:
3888 print("fatal: manifest '%s' not available" % manifest_name,
3889 file=sys.stderr)
3890 print('fatal: %s' % str(e), file=sys.stderr)
3891 return False
3892
LaMont Jones55ee3042022-04-06 17:10:21 +00003893 if not this_manifest_only:
3894 for submanifest in self.manifest.submanifests.values():
LaMont Jonesb90a4222022-04-14 15:00:09 +00003895 spec = submanifest.ToSubmanifestSpec()
LaMont Jones55ee3042022-04-06 17:10:21 +00003896 submanifest.repo_client.manifestProject.Sync(
3897 manifest_url=spec.manifestUrl,
3898 manifest_branch=spec.revision,
3899 standalone_manifest=standalone_manifest,
3900 groups=self.manifest_groups,
3901 platform=platform,
3902 mirror=mirror,
3903 dissociate=dissociate,
3904 reference=reference,
3905 worktree=worktree,
3906 submodules=submodules,
3907 archive=archive,
3908 partial_clone=partial_clone,
3909 clone_filter=clone_filter,
3910 partial_clone_exclude=partial_clone_exclude,
3911 clone_bundle=clone_bundle,
3912 git_lfs=git_lfs,
3913 use_superproject=use_superproject,
3914 verbose=verbose,
3915 current_branch_only=current_branch_only,
3916 tags=tags,
3917 depth=depth,
3918 git_event_log=git_event_log,
3919 manifest_name=spec.manifestName,
3920 this_manifest_only=False,
3921 outer_manifest=False,
3922 )
LaMont Jones409407a2022-04-05 21:21:56 +00003923
LaMont Jones0ddb6772022-05-20 09:11:54 +00003924 # Lastly, if the manifest has a <superproject> then have the superproject
LaMont Jonesff6b1da2022-06-01 21:03:34 +00003925 # sync it (if it will be used).
3926 if git_superproject.UseSuperproject(use_superproject, self.manifest):
LaMont Jones0ddb6772022-05-20 09:11:54 +00003927 sync_result = self.manifest.superproject.Sync(git_event_log)
LaMont Jonesa2ff20d2022-04-07 16:49:06 +00003928 if not sync_result.success:
3929 print('warning: git update of superproject for '
3930 f'{self.manifest.path_prefix} failed, repo sync will not use '
3931 'superproject to fetch source; while this error is not fatal, '
3932 'and you can continue to run repo sync, please run repo init '
3933 'with the --no-use-superproject option to stop seeing this '
3934 'warning', file=sys.stderr)
3935 if sync_result.fatal and use_superproject is not None:
3936 return False
LaMont Jones409407a2022-04-05 21:21:56 +00003937
LaMont Jones9b03f152022-03-29 23:01:18 +00003938 return True
3939
3940 def _ConfigureDepth(self, depth):
3941 """Configure the depth we'll sync down.
3942
3943 Args:
3944 depth: an int, how deep of a partial clone to create.
3945 """
3946 # Opt.depth will be non-None if user actually passed --depth to repo init.
3947 if depth is not None:
3948 if depth > 0:
3949 # Positive values will set the depth.
3950 depth = str(depth)
3951 else:
3952 # Negative numbers will clear the depth; passing None to SetString
3953 # will do that.
3954 depth = None
3955
3956 # We store the depth in the main manifest project.
3957 self.config.SetString('repo.depth', depth)