Allow the bot_update api to not fetch tags from the git server.
There are 3 layers modified, starting from the bottom up:
1. git_cache.py populate
Now takes a --no-fetch-tags option. If specified, the cache will not
fetch updated tags from the server by passing --no-tags to git fetch.
This prevents the server from sending all the tag refs. In chromium
this prevents significant time bottlenecks dealing with 10k+ tags.
2. bot_update.py
bot_update has to deal with multiple git repos, it has the root repo
that is checked out through git-cache, as well as repos synched via
DEPS and gclient.
The script now takes a --no_fetch_tags option. If specified,
the git-cache populate call will include --no-fetch-tags. Otherwise, it
won't. So this controls (for chromium) if fetches to the src.git server
are passed --no-tags.
3. bot_update/api.py
This is the entry point for recipes and also has to deal with multiple
git repos. The behaviour at this point is not changed from the default.
A |no_fetch_tags| parameter is added to ensure_checkout() that defaults
to False.
This CL is a refactor with no intended behavior change.
The next step will be to change the chromium build repo to pass True
for ensure_checkout(no_fetch_tags) on chromium trybots.
This represents solution #2 in
https://docs.google.com/document/d/1hDIunJjjfpmr50y3YnZ4o3aq1NZF4gJa1TS0p7AIL64/edit#
Bug: 1019824
Change-Id: I935430603299daa9e301a95a5184c0ce486fd912
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1894352
Reviewed-by: Aaron Gable <agable@chromium.org>
Reviewed-by: Erik Chen <erikchen@chromium.org>
Commit-Queue: danakj <danakj@chromium.org>
Auto-Submit: danakj <danakj@chromium.org>
diff --git a/git_cache.py b/git_cache.py
index 772dbea..4a1ec03 100755
--- a/git_cache.py
+++ b/git_cache.py
@@ -536,15 +536,18 @@
'but failed. Continuing with non-optimized repository.'
% len(pack_files))
- def _fetch(self, rundir, verbose, depth, reset_fetch_config):
+ def _fetch(self, rundir, verbose, depth, no_fetch_tags, reset_fetch_config):
self.config(rundir, reset_fetch_config)
v = []
d = []
+ t = []
if verbose:
v = ['-v', '--progress']
if depth:
d = ['--depth', str(depth)]
- fetch_cmd = ['fetch'] + v + d + ['origin']
+ if no_fetch_tags:
+ t = ['--no-tags']
+ fetch_cmd = ['fetch'] + v + d + t + ['origin']
fetch_specs = subprocess.check_output(
[self.git_exe, 'config', '--get-all', 'remote.origin.fetch'],
cwd=rundir).strip().splitlines()
@@ -585,8 +588,14 @@
raise ClobberNeeded() # Corrupted cache.
logging.warn('Fetch of %s failed' % spec)
- def populate(self, depth=None, shallow=False, bootstrap=False,
- verbose=False, ignore_lock=False, lock_timeout=0,
+ def populate(self,
+ depth=None,
+ no_fetch_tags=False,
+ shallow=False,
+ bootstrap=False,
+ verbose=False,
+ ignore_lock=False,
+ lock_timeout=0,
reset_fetch_config=False):
assert self.GetCachePath()
if shallow and not depth:
@@ -599,13 +608,15 @@
try:
self._ensure_bootstrapped(depth, bootstrap)
- self._fetch(self.mirror_path, verbose, depth, reset_fetch_config)
+ self._fetch(self.mirror_path, verbose, depth, no_fetch_tags,
+ reset_fetch_config)
except ClobberNeeded:
# This is a major failure, we need to clean and force a bootstrap.
gclient_utils.rmtree(self.mirror_path)
self.print(GIT_CACHE_CORRUPT_MESSAGE)
self._ensure_bootstrapped(depth, bootstrap, force=True)
- self._fetch(self.mirror_path, verbose, depth, reset_fetch_config)
+ self._fetch(self.mirror_path, verbose, depth, no_fetch_tags,
+ reset_fetch_config)
finally:
if not ignore_lock:
lockfile.unlock()
@@ -769,6 +780,11 @@
"""Ensure that the cache has all up-to-date objects for the given repo."""
parser.add_option('--depth', type='int',
help='Only cache DEPTH commits of history')
+ parser.add_option(
+ '--no-fetch-tags',
+ action='store_true',
+ help=('Don\'t fetch tags from the server. This can speed up '
+ 'fetch considerably when there are many tags.'))
parser.add_option('--shallow', '-s', action='store_true',
help='Only cache 10000 commits of history')
parser.add_option('--ref', action='append',
@@ -794,6 +810,7 @@
if options.break_locks:
mirror.unlock()
kwargs = {
+ 'no_fetch_tags': options.no_fetch_tags,
'verbose': options.verbose,
'shallow': options.shallow,
'bootstrap': not options.no_bootstrap,
@@ -813,6 +830,11 @@
parser.add_option('--no_bootstrap', '--no-bootstrap',
action='store_true',
help='Don\'t (re)bootstrap from Google Storage')
+ parser.add_option(
+ '--no-fetch-tags',
+ action='store_true',
+ help=('Don\'t fetch tags from the server. This can speed up '
+ 'fetch considerably when there are many tags.'))
options, args = parser.parse_args(args)
# Figure out which remotes to fetch. This mimics the behavior of regular
@@ -844,7 +866,9 @@
if git_dir.startswith(cachepath):
mirror = Mirror.FromPath(git_dir)
mirror.populate(
- bootstrap=not options.no_bootstrap, lock_timeout=options.timeout)
+ bootstrap=not options.no_bootstrap,
+ no_fetch_tags=options.no_fetch_tags,
+ lock_timeout=options.timeout)
return 0
for remote in remotes:
remote_url = subprocess.check_output(
@@ -854,7 +878,9 @@
mirror.print = lambda *args: None
print('Updating git cache...')
mirror.populate(
- bootstrap=not options.no_bootstrap, lock_timeout=options.timeout)
+ bootstrap=not options.no_bootstrap,
+ no_fetch_tags=options.no_fetch_tags,
+ lock_timeout=options.timeout)
subprocess.check_call([Mirror.git_exe, 'fetch', remote])
return 0