switch to 4 space indent
Leave the recipes/ code at 2 space to match the rest of the recipes
project in other repos.
Reformatted using:
files=( $(
git ls-tree -r --name-only HEAD | \
grep -Ev -e '^(third_party|recipes)/' | \
grep '\.py$';
git grep -l '#!/usr/bin/env.*python' | grep -v '\.py$'
) )
parallel ./yapf -i -- "${files[@]}"
~/chromiumos/chromite/contrib/reflow_overlong_comments "${files[@]}"
The files that still had strings that were too long were manually
reformatted because they were easy and only a few issues.
autoninja.py
clang_format.py
download_from_google_storage.py
fix_encoding.py
gclient_utils.py
git_cache.py
git_common.py
git_map_branches.py
git_reparent_branch.py
gn.py
my_activity.py
owners_finder.py
presubmit_canned_checks.py
reclient_helper.py
reclientreport.py
roll_dep.py
rustfmt.py
siso.py
split_cl.py
subcommand.py
subprocess2.py
swift_format.py
upload_to_google_storage.py
These files still had lines (strings) that were too long, so the pylint
warnings were suppressed with a TODO.
auth.py
gclient.py
gclient_eval.py
gclient_paths.py
gclient_scm.py
gerrit_util.py
git_cl.py
presubmit_canned_checks.py
presubmit_support.py
scm.py
Change-Id: Ia6535c4f2c48d46b589ec1e791dde6c6b2ea858f
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/4836379
Commit-Queue: Josip Sokcevic <sokcevic@chromium.org>
Auto-Submit: Mike Frysinger <vapier@chromium.org>
Reviewed-by: Josip Sokcevic <sokcevic@chromium.org>
diff --git a/git_cache.py b/git_cache.py
index 816f415..a15b3f9 100755
--- a/git_cache.py
+++ b/git_cache.py
@@ -2,7 +2,6 @@
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
-
"""A git command for managing a local cache of git repositories."""
import contextlib
@@ -35,19 +34,25 @@
GSUTIL_CP_SEMAPHORE = threading.Semaphore(2)
try:
- # pylint: disable=undefined-variable
- WinErr = WindowsError
+ # pylint: disable=undefined-variable
+ WinErr = WindowsError
except NameError:
- class WinErr(Exception):
- pass
+
+ class WinErr(Exception):
+ pass
+
class ClobberNeeded(Exception):
- pass
+ pass
-def exponential_backoff_retry(fn, excs=(Exception,), name=None, count=10,
- sleep_time=0.25, printerr=None):
- """Executes |fn| up to |count| times, backing off exponentially.
+def exponential_backoff_retry(fn,
+ excs=(Exception, ),
+ name=None,
+ count=10,
+ sleep_time=0.25,
+ printerr=None):
+ """Executes |fn| up to |count| times, backing off exponentially.
Args:
fn (callable): The function to execute. If this raises a handled
@@ -66,818 +71,862 @@
Returns: The return value of the successful fn.
"""
- printerr = printerr or logging.warning
- for i in range(count):
- try:
- return fn()
- except excs as e:
- if (i+1) >= count:
- raise
+ printerr = printerr or logging.warning
+ for i in range(count):
+ try:
+ return fn()
+ except excs as e:
+ if (i + 1) >= count:
+ raise
- printerr('Retrying %s in %.2f second(s) (%d / %d attempts): %s' % (
- (name or 'operation'), sleep_time, (i+1), count, e))
- time.sleep(sleep_time)
- sleep_time *= 2
+ printerr('Retrying %s in %.2f second(s) (%d / %d attempts): %s' %
+ ((name or 'operation'), sleep_time, (i + 1), count, e))
+ time.sleep(sleep_time)
+ sleep_time *= 2
class Mirror(object):
- git_exe = 'git.bat' if sys.platform.startswith('win') else 'git'
- gsutil_exe = os.path.join(
- os.path.dirname(os.path.abspath(__file__)), 'gsutil.py')
- cachepath_lock = threading.Lock()
+ git_exe = 'git.bat' if sys.platform.startswith('win') else 'git'
+ gsutil_exe = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+ 'gsutil.py')
+ cachepath_lock = threading.Lock()
- UNSET_CACHEPATH = object()
+ UNSET_CACHEPATH = object()
- # Used for tests
- _GIT_CONFIG_LOCATION = []
+ # Used for tests
+ _GIT_CONFIG_LOCATION = []
- @staticmethod
- def parse_fetch_spec(spec):
- """Parses and canonicalizes a fetch spec.
+ @staticmethod
+ def parse_fetch_spec(spec):
+ """Parses and canonicalizes a fetch spec.
Returns (fetchspec, value_regex), where value_regex can be used
with 'git config --replace-all'.
"""
- parts = spec.split(':', 1)
- src = parts[0].lstrip('+').rstrip('/')
- if not src.startswith('refs/'):
- src = 'refs/heads/%s' % src
- dest = parts[1].rstrip('/') if len(parts) > 1 else src
- regex = r'\+%s:.*' % src.replace('*', r'\*')
- return ('+%s:%s' % (src, dest), regex)
+ parts = spec.split(':', 1)
+ src = parts[0].lstrip('+').rstrip('/')
+ if not src.startswith('refs/'):
+ src = 'refs/heads/%s' % src
+ dest = parts[1].rstrip('/') if len(parts) > 1 else src
+ regex = r'\+%s:.*' % src.replace('*', r'\*')
+ return ('+%s:%s' % (src, dest), regex)
- def __init__(self, url, refs=None, commits=None, print_func=None):
- self.url = url
- self.fetch_specs = {self.parse_fetch_spec(ref) for ref in (refs or [])}
- self.fetch_commits = set(commits or [])
- self.basedir = self.UrlToCacheDir(url)
- self.mirror_path = os.path.join(self.GetCachePath(), self.basedir)
- if print_func:
- self.print = self.print_without_file
- self.print_func = print_func
- else:
- self.print = print
+ def __init__(self, url, refs=None, commits=None, print_func=None):
+ self.url = url
+ self.fetch_specs = {self.parse_fetch_spec(ref) for ref in (refs or [])}
+ self.fetch_commits = set(commits or [])
+ self.basedir = self.UrlToCacheDir(url)
+ self.mirror_path = os.path.join(self.GetCachePath(), self.basedir)
+ if print_func:
+ self.print = self.print_without_file
+ self.print_func = print_func
+ else:
+ self.print = print
- def print_without_file(self, message, **_kwargs):
- self.print_func(message)
+ def print_without_file(self, message, **_kwargs):
+ self.print_func(message)
- @contextlib.contextmanager
- def print_duration_of(self, what):
- start = time.time()
- try:
- yield
- finally:
- self.print('%s took %.1f minutes' % (what, (time.time() - start) / 60.0))
-
- @property
- def bootstrap_bucket(self):
- b = os.getenv('OVERRIDE_BOOTSTRAP_BUCKET')
- if b:
- return b
- u = urllib.parse.urlparse(self.url)
- if u.netloc == 'chromium.googlesource.com':
- return 'chromium-git-cache'
- # Not recognized.
- return None
-
- @property
- def _gs_path(self):
- return 'gs://%s/v2/%s' % (self.bootstrap_bucket, self.basedir)
-
- @classmethod
- def FromPath(cls, path):
- return cls(cls.CacheDirToUrl(path))
-
- @staticmethod
- def UrlToCacheDir(url):
- """Convert a git url to a normalized form for the cache dir path."""
- if os.path.isdir(url):
- # Ignore the drive letter in Windows
- url = os.path.splitdrive(url)[1]
- return url.replace('-', '--').replace(os.sep, '-')
-
- parsed = urllib.parse.urlparse(url)
- norm_url = parsed.netloc + parsed.path
- if norm_url.endswith('.git'):
- norm_url = norm_url[:-len('.git')]
-
- # Use the same dir for authenticated URLs and unauthenticated URLs.
- norm_url = norm_url.replace('googlesource.com/a/', 'googlesource.com/')
-
- return norm_url.replace('-', '--').replace('/', '-').lower()
-
- @staticmethod
- def CacheDirToUrl(path):
- """Convert a cache dir path to its corresponding url."""
- netpath = re.sub(r'\b-\b', '/', os.path.basename(path)).replace('--', '-')
- return 'https://%s' % netpath
-
- @classmethod
- def SetCachePath(cls, cachepath):
- with cls.cachepath_lock:
- setattr(cls, 'cachepath', cachepath)
-
- @classmethod
- def GetCachePath(cls):
- with cls.cachepath_lock:
- if not hasattr(cls, 'cachepath'):
+ @contextlib.contextmanager
+ def print_duration_of(self, what):
+ start = time.time()
try:
- cachepath = subprocess.check_output(
- [cls.git_exe, 'config'] +
- cls._GIT_CONFIG_LOCATION +
- ['cache.cachepath']).decode('utf-8', 'ignore').strip()
+ yield
+ finally:
+ self.print('%s took %.1f minutes' % (what,
+ (time.time() - start) / 60.0))
+
+ @property
+ def bootstrap_bucket(self):
+ b = os.getenv('OVERRIDE_BOOTSTRAP_BUCKET')
+ if b:
+ return b
+ u = urllib.parse.urlparse(self.url)
+ if u.netloc == 'chromium.googlesource.com':
+ return 'chromium-git-cache'
+ # Not recognized.
+ return None
+
+ @property
+ def _gs_path(self):
+ return 'gs://%s/v2/%s' % (self.bootstrap_bucket, self.basedir)
+
+ @classmethod
+ def FromPath(cls, path):
+ return cls(cls.CacheDirToUrl(path))
+
+ @staticmethod
+ def UrlToCacheDir(url):
+ """Convert a git url to a normalized form for the cache dir path."""
+ if os.path.isdir(url):
+ # Ignore the drive letter in Windows
+ url = os.path.splitdrive(url)[1]
+ return url.replace('-', '--').replace(os.sep, '-')
+
+ parsed = urllib.parse.urlparse(url)
+ norm_url = parsed.netloc + parsed.path
+ if norm_url.endswith('.git'):
+ norm_url = norm_url[:-len('.git')]
+
+ # Use the same dir for authenticated URLs and unauthenticated URLs.
+ norm_url = norm_url.replace('googlesource.com/a/', 'googlesource.com/')
+
+ return norm_url.replace('-', '--').replace('/', '-').lower()
+
+ @staticmethod
+ def CacheDirToUrl(path):
+ """Convert a cache dir path to its corresponding url."""
+ netpath = re.sub(r'\b-\b', '/',
+ os.path.basename(path)).replace('--', '-')
+ return 'https://%s' % netpath
+
+ @classmethod
+ def SetCachePath(cls, cachepath):
+ with cls.cachepath_lock:
+ setattr(cls, 'cachepath', cachepath)
+
+ @classmethod
+ def GetCachePath(cls):
+ with cls.cachepath_lock:
+ if not hasattr(cls, 'cachepath'):
+ try:
+ cachepath = subprocess.check_output(
+ [cls.git_exe, 'config'] + cls._GIT_CONFIG_LOCATION +
+ ['cache.cachepath']).decode('utf-8', 'ignore').strip()
+ except subprocess.CalledProcessError:
+ cachepath = os.environ.get('GIT_CACHE_PATH',
+ cls.UNSET_CACHEPATH)
+ setattr(cls, 'cachepath', cachepath)
+
+ ret = getattr(cls, 'cachepath')
+ if ret is cls.UNSET_CACHEPATH:
+ raise RuntimeError('No cache.cachepath git configuration or '
+ '$GIT_CACHE_PATH is set.')
+ return ret
+
+ @staticmethod
+ def _GetMostRecentCacheDirectory(ls_out_set):
+ ready_file_pattern = re.compile(r'.*/(\d+).ready$')
+ ready_dirs = []
+
+ for name in ls_out_set:
+ m = ready_file_pattern.match(name)
+ # Given <path>/<number>.ready,
+ # we are interested in <path>/<number> directory
+ if m and (name[:-len('.ready')] + '/') in ls_out_set:
+ ready_dirs.append((int(m.group(1)), name[:-len('.ready')]))
+
+ if not ready_dirs:
+ return None
+
+ return max(ready_dirs)[1]
+
+ def Rename(self, src, dst):
+ # This is somehow racy on Windows.
+ # Catching OSError because WindowsError isn't portable and
+ # pylint complains.
+ exponential_backoff_retry(lambda: os.rename(src, dst),
+ excs=(OSError, ),
+ name='rename [%s] => [%s]' % (src, dst),
+ printerr=self.print)
+
+ def RunGit(self, cmd, print_stdout=True, **kwargs):
+ """Run git in a subprocess."""
+ cwd = kwargs.setdefault('cwd', self.mirror_path)
+ if "--git-dir" not in cmd:
+ cmd = ['--git-dir', os.path.abspath(cwd)] + cmd
+
+ kwargs.setdefault('print_stdout', False)
+ if print_stdout:
+ kwargs.setdefault('filter_fn', self.print)
+ env = kwargs.get('env') or kwargs.setdefault('env', os.environ.copy())
+ env.setdefault('GIT_ASKPASS', 'true')
+ env.setdefault('SSH_ASKPASS', 'true')
+ self.print('running "git %s" in "%s"' % (' '.join(cmd), cwd))
+ gclient_utils.CheckCallAndFilter([self.git_exe] + cmd, **kwargs)
+
+ def config(self, reset_fetch_config=False):
+ if reset_fetch_config:
+ try:
+ self.RunGit(['config', '--unset-all', 'remote.origin.fetch'])
+ except subprocess.CalledProcessError as e:
+ # If exit code was 5, it means we attempted to unset a config
+ # that didn't exist. Ignore it.
+ if e.returncode != 5:
+ raise
+
+ # Don't run git-gc in a daemon. Bad things can happen if it gets
+ # killed.
+ try:
+ self.RunGit(['config', 'gc.autodetach', '0'])
except subprocess.CalledProcessError:
- cachepath = os.environ.get('GIT_CACHE_PATH', cls.UNSET_CACHEPATH)
- setattr(cls, 'cachepath', cachepath)
+ # Hard error, need to clobber.
+ raise ClobberNeeded()
- ret = getattr(cls, 'cachepath')
- if ret is cls.UNSET_CACHEPATH:
- raise RuntimeError('No cache.cachepath git configuration or '
- '$GIT_CACHE_PATH is set.')
- return ret
+ # Don't combine pack files into one big pack file. It's really slow for
+ # repositories, and there's no way to track progress and make sure it's
+ # not stuck.
+ if self.supported_project():
+ self.RunGit(['config', 'gc.autopacklimit', '0'])
- @staticmethod
- def _GetMostRecentCacheDirectory(ls_out_set):
- ready_file_pattern = re.compile(r'.*/(\d+).ready$')
- ready_dirs = []
+ # Allocate more RAM for cache-ing delta chains, for better performance
+ # of "Resolving deltas".
+ self.RunGit([
+ 'config', 'core.deltaBaseCacheLimit',
+ gclient_utils.DefaultDeltaBaseCacheLimit()
+ ])
- for name in ls_out_set:
- m = ready_file_pattern.match(name)
- # Given <path>/<number>.ready,
- # we are interested in <path>/<number> directory
- if m and (name[:-len('.ready')] + '/') in ls_out_set:
- ready_dirs.append((int(m.group(1)), name[:-len('.ready')]))
+ self.RunGit(['config', 'remote.origin.url', self.url])
+ self.RunGit([
+ 'config', '--replace-all', 'remote.origin.fetch',
+ '+refs/heads/*:refs/heads/*', r'\+refs/heads/\*:.*'
+ ])
+ for spec, value_regex in self.fetch_specs:
+ self.RunGit([
+ 'config', '--replace-all', 'remote.origin.fetch', spec,
+ value_regex
+ ])
- if not ready_dirs:
- return None
-
- return max(ready_dirs)[1]
-
- def Rename(self, src, dst):
- # This is somehow racy on Windows.
- # Catching OSError because WindowsError isn't portable and
- # pylint complains.
- exponential_backoff_retry(
- lambda: os.rename(src, dst),
- excs=(OSError,),
- name='rename [%s] => [%s]' % (src, dst),
- printerr=self.print)
-
- def RunGit(self, cmd, print_stdout=True, **kwargs):
- """Run git in a subprocess."""
- cwd = kwargs.setdefault('cwd', self.mirror_path)
- if "--git-dir" not in cmd:
- cmd = ['--git-dir', os.path.abspath(cwd)] + cmd
-
- kwargs.setdefault('print_stdout', False)
- if print_stdout:
- kwargs.setdefault('filter_fn', self.print)
- env = kwargs.get('env') or kwargs.setdefault('env', os.environ.copy())
- env.setdefault('GIT_ASKPASS', 'true')
- env.setdefault('SSH_ASKPASS', 'true')
- self.print('running "git %s" in "%s"' % (' '.join(cmd), cwd))
- gclient_utils.CheckCallAndFilter([self.git_exe] + cmd, **kwargs)
-
- def config(self, reset_fetch_config=False):
- if reset_fetch_config:
- try:
- self.RunGit(['config', '--unset-all', 'remote.origin.fetch'])
- except subprocess.CalledProcessError as e:
- # If exit code was 5, it means we attempted to unset a config that
- # didn't exist. Ignore it.
- if e.returncode != 5:
- raise
-
- # Don't run git-gc in a daemon. Bad things can happen if it gets killed.
- try:
- self.RunGit(['config', 'gc.autodetach', '0'])
- except subprocess.CalledProcessError:
- # Hard error, need to clobber.
- raise ClobberNeeded()
-
- # Don't combine pack files into one big pack file. It's really slow for
- # repositories, and there's no way to track progress and make sure it's
- # not stuck.
- if self.supported_project():
- self.RunGit(['config', 'gc.autopacklimit', '0'])
-
- # Allocate more RAM for cache-ing delta chains, for better performance
- # of "Resolving deltas".
- self.RunGit([
- 'config', 'core.deltaBaseCacheLimit',
- gclient_utils.DefaultDeltaBaseCacheLimit()
- ])
-
- self.RunGit(['config', 'remote.origin.url', self.url])
- self.RunGit([
- 'config', '--replace-all', 'remote.origin.fetch',
- '+refs/heads/*:refs/heads/*', r'\+refs/heads/\*:.*'
- ])
- for spec, value_regex in self.fetch_specs:
- self.RunGit(
- ['config', '--replace-all', 'remote.origin.fetch', spec, value_regex])
-
- def bootstrap_repo(self, directory):
- """Bootstrap the repo from Google Storage if possible.
+ def bootstrap_repo(self, directory):
+ """Bootstrap the repo from Google Storage if possible.
More apt-ly named bootstrap_repo_from_cloud_if_possible_else_do_nothing().
"""
- if not self.bootstrap_bucket:
- return False
+ if not self.bootstrap_bucket:
+ return False
- gsutil = Gsutil(self.gsutil_exe, boto_path=None)
+ gsutil = Gsutil(self.gsutil_exe, boto_path=None)
- # Get the most recent version of the directory.
- # This is determined from the most recent version of a .ready file.
- # The .ready file is only uploaded when an entire directory has been
- # uploaded to GS.
- _, ls_out, ls_err = gsutil.check_call('ls', self._gs_path)
- ls_out_set = set(ls_out.strip().splitlines())
- latest_dir = self._GetMostRecentCacheDirectory(ls_out_set)
+ # Get the most recent version of the directory.
+ # This is determined from the most recent version of a .ready file.
+ # The .ready file is only uploaded when an entire directory has been
+ # uploaded to GS.
+ _, ls_out, ls_err = gsutil.check_call('ls', self._gs_path)
+ ls_out_set = set(ls_out.strip().splitlines())
+ latest_dir = self._GetMostRecentCacheDirectory(ls_out_set)
- if not latest_dir:
- self.print('No bootstrap file for %s found in %s, stderr:\n %s' %
- (self.mirror_path, self.bootstrap_bucket,
- ' '.join((ls_err or '').splitlines(True))))
- return False
+ if not latest_dir:
+ self.print('No bootstrap file for %s found in %s, stderr:\n %s' %
+ (self.mirror_path, self.bootstrap_bucket, ' '.join(
+ (ls_err or '').splitlines(True))))
+ return False
- try:
- # create new temporary directory locally
- tempdir = tempfile.mkdtemp(prefix='_cache_tmp', dir=self.GetCachePath())
- self.RunGit(['init', '-b', 'main', '--bare'], cwd=tempdir)
- self.print('Downloading files in %s/* into %s.' %
- (latest_dir, tempdir))
- with self.print_duration_of('download'):
- with GSUTIL_CP_SEMAPHORE:
- code = gsutil.call('-m', 'cp', '-r', latest_dir + "/*",
- tempdir)
- if code:
- return False
- # A quick validation that all references are valid.
- self.RunGit(['for-each-ref'], print_stdout=False, cwd=tempdir)
- except Exception as e:
- self.print('Encountered error: %s' % str(e), file=sys.stderr)
- gclient_utils.rmtree(tempdir)
- return False
- # delete the old directory
- if os.path.exists(directory):
- gclient_utils.rmtree(directory)
- self.Rename(tempdir, directory)
- return True
+ try:
+ # create new temporary directory locally
+ tempdir = tempfile.mkdtemp(prefix='_cache_tmp',
+ dir=self.GetCachePath())
+ self.RunGit(['init', '-b', 'main', '--bare'], cwd=tempdir)
+ self.print('Downloading files in %s/* into %s.' %
+ (latest_dir, tempdir))
+ with self.print_duration_of('download'):
+ with GSUTIL_CP_SEMAPHORE:
+ code = gsutil.call('-m', 'cp', '-r', latest_dir + "/*",
+ tempdir)
+ if code:
+ return False
+ # A quick validation that all references are valid.
+ self.RunGit(['for-each-ref'], print_stdout=False, cwd=tempdir)
+ except Exception as e:
+ self.print('Encountered error: %s' % str(e), file=sys.stderr)
+ gclient_utils.rmtree(tempdir)
+ return False
+ # delete the old directory
+ if os.path.exists(directory):
+ gclient_utils.rmtree(directory)
+ self.Rename(tempdir, directory)
+ return True
- def contains_revision(self, revision):
- if not self.exists():
- return False
+ def contains_revision(self, revision):
+ if not self.exists():
+ return False
- if sys.platform.startswith('win'):
- # Windows .bat scripts use ^ as escape sequence, which means we have to
- # escape it with itself for every .bat invocation.
- needle = '%s^^^^{commit}' % revision
- else:
- needle = '%s^{commit}' % revision
- try:
- # cat-file exits with 0 on success, that is git object of given hash was
- # found.
- self.RunGit(['cat-file', '-e', needle])
- return True
- except subprocess.CalledProcessError:
- self.print('Commit with hash "%s" not found' % revision, file=sys.stderr)
- return False
+ if sys.platform.startswith('win'):
+ # Windows .bat scripts use ^ as escape sequence, which means we have
+ # to escape it with itself for every .bat invocation.
+ needle = '%s^^^^{commit}' % revision
+ else:
+ needle = '%s^{commit}' % revision
+ try:
+ # cat-file exits with 0 on success, that is git object of given hash
+ # was found.
+ self.RunGit(['cat-file', '-e', needle])
+ return True
+ except subprocess.CalledProcessError:
+ self.print('Commit with hash "%s" not found' % revision,
+ file=sys.stderr)
+ return False
- def exists(self):
- return os.path.isfile(os.path.join(self.mirror_path, 'config'))
+ def exists(self):
+ return os.path.isfile(os.path.join(self.mirror_path, 'config'))
- def supported_project(self):
- """Returns true if this repo is known to have a bootstrap zip file."""
- u = urllib.parse.urlparse(self.url)
- return u.netloc in [
- 'chromium.googlesource.com',
- 'chrome-internal.googlesource.com']
+ def supported_project(self):
+ """Returns true if this repo is known to have a bootstrap zip file."""
+ u = urllib.parse.urlparse(self.url)
+ return u.netloc in [
+ 'chromium.googlesource.com', 'chrome-internal.googlesource.com'
+ ]
- def _preserve_fetchspec(self):
- """Read and preserve remote.origin.fetch from an existing mirror.
+ def _preserve_fetchspec(self):
+ """Read and preserve remote.origin.fetch from an existing mirror.
This modifies self.fetch_specs.
"""
- if not self.exists():
- return
- try:
- config_fetchspecs = subprocess.check_output(
- [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'],
- cwd=self.mirror_path).decode('utf-8', 'ignore')
- for fetchspec in config_fetchspecs.splitlines():
- self.fetch_specs.add(self.parse_fetch_spec(fetchspec))
- except subprocess.CalledProcessError:
- logging.warning(
- 'Tried and failed to preserve remote.origin.fetch from the '
- 'existing cache directory. You may need to manually edit '
- '%s and "git cache fetch" again.' %
- os.path.join(self.mirror_path, 'config'))
+ if not self.exists():
+ return
+ try:
+ config_fetchspecs = subprocess.check_output(
+ [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'],
+ cwd=self.mirror_path).decode('utf-8', 'ignore')
+ for fetchspec in config_fetchspecs.splitlines():
+ self.fetch_specs.add(self.parse_fetch_spec(fetchspec))
+ except subprocess.CalledProcessError:
+ logging.warning(
+ 'Tried and failed to preserve remote.origin.fetch from the '
+ 'existing cache directory. You may need to manually edit '
+ '%s and "git cache fetch" again.' %
+ os.path.join(self.mirror_path, 'config'))
- def _ensure_bootstrapped(
- self, depth, bootstrap, reset_fetch_config, force=False):
- pack_dir = os.path.join(self.mirror_path, 'objects', 'pack')
- pack_files = []
- if os.path.isdir(pack_dir):
- pack_files = [f for f in os.listdir(pack_dir) if f.endswith('.pack')]
- self.print('%s has %d .pack files, re-bootstrapping if >%d or ==0' %
- (self.mirror_path, len(pack_files), GC_AUTOPACKLIMIT))
+ def _ensure_bootstrapped(self,
+ depth,
+ bootstrap,
+ reset_fetch_config,
+ force=False):
+ pack_dir = os.path.join(self.mirror_path, 'objects', 'pack')
+ pack_files = []
+ if os.path.isdir(pack_dir):
+ pack_files = [
+ f for f in os.listdir(pack_dir) if f.endswith('.pack')
+ ]
+ self.print('%s has %d .pack files, re-bootstrapping if >%d or ==0' %
+ (self.mirror_path, len(pack_files), GC_AUTOPACKLIMIT))
- # master->main branch migration left the cache in some builders to have its
- # HEAD still pointing to refs/heads/master. This causes bot_update to fail.
- # If in this state, delete the cache and force bootstrap.
- try:
- with open(os.path.join(self.mirror_path, 'HEAD')) as f:
- head_ref = f.read()
- except FileNotFoundError:
- head_ref = ''
+ # master->main branch migration left the cache in some builders to have
+ # its HEAD still pointing to refs/heads/master. This causes bot_update
+ # to fail. If in this state, delete the cache and force bootstrap.
+ try:
+ with open(os.path.join(self.mirror_path, 'HEAD')) as f:
+ head_ref = f.read()
+ except FileNotFoundError:
+ head_ref = ''
- # Check only when HEAD points to master.
- if 'master' in head_ref:
- # Some repos could still have master so verify if the ref exists first.
- show_ref_master_cmd = subprocess.run(
- [Mirror.git_exe, 'show-ref', '--verify', 'refs/heads/master'],
- cwd=self.mirror_path)
+ # Check only when HEAD points to master.
+ if 'master' in head_ref:
+ # Some repos could still have master so verify if the ref exists
+ # first.
+ show_ref_master_cmd = subprocess.run(
+ [Mirror.git_exe, 'show-ref', '--verify', 'refs/heads/master'],
+ cwd=self.mirror_path)
- if show_ref_master_cmd.returncode != 0:
- # Remove mirror
- gclient_utils.rmtree(self.mirror_path)
+ if show_ref_master_cmd.returncode != 0:
+ # Remove mirror
+ gclient_utils.rmtree(self.mirror_path)
- # force bootstrap
- force = True
+ # force bootstrap
+ force = True
- should_bootstrap = (force or
- not self.exists() or
- len(pack_files) > GC_AUTOPACKLIMIT or
- len(pack_files) == 0)
+ should_bootstrap = (force or not self.exists()
+ or len(pack_files) > GC_AUTOPACKLIMIT
+ or len(pack_files) == 0)
- if not should_bootstrap:
- if depth and os.path.exists(os.path.join(self.mirror_path, 'shallow')):
- logging.warning(
- 'Shallow fetch requested, but repo cache already exists.')
- return
+ if not should_bootstrap:
+ if depth and os.path.exists(
+ os.path.join(self.mirror_path, 'shallow')):
+ logging.warning(
+ 'Shallow fetch requested, but repo cache already exists.')
+ return
- if not self.exists():
- if os.path.exists(self.mirror_path):
- # If the mirror path exists but self.exists() returns false, we're
- # in an unexpected state. Nuke the previous mirror directory and
- # start fresh.
- gclient_utils.rmtree(self.mirror_path)
- os.mkdir(self.mirror_path)
- elif not reset_fetch_config:
- # Re-bootstrapping an existing mirror; preserve existing fetch spec.
- self._preserve_fetchspec()
+ if not self.exists():
+ if os.path.exists(self.mirror_path):
+ # If the mirror path exists but self.exists() returns false,
+ # we're in an unexpected state. Nuke the previous mirror
+ # directory and start fresh.
+ gclient_utils.rmtree(self.mirror_path)
+ os.mkdir(self.mirror_path)
+ elif not reset_fetch_config:
+ # Re-bootstrapping an existing mirror; preserve existing fetch spec.
+ self._preserve_fetchspec()
- bootstrapped = (not depth and bootstrap and
- self.bootstrap_repo(self.mirror_path))
+ bootstrapped = (not depth and bootstrap
+ and self.bootstrap_repo(self.mirror_path))
- if not bootstrapped:
- if not self.exists() or not self.supported_project():
- # Bootstrap failed due to:
- # 1. No previous cache.
- # 2. Project doesn't have a bootstrap folder.
- # Start with a bare git dir.
- self.RunGit(['init', '--bare'])
- # Set appropriate symbolic-ref
- remote_info = exponential_backoff_retry(lambda: subprocess.check_output(
+ if not bootstrapped:
+ if not self.exists() or not self.supported_project():
+ # Bootstrap failed due to:
+ # 1. No previous cache.
+ # 2. Project doesn't have a bootstrap folder.
+ # Start with a bare git dir.
+ self.RunGit(['init', '--bare'])
+ # Set appropriate symbolic-ref
+ remote_info = exponential_backoff_retry(
+ lambda: subprocess.check_output(
+ [
+ self.git_exe, '--git-dir',
+ os.path.abspath(self.mirror_path), 'remote', 'show',
+ self.url
+ ],
+ cwd=self.mirror_path).decode('utf-8', 'ignore').strip())
+ default_branch_regexp = re.compile(r'HEAD branch: (.*)$')
+ m = default_branch_regexp.search(remote_info, re.MULTILINE)
+ if m:
+ self.RunGit(
+ ['symbolic-ref', 'HEAD', 'refs/heads/' + m.groups()[0]])
+ else:
+ # Bootstrap failed, previous cache exists; warn and continue.
+ logging.warning(
+ 'Git cache has a lot of pack files (%d). Tried to '
+ 're-bootstrap but failed. Continuing with non-optimized '
+ 'repository.' % len(pack_files))
+
+ def _fetch(self,
+ verbose,
+ depth,
+ no_fetch_tags,
+ reset_fetch_config,
+ prune=True):
+ self.config(reset_fetch_config)
+
+ fetch_cmd = ['fetch']
+ if verbose:
+ fetch_cmd.extend(['-v', '--progress'])
+ if depth:
+ fetch_cmd.extend(['--depth', str(depth)])
+ if no_fetch_tags:
+ fetch_cmd.append('--no-tags')
+ if prune:
+ fetch_cmd.append('--prune')
+ fetch_cmd.append('origin')
+
+ fetch_specs = subprocess.check_output(
[
self.git_exe, '--git-dir',
- os.path.abspath(self.mirror_path), 'remote', 'show', self.url
+ os.path.abspath(self.mirror_path), 'config', '--get-all',
+ 'remote.origin.fetch'
],
- cwd=self.mirror_path).decode('utf-8', 'ignore').strip())
- default_branch_regexp = re.compile(r'HEAD branch: (.*)$')
- m = default_branch_regexp.search(remote_info, re.MULTILINE)
- if m:
- self.RunGit(['symbolic-ref', 'HEAD', 'refs/heads/' + m.groups()[0]])
- else:
- # Bootstrap failed, previous cache exists; warn and continue.
- logging.warning(
- 'Git cache has a lot of pack files (%d). Tried to re-bootstrap '
- 'but failed. Continuing with non-optimized repository.' %
- len(pack_files))
+ cwd=self.mirror_path).decode('utf-8',
+ 'ignore').strip().splitlines()
+ for spec in fetch_specs:
+ try:
+ self.print('Fetching %s' % spec)
+ with self.print_duration_of('fetch %s' % spec):
+ self.RunGit(fetch_cmd + [spec], retry=True)
+ except subprocess.CalledProcessError:
+ if spec == '+refs/heads/*:refs/heads/*':
+ raise ClobberNeeded() # Corrupted cache.
+ logging.warning('Fetch of %s failed' % spec)
+ for commit in self.fetch_commits:
+ self.print('Fetching %s' % commit)
+ try:
+ with self.print_duration_of('fetch %s' % commit):
+ self.RunGit(['fetch', 'origin', commit], retry=True)
+ except subprocess.CalledProcessError:
+ logging.warning('Fetch of %s failed' % commit)
- def _fetch(self,
- verbose,
- depth,
- no_fetch_tags,
- reset_fetch_config,
- prune=True):
- self.config(reset_fetch_config)
+ def populate(self,
+ depth=None,
+ no_fetch_tags=False,
+ shallow=False,
+ bootstrap=False,
+ verbose=False,
+ lock_timeout=0,
+ reset_fetch_config=False):
+ assert self.GetCachePath()
+ if shallow and not depth:
+ depth = 10000
+ gclient_utils.safe_makedirs(self.GetCachePath())
- fetch_cmd = ['fetch']
- if verbose:
- fetch_cmd.extend(['-v', '--progress'])
- if depth:
- fetch_cmd.extend(['--depth', str(depth)])
- if no_fetch_tags:
- fetch_cmd.append('--no-tags')
- if prune:
- fetch_cmd.append('--prune')
- fetch_cmd.append('origin')
+ with lockfile.lock(self.mirror_path, lock_timeout):
+ try:
+ self._ensure_bootstrapped(depth, bootstrap, reset_fetch_config)
+ self._fetch(verbose, depth, no_fetch_tags, reset_fetch_config)
+ except ClobberNeeded:
+ # This is a major failure, we need to clean and force a
+ # bootstrap.
+ gclient_utils.rmtree(self.mirror_path)
+ self.print(GIT_CACHE_CORRUPT_MESSAGE)
+ self._ensure_bootstrapped(depth,
+ bootstrap,
+ reset_fetch_config,
+ force=True)
+ self._fetch(verbose, depth, no_fetch_tags, reset_fetch_config)
- fetch_specs = subprocess.check_output([
- self.git_exe, '--git-dir',
- os.path.abspath(self.mirror_path), 'config', '--get-all',
- 'remote.origin.fetch'
- ],
- cwd=self.mirror_path).decode(
- 'utf-8',
- 'ignore').strip().splitlines()
- for spec in fetch_specs:
- try:
- self.print('Fetching %s' % spec)
- with self.print_duration_of('fetch %s' % spec):
- self.RunGit(fetch_cmd + [spec], retry=True)
- except subprocess.CalledProcessError:
- if spec == '+refs/heads/*:refs/heads/*':
- raise ClobberNeeded() # Corrupted cache.
- logging.warning('Fetch of %s failed' % spec)
- for commit in self.fetch_commits:
- self.print('Fetching %s' % commit)
- try:
- with self.print_duration_of('fetch %s' % commit):
- self.RunGit(['fetch', 'origin', commit], retry=True)
- except subprocess.CalledProcessError:
- logging.warning('Fetch of %s failed' % commit)
+ def update_bootstrap(self, prune=False, gc_aggressive=False):
+ # NOTE: There have been cases where repos were being recursively
+ # uploaded to google storage. E.g.
+ # `<host_url>-<repo>/<gen_number>/<host_url>-<repo>/` in GS and
+ # <host_url>-<repo>/<host_url>-<repo>/ on the bot. Check for recursed
+ # files on the bot here and remove them if found before we upload to GS.
+ # See crbug.com/1370443; keep this check until root cause is found.
+ recursed_dir = os.path.join(self.mirror_path,
+ self.mirror_path.split(os.path.sep)[-1])
+ if os.path.exists(recursed_dir):
+ self.print('Deleting unexpected directory: %s' % recursed_dir)
+ gclient_utils.rmtree(recursed_dir)
- def populate(self,
- depth=None,
- no_fetch_tags=False,
- shallow=False,
- bootstrap=False,
- verbose=False,
- lock_timeout=0,
- reset_fetch_config=False):
- assert self.GetCachePath()
- if shallow and not depth:
- depth = 10000
- gclient_utils.safe_makedirs(self.GetCachePath())
+ # The folder is <git number>
+ gen_number = subprocess.check_output([self.git_exe, 'number'],
+ cwd=self.mirror_path).decode(
+ 'utf-8', 'ignore').strip()
+ gsutil = Gsutil(path=self.gsutil_exe, boto_path=None)
- with lockfile.lock(self.mirror_path, lock_timeout):
- try:
- self._ensure_bootstrapped(depth, bootstrap, reset_fetch_config)
- self._fetch(verbose, depth, no_fetch_tags, reset_fetch_config)
- except ClobberNeeded:
- # This is a major failure, we need to clean and force a bootstrap.
- gclient_utils.rmtree(self.mirror_path)
- self.print(GIT_CACHE_CORRUPT_MESSAGE)
- self._ensure_bootstrapped(depth,
- bootstrap,
- reset_fetch_config,
- force=True)
- self._fetch(verbose, depth, no_fetch_tags, reset_fetch_config)
+ dest_prefix = '%s/%s' % (self._gs_path, gen_number)
- def update_bootstrap(self, prune=False, gc_aggressive=False):
- # NOTE: There have been cases where repos were being recursively uploaded
- # to google storage.
- # E.g. `<host_url>-<repo>/<gen_number>/<host_url>-<repo>/` in GS and
- # <host_url>-<repo>/<host_url>-<repo>/ on the bot.
- # Check for recursed files on the bot here and remove them if found
- # before we upload to GS.
- # See crbug.com/1370443; keep this check until root cause is found.
- recursed_dir = os.path.join(self.mirror_path,
- self.mirror_path.split(os.path.sep)[-1])
- if os.path.exists(recursed_dir):
- self.print('Deleting unexpected directory: %s' % recursed_dir)
- gclient_utils.rmtree(recursed_dir)
+ # ls_out lists contents in the format: gs://blah/blah/123...
+ self.print('running "gsutil ls %s":' % self._gs_path)
+ ls_code, ls_out, ls_error = gsutil.check_call_with_retries(
+ 'ls', self._gs_path)
+ if ls_code != 0:
+ self.print(ls_error)
+ else:
+ self.print(ls_out)
- # The folder is <git number>
- gen_number = subprocess.check_output([self.git_exe, 'number'],
- cwd=self.mirror_path).decode(
- 'utf-8', 'ignore').strip()
- gsutil = Gsutil(path=self.gsutil_exe, boto_path=None)
+ # Check to see if folder already exists in gs
+ ls_out_set = set(ls_out.strip().splitlines())
+ if (dest_prefix + '/' in ls_out_set
+ and dest_prefix + '.ready' in ls_out_set):
+ print('Cache %s already exists.' % dest_prefix)
+ return
- dest_prefix = '%s/%s' % (self._gs_path, gen_number)
+ # Reduce the number of individual files to download & write on disk.
+ self.RunGit(['pack-refs', '--all'])
- # ls_out lists contents in the format: gs://blah/blah/123...
- self.print('running "gsutil ls %s":' % self._gs_path)
- ls_code, ls_out, ls_error = gsutil.check_call_with_retries(
- 'ls', self._gs_path)
- if ls_code != 0:
- self.print(ls_error)
- else:
- self.print(ls_out)
+ # Run Garbage Collect to compress packfile.
+ gc_args = ['gc', '--prune=all']
+ if gc_aggressive:
+ # The default "gc --aggressive" is often too aggressive for some
+ # machines, since it attempts to create as many threads as there are
+ # CPU cores, while not limiting per-thread memory usage, which puts
+ # too much pressure on RAM on high-core machines, causing them to
+ # thrash. Using lower-level commands gives more control over those
+ # settings.
- # Check to see if folder already exists in gs
- ls_out_set = set(ls_out.strip().splitlines())
- if (dest_prefix + '/' in ls_out_set and
- dest_prefix + '.ready' in ls_out_set):
- print('Cache %s already exists.' % dest_prefix)
- return
+ # This might not be strictly necessary, but it's fast and is
+ # normally run by 'gc --aggressive', so it shouldn't hurt.
+ self.RunGit(['reflog', 'expire', '--all'])
- # Reduce the number of individual files to download & write on disk.
- self.RunGit(['pack-refs', '--all'])
+ # These are the default repack settings for 'gc --aggressive'.
+ gc_args = [
+ 'repack', '-d', '-l', '-f', '--depth=50', '--window=250', '-A',
+ '--unpack-unreachable=all'
+ ]
+ # A 1G memory limit seems to provide comparable pack results as the
+ # default, even for our largest repos, while preventing runaway
+ # memory (at least on current Chromium builders which have about 4G
+ # RAM per core).
+ gc_args.append('--window-memory=1g')
+ # NOTE: It might also be possible to avoid thrashing with a larger
+ # window (e.g. "--window-memory=2g") by limiting the number of
+ # threads created (e.g. "--threads=[cores/2]"). Some limited testing
+ # didn't show much difference in outcomes on our current repos, but
+ # it might be worth trying if the repos grow much larger and the
+ # packs don't seem to be getting compressed enough.
+ self.RunGit(gc_args)
- # Run Garbage Collect to compress packfile.
- gc_args = ['gc', '--prune=all']
- if gc_aggressive:
- # The default "gc --aggressive" is often too aggressive for some machines,
- # since it attempts to create as many threads as there are CPU cores,
- # while not limiting per-thread memory usage, which puts too much pressure
- # on RAM on high-core machines, causing them to thrash. Using lower-level
- # commands gives more control over those settings.
+ self.print('running "gsutil -m rsync -r -d %s %s"' %
+ (self.mirror_path, dest_prefix))
+ gsutil.call('-m', 'rsync', '-r', '-d', self.mirror_path, dest_prefix)
- # This might not be strictly necessary, but it's fast and is normally run
- # by 'gc --aggressive', so it shouldn't hurt.
- self.RunGit(['reflog', 'expire', '--all'])
+ # Create .ready file and upload
+ _, ready_file_name = tempfile.mkstemp(suffix='.ready')
+ try:
+ self.print('running "gsutil cp %s %s.ready"' %
+ (ready_file_name, dest_prefix))
+ gsutil.call('cp', ready_file_name, '%s.ready' % (dest_prefix))
+ finally:
+ os.remove(ready_file_name)
- # These are the default repack settings for 'gc --aggressive'.
- gc_args = ['repack', '-d', '-l', '-f', '--depth=50', '--window=250', '-A',
- '--unpack-unreachable=all']
- # A 1G memory limit seems to provide comparable pack results as the
- # default, even for our largest repos, while preventing runaway memory (at
- # least on current Chromium builders which have about 4G RAM per core).
- gc_args.append('--window-memory=1g')
- # NOTE: It might also be possible to avoid thrashing with a larger window
- # (e.g. "--window-memory=2g") by limiting the number of threads created
- # (e.g. "--threads=[cores/2]"). Some limited testing didn't show much
- # difference in outcomes on our current repos, but it might be worth
- # trying if the repos grow much larger and the packs don't seem to be
- # getting compressed enough.
- self.RunGit(gc_args)
+ # remove all other directory/.ready files in the same gs_path
+ # except for the directory/.ready file previously created
+ # which can be used for bootstrapping while the current one is
+ # being uploaded
+ if not prune:
+ return
+ prev_dest_prefix = self._GetMostRecentCacheDirectory(ls_out_set)
+ if not prev_dest_prefix:
+ return
+ for path in ls_out_set:
+ if path in (prev_dest_prefix + '/', prev_dest_prefix + '.ready'):
+ continue
+ if path.endswith('.ready'):
+ gsutil.call('rm', path)
+ continue
+ gsutil.call('-m', 'rm', '-r', path)
- self.print('running "gsutil -m rsync -r -d %s %s"' %
- (self.mirror_path, dest_prefix))
- gsutil.call('-m', 'rsync', '-r', '-d', self.mirror_path, dest_prefix)
-
- # Create .ready file and upload
- _, ready_file_name = tempfile.mkstemp(suffix='.ready')
- try:
- self.print('running "gsutil cp %s %s.ready"' %
- (ready_file_name, dest_prefix))
- gsutil.call('cp', ready_file_name, '%s.ready' % (dest_prefix))
- finally:
- os.remove(ready_file_name)
-
- # remove all other directory/.ready files in the same gs_path
- # except for the directory/.ready file previously created
- # which can be used for bootstrapping while the current one is
- # being uploaded
- if not prune:
- return
- prev_dest_prefix = self._GetMostRecentCacheDirectory(ls_out_set)
- if not prev_dest_prefix:
- return
- for path in ls_out_set:
- if path in (prev_dest_prefix + '/', prev_dest_prefix + '.ready'):
- continue
- if path.endswith('.ready'):
- gsutil.call('rm', path)
- continue
- gsutil.call('-m', 'rm', '-r', path)
-
-
- @staticmethod
- def DeleteTmpPackFiles(path):
- pack_dir = os.path.join(path, 'objects', 'pack')
- if not os.path.isdir(pack_dir):
- return
- pack_files = [f for f in os.listdir(pack_dir) if
- f.startswith('.tmp-') or f.startswith('tmp_pack_')]
- for f in pack_files:
- f = os.path.join(pack_dir, f)
- try:
- os.remove(f)
- logging.warning('Deleted stale temporary pack file %s' % f)
- except OSError:
- logging.warning('Unable to delete temporary pack file %s' % f)
+ @staticmethod
+ def DeleteTmpPackFiles(path):
+ pack_dir = os.path.join(path, 'objects', 'pack')
+ if not os.path.isdir(pack_dir):
+ return
+ pack_files = [
+ f for f in os.listdir(pack_dir)
+ if f.startswith('.tmp-') or f.startswith('tmp_pack_')
+ ]
+ for f in pack_files:
+ f = os.path.join(pack_dir, f)
+ try:
+ os.remove(f)
+ logging.warning('Deleted stale temporary pack file %s' % f)
+ except OSError:
+ logging.warning('Unable to delete temporary pack file %s' % f)
@subcommand.usage('[url of repo to check for caching]')
@metrics.collector.collect_metrics('git cache exists')
def CMDexists(parser, args):
- """Check to see if there already is a cache of the given repo."""
- _, args = parser.parse_args(args)
- if not len(args) == 1:
- parser.error('git cache exists only takes exactly one repo url.')
- url = args[0]
- mirror = Mirror(url)
- if mirror.exists():
- print(mirror.mirror_path)
- return 0
- return 1
+ """Check to see if there already is a cache of the given repo."""
+ _, args = parser.parse_args(args)
+ if not len(args) == 1:
+ parser.error('git cache exists only takes exactly one repo url.')
+ url = args[0]
+ mirror = Mirror(url)
+ if mirror.exists():
+ print(mirror.mirror_path)
+ return 0
+ return 1
@subcommand.usage('[url of repo to create a bootstrap zip file]')
@metrics.collector.collect_metrics('git cache update-bootstrap')
def CMDupdate_bootstrap(parser, args):
- """Create and uploads a bootstrap tarball."""
- # Lets just assert we can't do this on Windows.
- if sys.platform.startswith('win'):
- print('Sorry, update bootstrap will not work on Windows.', file=sys.stderr)
- return 1
+ """Create and uploads a bootstrap tarball."""
+ # Lets just assert we can't do this on Windows.
+ if sys.platform.startswith('win'):
+ print('Sorry, update bootstrap will not work on Windows.',
+ file=sys.stderr)
+ return 1
- parser.add_option('--skip-populate', action='store_true',
- help='Skips "populate" step if mirror already exists.')
- parser.add_option('--gc-aggressive', action='store_true',
- help='Run aggressive repacking of the repo.')
- parser.add_option('--prune', action='store_true',
- help='Prune all other cached bundles of the same repo.')
+ parser.add_option('--skip-populate',
+ action='store_true',
+ help='Skips "populate" step if mirror already exists.')
+ parser.add_option('--gc-aggressive',
+ action='store_true',
+ help='Run aggressive repacking of the repo.')
+ parser.add_option('--prune',
+ action='store_true',
+ help='Prune all other cached bundles of the same repo.')
- populate_args = args[:]
- options, args = parser.parse_args(args)
- url = args[0]
- mirror = Mirror(url)
- if not options.skip_populate or not mirror.exists():
- CMDpopulate(parser, populate_args)
- else:
- print('Skipped populate step.')
+ populate_args = args[:]
+ options, args = parser.parse_args(args)
+ url = args[0]
+ mirror = Mirror(url)
+ if not options.skip_populate or not mirror.exists():
+ CMDpopulate(parser, populate_args)
+ else:
+ print('Skipped populate step.')
- # Get the repo directory.
- _, args2 = parser.parse_args(args)
- url = args2[0]
- mirror = Mirror(url)
- mirror.update_bootstrap(options.prune, options.gc_aggressive)
- return 0
+ # Get the repo directory.
+ _, args2 = parser.parse_args(args)
+ url = args2[0]
+ mirror = Mirror(url)
+ mirror.update_bootstrap(options.prune, options.gc_aggressive)
+ return 0
@subcommand.usage('[url of repo to add to or update in cache]')
@metrics.collector.collect_metrics('git cache populate')
def CMDpopulate(parser, args):
- """Ensure that the cache has all up-to-date objects for the given repo."""
- parser.add_option('--depth', type='int',
- help='Only cache DEPTH commits of history')
- parser.add_option(
- '--no-fetch-tags',
- action='store_true',
- help=('Don\'t fetch tags from the server. This can speed up '
- 'fetch considerably when there are many tags.'))
- parser.add_option('--shallow', '-s', action='store_true',
- help='Only cache 10000 commits of history')
- parser.add_option('--ref', action='append',
- help='Specify additional refs to be fetched')
- parser.add_option('--commit', action='append',
- help='Specify additional commits to be fetched')
- parser.add_option('--no_bootstrap', '--no-bootstrap',
- action='store_true',
- help='Don\'t bootstrap from Google Storage')
- parser.add_option('--ignore_locks',
- '--ignore-locks',
- action='store_true',
- help='NOOP. This flag will be removed in the future.')
- parser.add_option('--break-locks',
- action='store_true',
- help='Break any existing lock instead of just ignoring it')
- parser.add_option('--reset-fetch-config', action='store_true', default=False,
- help='Reset the fetch config before populating the cache.')
+ """Ensure that the cache has all up-to-date objects for the given repo."""
+ parser.add_option('--depth',
+ type='int',
+ help='Only cache DEPTH commits of history')
+ parser.add_option(
+ '--no-fetch-tags',
+ action='store_true',
+ help=('Don\'t fetch tags from the server. This can speed up '
+ 'fetch considerably when there are many tags.'))
+ parser.add_option('--shallow',
+ '-s',
+ action='store_true',
+ help='Only cache 10000 commits of history')
+ parser.add_option('--ref',
+ action='append',
+ help='Specify additional refs to be fetched')
+ parser.add_option('--commit',
+ action='append',
+ help='Specify additional commits to be fetched')
+ parser.add_option('--no_bootstrap',
+ '--no-bootstrap',
+ action='store_true',
+ help='Don\'t bootstrap from Google Storage')
+ parser.add_option('--ignore_locks',
+ '--ignore-locks',
+ action='store_true',
+ help='NOOP. This flag will be removed in the future.')
+ parser.add_option(
+ '--break-locks',
+ action='store_true',
+ help='Break any existing lock instead of just ignoring it')
+ parser.add_option(
+ '--reset-fetch-config',
+ action='store_true',
+ default=False,
+ help='Reset the fetch config before populating the cache.')
- options, args = parser.parse_args(args)
- if not len(args) == 1:
- parser.error('git cache populate only takes exactly one repo url.')
- if options.ignore_locks:
- print('ignore_locks is no longer used. Please remove its usage.')
- if options.break_locks:
- print('break_locks is no longer used. Please remove its usage.')
- url = args[0]
+ options, args = parser.parse_args(args)
+ if not len(args) == 1:
+ parser.error('git cache populate only takes exactly one repo url.')
+ if options.ignore_locks:
+ print('ignore_locks is no longer used. Please remove its usage.')
+ if options.break_locks:
+ print('break_locks is no longer used. Please remove its usage.')
+ url = args[0]
- mirror = Mirror(url, refs=options.ref, commits=options.commit)
- kwargs = {
- 'no_fetch_tags': options.no_fetch_tags,
- 'verbose': options.verbose,
- 'shallow': options.shallow,
- 'bootstrap': not options.no_bootstrap,
- 'lock_timeout': options.timeout,
- 'reset_fetch_config': options.reset_fetch_config,
- }
- if options.depth:
- kwargs['depth'] = options.depth
- mirror.populate(**kwargs)
+ mirror = Mirror(url, refs=options.ref, commits=options.commit)
+ kwargs = {
+ 'no_fetch_tags': options.no_fetch_tags,
+ 'verbose': options.verbose,
+ 'shallow': options.shallow,
+ 'bootstrap': not options.no_bootstrap,
+ 'lock_timeout': options.timeout,
+ 'reset_fetch_config': options.reset_fetch_config,
+ }
+ if options.depth:
+ kwargs['depth'] = options.depth
+ mirror.populate(**kwargs)
@subcommand.usage('Fetch new commits into cache and current checkout')
@metrics.collector.collect_metrics('git cache fetch')
def CMDfetch(parser, args):
- """Update mirror, and fetch in cwd."""
- parser.add_option('--all', action='store_true', help='Fetch all remotes')
- parser.add_option('--no_bootstrap', '--no-bootstrap',
- action='store_true',
- help='Don\'t (re)bootstrap from Google Storage')
- parser.add_option(
- '--no-fetch-tags',
- action='store_true',
- help=('Don\'t fetch tags from the server. This can speed up '
- 'fetch considerably when there are many tags.'))
- options, args = parser.parse_args(args)
+ """Update mirror, and fetch in cwd."""
+ parser.add_option('--all', action='store_true', help='Fetch all remotes')
+ parser.add_option('--no_bootstrap',
+ '--no-bootstrap',
+ action='store_true',
+ help='Don\'t (re)bootstrap from Google Storage')
+ parser.add_option(
+ '--no-fetch-tags',
+ action='store_true',
+ help=('Don\'t fetch tags from the server. This can speed up '
+ 'fetch considerably when there are many tags.'))
+ options, args = parser.parse_args(args)
- # Figure out which remotes to fetch. This mimics the behavior of regular
- # 'git fetch'. Note that in the case of "stacked" or "pipelined" branches,
- # this will NOT try to traverse up the branching structure to find the
- # ultimate remote to update.
- remotes = []
- if options.all:
- assert not args, 'fatal: fetch --all does not take a repository argument'
- remotes = subprocess.check_output([Mirror.git_exe, 'remote'])
- remotes = remotes.decode('utf-8', 'ignore').splitlines()
- elif args:
- remotes = args
- else:
- current_branch = subprocess.check_output(
- [Mirror.git_exe, 'rev-parse', '--abbrev-ref', 'HEAD'])
- current_branch = current_branch.decode('utf-8', 'ignore').strip()
- if current_branch != 'HEAD':
- upstream = subprocess.check_output(
- [Mirror.git_exe, 'config', 'branch.%s.remote' % current_branch])
- upstream = upstream.decode('utf-8', 'ignore').strip()
- if upstream and upstream != '.':
- remotes = [upstream]
- if not remotes:
- remotes = ['origin']
+ # Figure out which remotes to fetch. This mimics the behavior of regular
+ # 'git fetch'. Note that in the case of "stacked" or "pipelined" branches,
+ # this will NOT try to traverse up the branching structure to find the
+ # ultimate remote to update.
+ remotes = []
+ if options.all:
+ assert not args, 'fatal: fetch --all does not take repository argument'
+ remotes = subprocess.check_output([Mirror.git_exe, 'remote'])
+ remotes = remotes.decode('utf-8', 'ignore').splitlines()
+ elif args:
+ remotes = args
+ else:
+ current_branch = subprocess.check_output(
+ [Mirror.git_exe, 'rev-parse', '--abbrev-ref', 'HEAD'])
+ current_branch = current_branch.decode('utf-8', 'ignore').strip()
+ if current_branch != 'HEAD':
+ upstream = subprocess.check_output(
+ [Mirror.git_exe, 'config',
+ 'branch.%s.remote' % current_branch])
+ upstream = upstream.decode('utf-8', 'ignore').strip()
+ if upstream and upstream != '.':
+ remotes = [upstream]
+ if not remotes:
+ remotes = ['origin']
- cachepath = Mirror.GetCachePath()
- git_dir = os.path.abspath(subprocess.check_output(
- [Mirror.git_exe, 'rev-parse', '--git-dir']).decode('utf-8', 'ignore'))
- git_dir = os.path.abspath(git_dir)
- if git_dir.startswith(cachepath):
- mirror = Mirror.FromPath(git_dir)
- mirror.populate(
- bootstrap=not options.no_bootstrap,
- no_fetch_tags=options.no_fetch_tags,
- lock_timeout=options.timeout)
+ cachepath = Mirror.GetCachePath()
+ git_dir = os.path.abspath(
+ subprocess.check_output([Mirror.git_exe, 'rev-parse',
+ '--git-dir']).decode('utf-8', 'ignore'))
+ git_dir = os.path.abspath(git_dir)
+ if git_dir.startswith(cachepath):
+ mirror = Mirror.FromPath(git_dir)
+ mirror.populate(bootstrap=not options.no_bootstrap,
+ no_fetch_tags=options.no_fetch_tags,
+ lock_timeout=options.timeout)
+ return 0
+ for remote in remotes:
+ remote_url = subprocess.check_output(
+ [Mirror.git_exe, 'config',
+ 'remote.%s.url' % remote])
+ remote_url = remote_url.decode('utf-8', 'ignore').strip()
+ if remote_url.startswith(cachepath):
+ mirror = Mirror.FromPath(remote_url)
+ mirror.print = lambda *args: None
+ print('Updating git cache...')
+ mirror.populate(bootstrap=not options.no_bootstrap,
+ no_fetch_tags=options.no_fetch_tags,
+ lock_timeout=options.timeout)
+ subprocess.check_call([Mirror.git_exe, 'fetch', remote])
return 0
- for remote in remotes:
- remote_url = subprocess.check_output(
- [Mirror.git_exe, 'config', 'remote.%s.url' % remote])
- remote_url = remote_url.decode('utf-8', 'ignore').strip()
- if remote_url.startswith(cachepath):
- mirror = Mirror.FromPath(remote_url)
- mirror.print = lambda *args: None
- print('Updating git cache...')
- mirror.populate(
- bootstrap=not options.no_bootstrap,
- no_fetch_tags=options.no_fetch_tags,
- lock_timeout=options.timeout)
- subprocess.check_call([Mirror.git_exe, 'fetch', remote])
- return 0
@subcommand.usage('do not use - it is a noop.')
@metrics.collector.collect_metrics('git cache unlock')
def CMDunlock(parser, args):
- """This command does nothing."""
- print('This command does nothing and will be removed in the future.')
+ """This command does nothing."""
+ print('This command does nothing and will be removed in the future.')
class OptionParser(optparse.OptionParser):
- """Wrapper class for OptionParser to handle global options."""
+ """Wrapper class for OptionParser to handle global options."""
+ def __init__(self, *args, **kwargs):
+ optparse.OptionParser.__init__(self, *args, prog='git cache', **kwargs)
+ self.add_option(
+ '-c',
+ '--cache-dir',
+ help=('Path to the directory containing the caches. Normally '
+ 'deduced from git config cache.cachepath or '
+ '$GIT_CACHE_PATH.'))
+ self.add_option(
+ '-v',
+ '--verbose',
+ action='count',
+ default=1,
+ help='Increase verbosity (can be passed multiple times)')
+ self.add_option('-q',
+ '--quiet',
+ action='store_true',
+ help='Suppress all extraneous output')
+ self.add_option('--timeout',
+ type='int',
+ default=0,
+ help='Timeout for acquiring cache lock, in seconds')
- def __init__(self, *args, **kwargs):
- optparse.OptionParser.__init__(self, *args, prog='git cache', **kwargs)
- self.add_option('-c', '--cache-dir',
- help=(
- 'Path to the directory containing the caches. Normally '
- 'deduced from git config cache.cachepath or '
- '$GIT_CACHE_PATH.'))
- self.add_option('-v', '--verbose', action='count', default=1,
- help='Increase verbosity (can be passed multiple times)')
- self.add_option('-q', '--quiet', action='store_true',
- help='Suppress all extraneous output')
- self.add_option('--timeout', type='int', default=0,
- help='Timeout for acquiring cache lock, in seconds')
+ def parse_args(self, args=None, values=None):
+ # Create an optparse.Values object that will store only the actual
+ # passed options, without the defaults.
+ actual_options = optparse.Values()
+ _, args = optparse.OptionParser.parse_args(self, args, actual_options)
+ # Create an optparse.Values object with the default options.
+ options = optparse.Values(self.get_default_values().__dict__)
+ # Update it with the options passed by the user.
+ options._update_careful(actual_options.__dict__)
+ # Store the options passed by the user in an _actual_options attribute.
+ # We store only the keys, and not the values, since the values can
+ # contain arbitrary information, which might be PII.
+ metrics.collector.add('arguments', list(actual_options.__dict__.keys()))
- def parse_args(self, args=None, values=None):
- # Create an optparse.Values object that will store only the actual passed
- # options, without the defaults.
- actual_options = optparse.Values()
- _, args = optparse.OptionParser.parse_args(self, args, actual_options)
- # Create an optparse.Values object with the default options.
- options = optparse.Values(self.get_default_values().__dict__)
- # Update it with the options passed by the user.
- options._update_careful(actual_options.__dict__)
- # Store the options passed by the user in an _actual_options attribute.
- # We store only the keys, and not the values, since the values can contain
- # arbitrary information, which might be PII.
- metrics.collector.add('arguments', list(actual_options.__dict__.keys()))
+ if options.quiet:
+ options.verbose = 0
- if options.quiet:
- options.verbose = 0
+ levels = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG]
+ logging.basicConfig(level=levels[min(options.verbose, len(levels) - 1)])
- levels = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG]
- logging.basicConfig(level=levels[min(options.verbose, len(levels) - 1)])
+ try:
+ global_cache_dir = Mirror.GetCachePath()
+ except RuntimeError:
+ global_cache_dir = None
+ if options.cache_dir:
+ if global_cache_dir and (os.path.abspath(options.cache_dir) !=
+ os.path.abspath(global_cache_dir)):
+ logging.warning(
+ 'Overriding globally-configured cache directory.')
+ Mirror.SetCachePath(options.cache_dir)
- try:
- global_cache_dir = Mirror.GetCachePath()
- except RuntimeError:
- global_cache_dir = None
- if options.cache_dir:
- if global_cache_dir and (
- os.path.abspath(options.cache_dir) !=
- os.path.abspath(global_cache_dir)):
- logging.warning('Overriding globally-configured cache directory.')
- Mirror.SetCachePath(options.cache_dir)
-
- return options, args
+ return options, args
def main(argv):
- dispatcher = subcommand.CommandDispatcher(__name__)
- return dispatcher.execute(OptionParser(), argv)
+ dispatcher = subcommand.CommandDispatcher(__name__)
+ return dispatcher.execute(OptionParser(), argv)
if __name__ == '__main__':
- try:
- with metrics.collector.print_notice_and_exit():
- sys.exit(main(sys.argv[1:]))
- except KeyboardInterrupt:
- sys.stderr.write('interrupted\n')
- sys.exit(1)
+ try:
+ with metrics.collector.print_notice_and_exit():
+ sys.exit(main(sys.argv[1:]))
+ except KeyboardInterrupt:
+ sys.stderr.write('interrupted\n')
+ sys.exit(1)