blob: 3229202104c6defe9a7e0cfcb1e8aec880198e4d [file] [log] [blame]
agable@chromium.org5a306a22014-02-24 22:13:59 +00001#!/usr/bin/env python
2# Copyright 2014 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""A git command for managing a local cache of git repositories."""
7
szager@chromium.org848fd492014-04-09 19:06:44 +00008from __future__ import print_function
agable@chromium.org5a306a22014-02-24 22:13:59 +00009import errno
10import logging
11import optparse
12import os
szager@chromium.org174766f2014-05-13 21:27:46 +000013import re
agable@chromium.org5a306a22014-02-24 22:13:59 +000014import tempfile
szager@chromium.org1132f5f2014-08-23 01:57:59 +000015import threading
pgervais@chromium.orgf3726102014-04-17 17:24:15 +000016import time
agable@chromium.org5a306a22014-02-24 22:13:59 +000017import subprocess
18import sys
19import urlparse
hinoka@google.com776a2c32014-04-25 07:54:25 +000020import zipfile
agable@chromium.org5a306a22014-02-24 22:13:59 +000021
hinoka@google.com563559c2014-04-02 00:36:24 +000022from download_from_google_storage import Gsutil
agable@chromium.org5a306a22014-02-24 22:13:59 +000023import gclient_utils
24import subcommand
25
szager@chromium.org301a7c32014-06-16 17:13:50 +000026# Analogous to gc.autopacklimit git config.
27GC_AUTOPACKLIMIT = 50
28
hinoka@chromium.orgaa1e1a42014-06-26 21:58:51 +000029GIT_CACHE_CORRUPT_MESSAGE = 'WARNING: The Git cache is corrupt.'
30
szager@chromium.org848fd492014-04-09 19:06:44 +000031try:
Quinten Yearsleyb2cc4a92016-12-15 13:53:26 -080032 # pylint: disable=undefined-variable
szager@chromium.org848fd492014-04-09 19:06:44 +000033 WinErr = WindowsError
34except NameError:
35 class WinErr(Exception):
36 pass
agable@chromium.org5a306a22014-02-24 22:13:59 +000037
hinokadcd84042016-06-09 14:26:17 -070038class ClobberNeeded(Exception):
hinoka@chromium.orgaa1e1a42014-06-26 21:58:51 +000039 pass
agable@chromium.org5a306a22014-02-24 22:13:59 +000040
dnj4625b5a2016-11-10 18:23:26 -080041
42def exponential_backoff_retry(fn, excs=(Exception,), name=None, count=10,
43 sleep_time=0.25, printerr=None):
44 """Executes |fn| up to |count| times, backing off exponentially.
45
46 Args:
47 fn (callable): The function to execute. If this raises a handled
48 exception, the function will retry with exponential backoff.
49 excs (tuple): A tuple of Exception types to handle. If one of these is
50 raised by |fn|, a retry will be attempted. If |fn| raises an Exception
51 that is not in this list, it will immediately pass through. If |excs|
52 is empty, the Exception base class will be used.
53 name (str): Optional operation name to print in the retry string.
54 count (int): The number of times to try before allowing the exception to
55 pass through.
56 sleep_time (float): The initial number of seconds to sleep in between
57 retries. This will be doubled each retry.
58 printerr (callable): Function that will be called with the error string upon
59 failures. If None, |logging.warning| will be used.
60
61 Returns: The return value of the successful fn.
62 """
63 printerr = printerr or logging.warning
64 for i in xrange(count):
65 try:
66 return fn()
67 except excs as e:
68 if (i+1) >= count:
69 raise
70
71 printerr('Retrying %s in %.2f second(s) (%d / %d attempts): %s' % (
72 (name or 'operation'), sleep_time, (i+1), count, e))
73 time.sleep(sleep_time)
74 sleep_time *= 2
75
76
szager@chromium.org848fd492014-04-09 19:06:44 +000077class Mirror(object):
78
79 git_exe = 'git.bat' if sys.platform.startswith('win') else 'git'
80 gsutil_exe = os.path.join(
hinoka@chromium.orgb091aa52014-12-20 01:47:31 +000081 os.path.dirname(os.path.abspath(__file__)), 'gsutil.py')
szager@chromium.org848fd492014-04-09 19:06:44 +000082
szager@chromium.org66c8b852015-09-22 23:19:07 +000083 @staticmethod
84 def parse_fetch_spec(spec):
85 """Parses and canonicalizes a fetch spec.
86
87 Returns (fetchspec, value_regex), where value_regex can be used
88 with 'git config --replace-all'.
89 """
90 parts = spec.split(':', 1)
91 src = parts[0].lstrip('+').rstrip('/')
92 if not src.startswith('refs/'):
93 src = 'refs/heads/%s' % src
94 dest = parts[1].rstrip('/') if len(parts) > 1 else src
95 regex = r'\+%s:.*' % src.replace('*', r'\*')
96 return ('+%s:%s' % (src, dest), regex)
97
szager@chromium.org848fd492014-04-09 19:06:44 +000098 def __init__(self, url, refs=None, print_func=None):
99 self.url = url
szager@chromium.org66c8b852015-09-22 23:19:07 +0000100 self.fetch_specs = set([self.parse_fetch_spec(ref) for ref in (refs or [])])
szager@chromium.org848fd492014-04-09 19:06:44 +0000101 self.basedir = self.UrlToCacheDir(url)
102 self.mirror_path = os.path.join(self.GetCachePath(), self.basedir)
loislo@chromium.org0fb693f2014-12-25 15:28:22 +0000103 if print_func:
104 self.print = self.print_without_file
105 self.print_func = print_func
106 else:
107 self.print = print
108
dnj4625b5a2016-11-10 18:23:26 -0800109 def print_without_file(self, message, **_kwargs):
loislo@chromium.org0fb693f2014-12-25 15:28:22 +0000110 self.print_func(message)
szager@chromium.org848fd492014-04-09 19:06:44 +0000111
hinoka@chromium.orgf8fa23d2014-06-05 01:00:04 +0000112 @property
113 def bootstrap_bucket(self):
Ryan Tseng3beabd02017-03-15 13:57:58 -0700114 u = urlparse.urlparse(self.url)
115 if u.netloc == 'chromium.googlesource.com':
hinoka@chromium.orgf8fa23d2014-06-05 01:00:04 +0000116 return 'chromium-git-cache'
Ryan Tseng3beabd02017-03-15 13:57:58 -0700117 elif u.netloc == 'chrome-internal.googlesource.com':
118 return 'chrome-git-cache'
119 # Not recognized.
120 return None
hinoka@chromium.orgf8fa23d2014-06-05 01:00:04 +0000121
szager@chromium.org174766f2014-05-13 21:27:46 +0000122 @classmethod
123 def FromPath(cls, path):
124 return cls(cls.CacheDirToUrl(path))
125
szager@chromium.org848fd492014-04-09 19:06:44 +0000126 @staticmethod
127 def UrlToCacheDir(url):
128 """Convert a git url to a normalized form for the cache dir path."""
129 parsed = urlparse.urlparse(url)
130 norm_url = parsed.netloc + parsed.path
131 if norm_url.endswith('.git'):
132 norm_url = norm_url[:-len('.git')]
133 return norm_url.replace('-', '--').replace('/', '-').lower()
134
135 @staticmethod
szager@chromium.org174766f2014-05-13 21:27:46 +0000136 def CacheDirToUrl(path):
137 """Convert a cache dir path to its corresponding url."""
138 netpath = re.sub(r'\b-\b', '/', os.path.basename(path)).replace('--', '-')
139 return 'https://%s' % netpath
140
szager@chromium.org848fd492014-04-09 19:06:44 +0000141 @classmethod
142 def SetCachePath(cls, cachepath):
Ryan Tsengc3eb3fa2017-10-09 17:15:42 -0700143 setattr(cls, 'cachepath', cachepath)
szager@chromium.org848fd492014-04-09 19:06:44 +0000144
145 @classmethod
146 def GetCachePath(cls):
Ryan Tsengc3eb3fa2017-10-09 17:15:42 -0700147 if not hasattr(cls, 'cachepath'):
148 try:
149 cachepath = subprocess.check_output(
150 [cls.git_exe, 'config', '--global', 'cache.cachepath']).strip()
151 except subprocess.CalledProcessError:
152 cachepath = None
153 if not cachepath:
154 raise RuntimeError(
155 'No global cache.cachepath git configuration found.')
156 setattr(cls, 'cachepath', cachepath)
157 return getattr(cls, 'cachepath')
szager@chromium.org848fd492014-04-09 19:06:44 +0000158
dnj4625b5a2016-11-10 18:23:26 -0800159 def Rename(self, src, dst):
160 # This is somehow racy on Windows.
161 # Catching OSError because WindowsError isn't portable and
162 # pylint complains.
163 exponential_backoff_retry(
164 lambda: os.rename(src, dst),
165 excs=(OSError,),
166 name='rename [%s] => [%s]' % (src, dst),
167 printerr=self.print)
168
szager@chromium.org848fd492014-04-09 19:06:44 +0000169 def RunGit(self, cmd, **kwargs):
170 """Run git in a subprocess."""
171 cwd = kwargs.setdefault('cwd', self.mirror_path)
172 kwargs.setdefault('print_stdout', False)
173 kwargs.setdefault('filter_fn', self.print)
174 env = kwargs.get('env') or kwargs.setdefault('env', os.environ.copy())
175 env.setdefault('GIT_ASKPASS', 'true')
176 env.setdefault('SSH_ASKPASS', 'true')
177 self.print('running "git %s" in "%s"' % (' '.join(cmd), cwd))
178 gclient_utils.CheckCallAndFilter([self.git_exe] + cmd, **kwargs)
179
180 def config(self, cwd=None):
181 if cwd is None:
182 cwd = self.mirror_path
szager@chromium.org301a7c32014-06-16 17:13:50 +0000183
184 # Don't run git-gc in a daemon. Bad things can happen if it gets killed.
hinokadcd84042016-06-09 14:26:17 -0700185 try:
186 self.RunGit(['config', 'gc.autodetach', '0'], cwd=cwd)
187 except subprocess.CalledProcessError:
188 # Hard error, need to clobber.
189 raise ClobberNeeded()
szager@chromium.org301a7c32014-06-16 17:13:50 +0000190
191 # Don't combine pack files into one big pack file. It's really slow for
192 # repositories, and there's no way to track progress and make sure it's
193 # not stuck.
Ryan Tseng3beabd02017-03-15 13:57:58 -0700194 if self.supported_project():
195 self.RunGit(['config', 'gc.autopacklimit', '0'], cwd=cwd)
szager@chromium.org301a7c32014-06-16 17:13:50 +0000196
197 # Allocate more RAM for cache-ing delta chains, for better performance
198 # of "Resolving deltas".
szager@chromium.org848fd492014-04-09 19:06:44 +0000199 self.RunGit(['config', 'core.deltaBaseCacheLimit',
hinoka@chromium.org8e095af2015-06-10 19:19:07 +0000200 gclient_utils.DefaultDeltaBaseCacheLimit()], cwd=cwd)
szager@chromium.org301a7c32014-06-16 17:13:50 +0000201
hinoka@chromium.org8e095af2015-06-10 19:19:07 +0000202 self.RunGit(['config', 'remote.origin.url', self.url], cwd=cwd)
szager@chromium.org848fd492014-04-09 19:06:44 +0000203 self.RunGit(['config', '--replace-all', 'remote.origin.fetch',
hinoka@chromium.org8e095af2015-06-10 19:19:07 +0000204 '+refs/heads/*:refs/heads/*', r'\+refs/heads/\*:.*'], cwd=cwd)
szager@chromium.org66c8b852015-09-22 23:19:07 +0000205 for spec, value_regex in self.fetch_specs:
szager@chromium.org965c44f2014-08-19 21:19:19 +0000206 self.RunGit(
szager@chromium.org66c8b852015-09-22 23:19:07 +0000207 ['config', '--replace-all', 'remote.origin.fetch', spec, value_regex],
hinoka@chromium.org8e095af2015-06-10 19:19:07 +0000208 cwd=cwd)
szager@chromium.org848fd492014-04-09 19:06:44 +0000209
210 def bootstrap_repo(self, directory):
hinoka@chromium.orgaa1e1a42014-06-26 21:58:51 +0000211 """Bootstrap the repo from Google Stroage if possible.
212
213 More apt-ly named bootstrap_repo_from_cloud_if_possible_else_do_nothing().
214 """
szager@chromium.org848fd492014-04-09 19:06:44 +0000215
Ryan Tseng3beabd02017-03-15 13:57:58 -0700216 if not self.bootstrap_bucket:
217 return False
hinoka@google.com776a2c32014-04-25 07:54:25 +0000218 python_fallback = False
sbc@chromium.org9d0644d2015-06-05 23:16:54 +0000219 if (sys.platform.startswith('win') and
220 not gclient_utils.FindExecutable('7z')):
hinoka@google.com776a2c32014-04-25 07:54:25 +0000221 python_fallback = True
222 elif sys.platform.startswith('darwin'):
223 # The OSX version of unzip doesn't support zip64.
224 python_fallback = True
sbc@chromium.org9d0644d2015-06-05 23:16:54 +0000225 elif not gclient_utils.FindExecutable('unzip'):
hinoka@google.com776a2c32014-04-25 07:54:25 +0000226 python_fallback = True
szager@chromium.org848fd492014-04-09 19:06:44 +0000227
228 gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir)
hinoka@chromium.org199bc5f2014-12-17 02:17:14 +0000229 gsutil = Gsutil(self.gsutil_exe, boto_path=None)
szager@chromium.org848fd492014-04-09 19:06:44 +0000230 # Get the most recent version of the zipfile.
231 _, ls_out, _ = gsutil.check_call('ls', gs_folder)
232 ls_out_sorted = sorted(ls_out.splitlines())
233 if not ls_out_sorted:
234 # This repo is not on Google Storage.
235 return False
236 latest_checkout = ls_out_sorted[-1]
237
238 # Download zip file to a temporary directory.
239 try:
szager@chromium.org1cbf1042014-06-17 18:26:24 +0000240 tempdir = tempfile.mkdtemp(prefix='_cache_tmp', dir=self.GetCachePath())
szager@chromium.org848fd492014-04-09 19:06:44 +0000241 self.print('Downloading %s' % latest_checkout)
hinoka@chromium.orgc58d11d2014-06-09 23:34:35 +0000242 code = gsutil.call('cp', latest_checkout, tempdir)
szager@chromium.org848fd492014-04-09 19:06:44 +0000243 if code:
szager@chromium.org848fd492014-04-09 19:06:44 +0000244 return False
245 filename = os.path.join(tempdir, latest_checkout.split('/')[-1])
246
hinoka@google.com776a2c32014-04-25 07:54:25 +0000247 # Unpack the file with 7z on Windows, unzip on linux, or fallback.
248 if not python_fallback:
249 if sys.platform.startswith('win'):
250 cmd = ['7z', 'x', '-o%s' % directory, '-tzip', filename]
251 else:
252 cmd = ['unzip', filename, '-d', directory]
253 retcode = subprocess.call(cmd)
szager@chromium.org848fd492014-04-09 19:06:44 +0000254 else:
hinoka@google.com776a2c32014-04-25 07:54:25 +0000255 try:
256 with zipfile.ZipFile(filename, 'r') as f:
257 f.printdir()
258 f.extractall(directory)
259 except Exception as e:
260 self.print('Encountered error: %s' % str(e), file=sys.stderr)
261 retcode = 1
262 else:
263 retcode = 0
szager@chromium.org848fd492014-04-09 19:06:44 +0000264 finally:
265 # Clean up the downloaded zipfile.
dnj4625b5a2016-11-10 18:23:26 -0800266 #
267 # This is somehow racy on Windows.
268 # Catching OSError because WindowsError isn't portable and
269 # pylint complains.
270 exponential_backoff_retry(
271 lambda: gclient_utils.rm_file_or_tree(tempdir),
272 excs=(OSError,),
273 name='rmtree [%s]' % (tempdir,),
274 printerr=self.print)
szager@chromium.org848fd492014-04-09 19:06:44 +0000275
276 if retcode:
277 self.print(
278 'Extracting bootstrap zipfile %s failed.\n'
279 'Resuming normal operations.' % filename)
280 return False
281 return True
282
283 def exists(self):
284 return os.path.isfile(os.path.join(self.mirror_path, 'config'))
285
Ryan Tseng3beabd02017-03-15 13:57:58 -0700286 def supported_project(self):
287 """Returns true if this repo is known to have a bootstrap zip file."""
288 u = urlparse.urlparse(self.url)
289 return u.netloc in [
290 'chromium.googlesource.com',
291 'chrome-internal.googlesource.com']
292
szager@chromium.org66c8b852015-09-22 23:19:07 +0000293 def _preserve_fetchspec(self):
294 """Read and preserve remote.origin.fetch from an existing mirror.
295
296 This modifies self.fetch_specs.
297 """
298 if not self.exists():
299 return
300 try:
301 config_fetchspecs = subprocess.check_output(
302 [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'],
303 cwd=self.mirror_path)
304 for fetchspec in config_fetchspecs.splitlines():
305 self.fetch_specs.add(self.parse_fetch_spec(fetchspec))
306 except subprocess.CalledProcessError:
307 logging.warn('Tried and failed to preserve remote.origin.fetch from the '
308 'existing cache directory. You may need to manually edit '
309 '%s and "git cache fetch" again.'
310 % os.path.join(self.mirror_path, 'config'))
311
hinoka@chromium.orgaa1e1a42014-06-26 21:58:51 +0000312 def _ensure_bootstrapped(self, depth, bootstrap, force=False):
313 tempdir = None
hinoka@chromium.orgaa1e1a42014-06-26 21:58:51 +0000314 pack_dir = os.path.join(self.mirror_path, 'objects', 'pack')
315 pack_files = []
316
317 if os.path.isdir(pack_dir):
318 pack_files = [f for f in os.listdir(pack_dir) if f.endswith('.pack')]
319
320 should_bootstrap = (force or
szager@chromium.org66c8b852015-09-22 23:19:07 +0000321 not self.exists() or
hinoka@chromium.orgaa1e1a42014-06-26 21:58:51 +0000322 len(pack_files) > GC_AUTOPACKLIMIT)
323 if should_bootstrap:
szager@chromium.org66c8b852015-09-22 23:19:07 +0000324 if self.exists():
325 # Re-bootstrapping an existing mirror; preserve existing fetch spec.
326 self._preserve_fetchspec()
hinoka@chromium.orgaa1e1a42014-06-26 21:58:51 +0000327 tempdir = tempfile.mkdtemp(
328 prefix='_cache_tmp', suffix=self.basedir, dir=self.GetCachePath())
329 bootstrapped = not depth and bootstrap and self.bootstrap_repo(tempdir)
330 if bootstrapped:
331 # Bootstrap succeeded; delete previous cache, if any.
hinoka@chromium.org42f9adf2014-09-05 11:10:35 +0000332 gclient_utils.rmtree(self.mirror_path)
Ryan Tseng3beabd02017-03-15 13:57:58 -0700333 elif not self.exists() or not self.supported_project():
334 # Bootstrap failed due to either
335 # 1. No previous cache
336 # 2. Project doesn't have a bootstrap zip file
337 # Start with a bare git dir.
hinoka@chromium.orgaa1e1a42014-06-26 21:58:51 +0000338 self.RunGit(['init', '--bare'], cwd=tempdir)
339 else:
340 # Bootstrap failed, previous cache exists; warn and continue.
341 logging.warn(
342 'Git cache has a lot of pack files (%d). Tried to re-bootstrap '
343 'but failed. Continuing with non-optimized repository.'
344 % len(pack_files))
345 gclient_utils.rmtree(tempdir)
346 tempdir = None
347 else:
348 if depth and os.path.exists(os.path.join(self.mirror_path, 'shallow')):
349 logging.warn(
350 'Shallow fetch requested, but repo cache already exists.')
351 return tempdir
352
353 def _fetch(self, rundir, verbose, depth):
354 self.config(rundir)
355 v = []
356 d = []
357 if verbose:
358 v = ['-v', '--progress']
359 if depth:
360 d = ['--depth', str(depth)]
361 fetch_cmd = ['fetch'] + v + d + ['origin']
362 fetch_specs = subprocess.check_output(
363 [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'],
364 cwd=rundir).strip().splitlines()
365 for spec in fetch_specs:
366 try:
367 self.print('Fetching %s' % spec)
368 self.RunGit(fetch_cmd + [spec], cwd=rundir, retry=True)
369 except subprocess.CalledProcessError:
370 if spec == '+refs/heads/*:refs/heads/*':
hinokadcd84042016-06-09 14:26:17 -0700371 raise ClobberNeeded() # Corrupted cache.
hinoka@chromium.orgaa1e1a42014-06-26 21:58:51 +0000372 logging.warn('Fetch of %s failed' % spec)
373
Ryan Tsengc3eb3fa2017-10-09 17:15:42 -0700374 def populate(self, depth=None, shallow=False, bootstrap=False, verbose=False):
szager@chromium.orgb0a13a22014-06-18 00:52:25 +0000375 assert self.GetCachePath()
szager@chromium.org848fd492014-04-09 19:06:44 +0000376 if shallow and not depth:
377 depth = 10000
378 gclient_utils.safe_makedirs(self.GetCachePath())
379
hinoka@chromium.orgaa1e1a42014-06-26 21:58:51 +0000380 tempdir = None
szager@chromium.org108eced2014-06-19 21:22:43 +0000381 try:
hinoka@chromium.orgaa1e1a42014-06-26 21:58:51 +0000382 tempdir = self._ensure_bootstrapped(depth, bootstrap)
szager@chromium.org848fd492014-04-09 19:06:44 +0000383 rundir = tempdir or self.mirror_path
hinoka@chromium.orgaa1e1a42014-06-26 21:58:51 +0000384 self._fetch(rundir, verbose, depth)
hinokadcd84042016-06-09 14:26:17 -0700385 except ClobberNeeded:
hinoka@chromium.orgaa1e1a42014-06-26 21:58:51 +0000386 # This is a major failure, we need to clean and force a bootstrap.
387 gclient_utils.rmtree(rundir)
388 self.print(GIT_CACHE_CORRUPT_MESSAGE)
389 tempdir = self._ensure_bootstrapped(depth, bootstrap, force=True)
390 assert tempdir
391 self._fetch(tempdir or self.mirror_path, verbose, depth)
392 finally:
szager@chromium.org848fd492014-04-09 19:06:44 +0000393 if tempdir:
dnjb445ef52016-11-10 15:51:39 -0800394 if os.path.exists(self.mirror_path):
395 gclient_utils.rmtree(self.mirror_path)
dnj4625b5a2016-11-10 18:23:26 -0800396 self.Rename(tempdir, self.mirror_path)
szager@chromium.org848fd492014-04-09 19:06:44 +0000397
hinoka@chromium.orgc8444f32014-06-18 23:18:17 +0000398 def update_bootstrap(self, prune=False):
szager@chromium.org848fd492014-04-09 19:06:44 +0000399 # The files are named <git number>.zip
400 gen_number = subprocess.check_output(
401 [self.git_exe, 'number', 'master'], cwd=self.mirror_path).strip()
hinoka@chromium.org7b1cb6f2014-09-08 21:40:50 +0000402 # Run Garbage Collect to compress packfile.
403 self.RunGit(['gc', '--prune=all'])
szager@chromium.org848fd492014-04-09 19:06:44 +0000404 # Creating a temp file and then deleting it ensures we can use this name.
405 _, tmp_zipfile = tempfile.mkstemp(suffix='.zip')
406 os.remove(tmp_zipfile)
407 subprocess.call(['zip', '-r', tmp_zipfile, '.'], cwd=self.mirror_path)
408 gsutil = Gsutil(path=self.gsutil_exe, boto_path=None)
hinoka@chromium.orgc8444f32014-06-18 23:18:17 +0000409 gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir)
410 dest_name = '%s/%s.zip' % (gs_folder, gen_number)
szager@chromium.org848fd492014-04-09 19:06:44 +0000411 gsutil.call('cp', tmp_zipfile, dest_name)
412 os.remove(tmp_zipfile)
413
hinoka@chromium.orgc8444f32014-06-18 23:18:17 +0000414 # Remove all other files in the same directory.
415 if prune:
416 _, ls_out, _ = gsutil.check_call('ls', gs_folder)
417 for filename in ls_out.splitlines():
418 if filename == dest_name:
419 continue
420 gsutil.call('rm', filename)
421
szager@chromium.orgcdfcd7c2014-06-10 23:40:46 +0000422 @staticmethod
423 def DeleteTmpPackFiles(path):
424 pack_dir = os.path.join(path, 'objects', 'pack')
szager@chromium.org33418492014-06-18 19:03:39 +0000425 if not os.path.isdir(pack_dir):
426 return
szager@chromium.orgcdfcd7c2014-06-10 23:40:46 +0000427 pack_files = [f for f in os.listdir(pack_dir) if
428 f.startswith('.tmp-') or f.startswith('tmp_pack_')]
429 for f in pack_files:
430 f = os.path.join(pack_dir, f)
431 try:
432 os.remove(f)
433 logging.warn('Deleted stale temporary pack file %s' % f)
434 except OSError:
435 logging.warn('Unable to delete temporary pack file %s' % f)
szager@chromium.org174766f2014-05-13 21:27:46 +0000436
szager@chromium.org848fd492014-04-09 19:06:44 +0000437
agable@chromium.org5a306a22014-02-24 22:13:59 +0000438@subcommand.usage('[url of repo to check for caching]')
439def CMDexists(parser, args):
440 """Check to see if there already is a cache of the given repo."""
szager@chromium.org848fd492014-04-09 19:06:44 +0000441 _, args = parser.parse_args(args)
agable@chromium.org5a306a22014-02-24 22:13:59 +0000442 if not len(args) == 1:
443 parser.error('git cache exists only takes exactly one repo url.')
444 url = args[0]
szager@chromium.org848fd492014-04-09 19:06:44 +0000445 mirror = Mirror(url)
446 if mirror.exists():
447 print(mirror.mirror_path)
agable@chromium.org5a306a22014-02-24 22:13:59 +0000448 return 0
449 return 1
450
451
hinoka@google.com563559c2014-04-02 00:36:24 +0000452@subcommand.usage('[url of repo to create a bootstrap zip file]')
453def CMDupdate_bootstrap(parser, args):
454 """Create and uploads a bootstrap tarball."""
455 # Lets just assert we can't do this on Windows.
456 if sys.platform.startswith('win'):
szager@chromium.org848fd492014-04-09 19:06:44 +0000457 print('Sorry, update bootstrap will not work on Windows.', file=sys.stderr)
hinoka@google.com563559c2014-04-02 00:36:24 +0000458 return 1
459
hinoka@chromium.orgc8444f32014-06-18 23:18:17 +0000460 parser.add_option('--prune', action='store_true',
461 help='Prune all other cached zipballs of the same repo.')
462
hinoka@google.com563559c2014-04-02 00:36:24 +0000463 # First, we need to ensure the cache is populated.
464 populate_args = args[:]
pgervais@chromium.orgb9f27512014-08-08 15:52:33 +0000465 populate_args.append('--no-bootstrap')
hinoka@google.com563559c2014-04-02 00:36:24 +0000466 CMDpopulate(parser, populate_args)
467
468 # Get the repo directory.
hinoka@chromium.orgc8444f32014-06-18 23:18:17 +0000469 options, args = parser.parse_args(args)
hinoka@google.com563559c2014-04-02 00:36:24 +0000470 url = args[0]
szager@chromium.org848fd492014-04-09 19:06:44 +0000471 mirror = Mirror(url)
hinoka@chromium.orgc8444f32014-06-18 23:18:17 +0000472 mirror.update_bootstrap(options.prune)
szager@chromium.org848fd492014-04-09 19:06:44 +0000473 return 0
hinoka@google.com563559c2014-04-02 00:36:24 +0000474
475
agable@chromium.org5a306a22014-02-24 22:13:59 +0000476@subcommand.usage('[url of repo to add to or update in cache]')
477def CMDpopulate(parser, args):
478 """Ensure that the cache has all up-to-date objects for the given repo."""
479 parser.add_option('--depth', type='int',
480 help='Only cache DEPTH commits of history')
481 parser.add_option('--shallow', '-s', action='store_true',
482 help='Only cache 10000 commits of history')
483 parser.add_option('--ref', action='append',
484 help='Specify additional refs to be fetched')
pgervais@chromium.orgb9f27512014-08-08 15:52:33 +0000485 parser.add_option('--no_bootstrap', '--no-bootstrap',
486 action='store_true',
hinoka@google.com563559c2014-04-02 00:36:24 +0000487 help='Don\'t bootstrap from Google Storage')
488
agable@chromium.org5a306a22014-02-24 22:13:59 +0000489 options, args = parser.parse_args(args)
agable@chromium.org5a306a22014-02-24 22:13:59 +0000490 if not len(args) == 1:
491 parser.error('git cache populate only takes exactly one repo url.')
492 url = args[0]
493
szager@chromium.org848fd492014-04-09 19:06:44 +0000494 mirror = Mirror(url, refs=options.ref)
495 kwargs = {
496 'verbose': options.verbose,
497 'shallow': options.shallow,
498 'bootstrap': not options.no_bootstrap,
499 }
agable@chromium.org5a306a22014-02-24 22:13:59 +0000500 if options.depth:
szager@chromium.org848fd492014-04-09 19:06:44 +0000501 kwargs['depth'] = options.depth
502 mirror.populate(**kwargs)
agable@chromium.org5a306a22014-02-24 22:13:59 +0000503
504
szager@chromium.orgf3145112014-08-07 21:02:36 +0000505@subcommand.usage('Fetch new commits into cache and current checkout')
506def CMDfetch(parser, args):
507 """Update mirror, and fetch in cwd."""
508 parser.add_option('--all', action='store_true', help='Fetch all remotes')
szager@chromium.org66c8b852015-09-22 23:19:07 +0000509 parser.add_option('--no_bootstrap', '--no-bootstrap',
510 action='store_true',
511 help='Don\'t (re)bootstrap from Google Storage')
szager@chromium.orgf3145112014-08-07 21:02:36 +0000512 options, args = parser.parse_args(args)
513
514 # Figure out which remotes to fetch. This mimics the behavior of regular
515 # 'git fetch'. Note that in the case of "stacked" or "pipelined" branches,
516 # this will NOT try to traverse up the branching structure to find the
517 # ultimate remote to update.
518 remotes = []
519 if options.all:
520 assert not args, 'fatal: fetch --all does not take a repository argument'
521 remotes = subprocess.check_output([Mirror.git_exe, 'remote']).splitlines()
522 elif args:
523 remotes = args
524 else:
525 current_branch = subprocess.check_output(
526 [Mirror.git_exe, 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
527 if current_branch != 'HEAD':
528 upstream = subprocess.check_output(
529 [Mirror.git_exe, 'config', 'branch.%s.remote' % current_branch]
530 ).strip()
531 if upstream and upstream != '.':
532 remotes = [upstream]
533 if not remotes:
534 remotes = ['origin']
535
536 cachepath = Mirror.GetCachePath()
537 git_dir = os.path.abspath(subprocess.check_output(
538 [Mirror.git_exe, 'rev-parse', '--git-dir']))
539 git_dir = os.path.abspath(git_dir)
540 if git_dir.startswith(cachepath):
541 mirror = Mirror.FromPath(git_dir)
szager@chromium.orgdbb6f822016-02-02 22:59:30 +0000542 mirror.populate(
Ryan Tsengc3eb3fa2017-10-09 17:15:42 -0700543 bootstrap=not options.no_bootstrap,)
szager@chromium.orgf3145112014-08-07 21:02:36 +0000544 return 0
545 for remote in remotes:
546 remote_url = subprocess.check_output(
547 [Mirror.git_exe, 'config', 'remote.%s.url' % remote]).strip()
548 if remote_url.startswith(cachepath):
549 mirror = Mirror.FromPath(remote_url)
550 mirror.print = lambda *args: None
551 print('Updating git cache...')
szager@chromium.orgdbb6f822016-02-02 22:59:30 +0000552 mirror.populate(
Ryan Tsengc3eb3fa2017-10-09 17:15:42 -0700553 bootstrap=not options.no_bootstrap)
szager@chromium.orgf3145112014-08-07 21:02:36 +0000554 subprocess.check_call([Mirror.git_exe, 'fetch', remote])
555 return 0
556
557
agable@chromium.org5a306a22014-02-24 22:13:59 +0000558class OptionParser(optparse.OptionParser):
559 """Wrapper class for OptionParser to handle global options."""
560
561 def __init__(self, *args, **kwargs):
562 optparse.OptionParser.__init__(self, *args, prog='git cache', **kwargs)
563 self.add_option('-c', '--cache-dir',
564 help='Path to the directory containing the cache')
szager@chromium.org2c391af2014-05-23 09:07:15 +0000565 self.add_option('-v', '--verbose', action='count', default=1,
agable@chromium.org5a306a22014-02-24 22:13:59 +0000566 help='Increase verbosity (can be passed multiple times)')
szager@chromium.org2c391af2014-05-23 09:07:15 +0000567 self.add_option('-q', '--quiet', action='store_true',
568 help='Suppress all extraneous output')
agable@chromium.org5a306a22014-02-24 22:13:59 +0000569
570 def parse_args(self, args=None, values=None):
571 options, args = optparse.OptionParser.parse_args(self, args, values)
szager@chromium.org2c391af2014-05-23 09:07:15 +0000572 if options.quiet:
573 options.verbose = 0
574
575 levels = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG]
576 logging.basicConfig(level=levels[min(options.verbose, len(levels) - 1)])
agable@chromium.org5a306a22014-02-24 22:13:59 +0000577
578 try:
szager@chromium.org848fd492014-04-09 19:06:44 +0000579 global_cache_dir = Mirror.GetCachePath()
580 except RuntimeError:
581 global_cache_dir = None
582 if options.cache_dir:
583 if global_cache_dir and (
584 os.path.abspath(options.cache_dir) !=
585 os.path.abspath(global_cache_dir)):
586 logging.warn('Overriding globally-configured cache directory.')
587 Mirror.SetCachePath(options.cache_dir)
agable@chromium.org5a306a22014-02-24 22:13:59 +0000588
agable@chromium.org5a306a22014-02-24 22:13:59 +0000589 return options, args
590
591
592def main(argv):
593 dispatcher = subcommand.CommandDispatcher(__name__)
594 return dispatcher.execute(OptionParser(), argv)
595
596
597if __name__ == '__main__':
sbc@chromium.org013731e2015-02-26 18:28:43 +0000598 try:
599 sys.exit(main(sys.argv[1:]))
600 except KeyboardInterrupt:
601 sys.stderr.write('interrupted\n')
602 sys.exit(1)