blob: 9f835b5ffc144d8c6753c70585c09cea2250002b [file] [log] [blame]
agable@chromium.org5a306a22014-02-24 22:13:59 +00001#!/usr/bin/env python
2# Copyright 2014 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""A git command for managing a local cache of git repositories."""
7
szager@chromium.org848fd492014-04-09 19:06:44 +00008from __future__ import print_function
agable@chromium.org5a306a22014-02-24 22:13:59 +00009import errno
10import logging
11import optparse
12import os
szager@chromium.org174766f2014-05-13 21:27:46 +000013import re
agable@chromium.org5a306a22014-02-24 22:13:59 +000014import tempfile
szager@chromium.org1132f5f2014-08-23 01:57:59 +000015import threading
pgervais@chromium.orgf3726102014-04-17 17:24:15 +000016import time
agable@chromium.org5a306a22014-02-24 22:13:59 +000017import subprocess
18import sys
19import urlparse
hinoka@google.com776a2c32014-04-25 07:54:25 +000020import zipfile
agable@chromium.org5a306a22014-02-24 22:13:59 +000021
hinoka@google.com563559c2014-04-02 00:36:24 +000022from download_from_google_storage import Gsutil
agable@chromium.org5a306a22014-02-24 22:13:59 +000023import gclient_utils
24import subcommand
25
szager@chromium.org301a7c32014-06-16 17:13:50 +000026# Analogous to gc.autopacklimit git config.
27GC_AUTOPACKLIMIT = 50
28
hinoka@chromium.orgaa1e1a42014-06-26 21:58:51 +000029GIT_CACHE_CORRUPT_MESSAGE = 'WARNING: The Git cache is corrupt.'
30
szager@chromium.org848fd492014-04-09 19:06:44 +000031try:
32 # pylint: disable=E0602
33 WinErr = WindowsError
34except NameError:
35 class WinErr(Exception):
36 pass
agable@chromium.org5a306a22014-02-24 22:13:59 +000037
38class LockError(Exception):
39 pass
40
hinoka@chromium.orgaa1e1a42014-06-26 21:58:51 +000041class RefsHeadsFailedToFetch(Exception):
42 pass
agable@chromium.org5a306a22014-02-24 22:13:59 +000043
44class Lockfile(object):
45 """Class to represent a cross-platform process-specific lockfile."""
46
47 def __init__(self, path):
48 self.path = os.path.abspath(path)
49 self.lockfile = self.path + ".lock"
50 self.pid = os.getpid()
51
52 def _read_pid(self):
53 """Read the pid stored in the lockfile.
54
55 Note: This method is potentially racy. By the time it returns the lockfile
56 may have been unlocked, removed, or stolen by some other process.
57 """
58 try:
59 with open(self.lockfile, 'r') as f:
60 pid = int(f.readline().strip())
61 except (IOError, ValueError):
62 pid = None
63 return pid
64
65 def _make_lockfile(self):
66 """Safely creates a lockfile containing the current pid."""
67 open_flags = (os.O_CREAT | os.O_EXCL | os.O_WRONLY)
68 fd = os.open(self.lockfile, open_flags, 0o644)
69 f = os.fdopen(fd, 'w')
szager@chromium.org848fd492014-04-09 19:06:44 +000070 print(self.pid, file=f)
agable@chromium.org5a306a22014-02-24 22:13:59 +000071 f.close()
72
73 def _remove_lockfile(self):
pgervais@chromium.orgf3726102014-04-17 17:24:15 +000074 """Delete the lockfile. Complains (implicitly) if it doesn't exist.
75
76 See gclient_utils.py:rmtree docstring for more explanation on the
77 windows case.
78 """
79 if sys.platform == 'win32':
80 lockfile = os.path.normcase(self.lockfile)
81 for _ in xrange(3):
82 exitcode = subprocess.call(['cmd.exe', '/c',
83 'del', '/f', '/q', lockfile])
84 if exitcode == 0:
85 return
86 time.sleep(3)
87 raise LockError('Failed to remove lock: %s' % lockfile)
88 else:
89 os.remove(self.lockfile)
agable@chromium.org5a306a22014-02-24 22:13:59 +000090
91 def lock(self):
92 """Acquire the lock.
93
94 Note: This is a NON-BLOCKING FAIL-FAST operation.
95 Do. Or do not. There is no try.
96 """
97 try:
98 self._make_lockfile()
99 except OSError as e:
100 if e.errno == errno.EEXIST:
101 raise LockError("%s is already locked" % self.path)
102 else:
103 raise LockError("Failed to create %s (err %s)" % (self.path, e.errno))
104
105 def unlock(self):
106 """Release the lock."""
szager@chromium.org108eced2014-06-19 21:22:43 +0000107 try:
108 if not self.is_locked():
109 raise LockError("%s is not locked" % self.path)
110 if not self.i_am_locking():
111 raise LockError("%s is locked, but not by me" % self.path)
112 self._remove_lockfile()
113 except WinErr:
114 # Windows is unreliable when it comes to file locking. YMMV.
115 pass
agable@chromium.org5a306a22014-02-24 22:13:59 +0000116
117 def break_lock(self):
118 """Remove the lock, even if it was created by someone else."""
119 try:
120 self._remove_lockfile()
121 return True
122 except OSError as exc:
123 if exc.errno == errno.ENOENT:
124 return False
125 else:
126 raise
127
128 def is_locked(self):
129 """Test if the file is locked by anyone.
130
131 Note: This method is potentially racy. By the time it returns the lockfile
132 may have been unlocked, removed, or stolen by some other process.
133 """
134 return os.path.exists(self.lockfile)
135
136 def i_am_locking(self):
137 """Test if the file is locked by this process."""
138 return self.is_locked() and self.pid == self._read_pid()
139
agable@chromium.org5a306a22014-02-24 22:13:59 +0000140
szager@chromium.org848fd492014-04-09 19:06:44 +0000141class Mirror(object):
142
143 git_exe = 'git.bat' if sys.platform.startswith('win') else 'git'
144 gsutil_exe = os.path.join(
hinoka@chromium.orgb091aa52014-12-20 01:47:31 +0000145 os.path.dirname(os.path.abspath(__file__)), 'gsutil.py')
szager@chromium.org1132f5f2014-08-23 01:57:59 +0000146 cachepath_lock = threading.Lock()
szager@chromium.org848fd492014-04-09 19:06:44 +0000147
szager@chromium.org66c8b852015-09-22 23:19:07 +0000148 @staticmethod
149 def parse_fetch_spec(spec):
150 """Parses and canonicalizes a fetch spec.
151
152 Returns (fetchspec, value_regex), where value_regex can be used
153 with 'git config --replace-all'.
154 """
155 parts = spec.split(':', 1)
156 src = parts[0].lstrip('+').rstrip('/')
157 if not src.startswith('refs/'):
158 src = 'refs/heads/%s' % src
159 dest = parts[1].rstrip('/') if len(parts) > 1 else src
160 regex = r'\+%s:.*' % src.replace('*', r'\*')
161 return ('+%s:%s' % (src, dest), regex)
162
szager@chromium.org848fd492014-04-09 19:06:44 +0000163 def __init__(self, url, refs=None, print_func=None):
164 self.url = url
szager@chromium.org66c8b852015-09-22 23:19:07 +0000165 self.fetch_specs = set([self.parse_fetch_spec(ref) for ref in (refs or [])])
szager@chromium.org848fd492014-04-09 19:06:44 +0000166 self.basedir = self.UrlToCacheDir(url)
167 self.mirror_path = os.path.join(self.GetCachePath(), self.basedir)
loislo@chromium.org0fb693f2014-12-25 15:28:22 +0000168 if print_func:
169 self.print = self.print_without_file
170 self.print_func = print_func
171 else:
172 self.print = print
173
174 def print_without_file(self, message, **kwargs):
175 self.print_func(message)
szager@chromium.org848fd492014-04-09 19:06:44 +0000176
hinoka@chromium.orgf8fa23d2014-06-05 01:00:04 +0000177 @property
178 def bootstrap_bucket(self):
179 if 'chrome-internal' in self.url:
180 return 'chrome-git-cache'
181 else:
182 return 'chromium-git-cache'
183
szager@chromium.org174766f2014-05-13 21:27:46 +0000184 @classmethod
185 def FromPath(cls, path):
186 return cls(cls.CacheDirToUrl(path))
187
szager@chromium.org848fd492014-04-09 19:06:44 +0000188 @staticmethod
189 def UrlToCacheDir(url):
190 """Convert a git url to a normalized form for the cache dir path."""
191 parsed = urlparse.urlparse(url)
192 norm_url = parsed.netloc + parsed.path
193 if norm_url.endswith('.git'):
194 norm_url = norm_url[:-len('.git')]
195 return norm_url.replace('-', '--').replace('/', '-').lower()
196
197 @staticmethod
szager@chromium.org174766f2014-05-13 21:27:46 +0000198 def CacheDirToUrl(path):
199 """Convert a cache dir path to its corresponding url."""
200 netpath = re.sub(r'\b-\b', '/', os.path.basename(path)).replace('--', '-')
201 return 'https://%s' % netpath
202
szager@chromium.org848fd492014-04-09 19:06:44 +0000203 @classmethod
204 def SetCachePath(cls, cachepath):
szager@chromium.org84c56002014-08-23 03:33:28 +0000205 with cls.cachepath_lock:
206 setattr(cls, 'cachepath', cachepath)
szager@chromium.org848fd492014-04-09 19:06:44 +0000207
208 @classmethod
209 def GetCachePath(cls):
szager@chromium.org84c56002014-08-23 03:33:28 +0000210 with cls.cachepath_lock:
211 if not hasattr(cls, 'cachepath'):
212 try:
213 cachepath = subprocess.check_output(
214 [cls.git_exe, 'config', '--global', 'cache.cachepath']).strip()
215 except subprocess.CalledProcessError:
216 cachepath = None
217 if not cachepath:
218 raise RuntimeError(
219 'No global cache.cachepath git configuration found.')
220 setattr(cls, 'cachepath', cachepath)
221 return getattr(cls, 'cachepath')
szager@chromium.org848fd492014-04-09 19:06:44 +0000222
223 def RunGit(self, cmd, **kwargs):
224 """Run git in a subprocess."""
225 cwd = kwargs.setdefault('cwd', self.mirror_path)
226 kwargs.setdefault('print_stdout', False)
227 kwargs.setdefault('filter_fn', self.print)
228 env = kwargs.get('env') or kwargs.setdefault('env', os.environ.copy())
229 env.setdefault('GIT_ASKPASS', 'true')
230 env.setdefault('SSH_ASKPASS', 'true')
231 self.print('running "git %s" in "%s"' % (' '.join(cmd), cwd))
232 gclient_utils.CheckCallAndFilter([self.git_exe] + cmd, **kwargs)
233
234 def config(self, cwd=None):
235 if cwd is None:
236 cwd = self.mirror_path
szager@chromium.org301a7c32014-06-16 17:13:50 +0000237
238 # Don't run git-gc in a daemon. Bad things can happen if it gets killed.
hinoka@chromium.org8e095af2015-06-10 19:19:07 +0000239 self.RunGit(['config', 'gc.autodetach', '0'], cwd=cwd)
szager@chromium.org301a7c32014-06-16 17:13:50 +0000240
241 # Don't combine pack files into one big pack file. It's really slow for
242 # repositories, and there's no way to track progress and make sure it's
243 # not stuck.
hinoka@chromium.org8e095af2015-06-10 19:19:07 +0000244 self.RunGit(['config', 'gc.autopacklimit', '0'], cwd=cwd)
szager@chromium.org301a7c32014-06-16 17:13:50 +0000245
246 # Allocate more RAM for cache-ing delta chains, for better performance
247 # of "Resolving deltas".
szager@chromium.org848fd492014-04-09 19:06:44 +0000248 self.RunGit(['config', 'core.deltaBaseCacheLimit',
hinoka@chromium.org8e095af2015-06-10 19:19:07 +0000249 gclient_utils.DefaultDeltaBaseCacheLimit()], cwd=cwd)
szager@chromium.org301a7c32014-06-16 17:13:50 +0000250
hinoka@chromium.org8e095af2015-06-10 19:19:07 +0000251 self.RunGit(['config', 'remote.origin.url', self.url], cwd=cwd)
szager@chromium.org848fd492014-04-09 19:06:44 +0000252 self.RunGit(['config', '--replace-all', 'remote.origin.fetch',
hinoka@chromium.org8e095af2015-06-10 19:19:07 +0000253 '+refs/heads/*:refs/heads/*', r'\+refs/heads/\*:.*'], cwd=cwd)
szager@chromium.org66c8b852015-09-22 23:19:07 +0000254 for spec, value_regex in self.fetch_specs:
szager@chromium.org965c44f2014-08-19 21:19:19 +0000255 self.RunGit(
szager@chromium.org66c8b852015-09-22 23:19:07 +0000256 ['config', '--replace-all', 'remote.origin.fetch', spec, value_regex],
hinoka@chromium.org8e095af2015-06-10 19:19:07 +0000257 cwd=cwd)
szager@chromium.org848fd492014-04-09 19:06:44 +0000258
259 def bootstrap_repo(self, directory):
hinoka@chromium.orgaa1e1a42014-06-26 21:58:51 +0000260 """Bootstrap the repo from Google Stroage if possible.
261
262 More apt-ly named bootstrap_repo_from_cloud_if_possible_else_do_nothing().
263 """
szager@chromium.org848fd492014-04-09 19:06:44 +0000264
hinoka@google.com776a2c32014-04-25 07:54:25 +0000265 python_fallback = False
sbc@chromium.org9d0644d2015-06-05 23:16:54 +0000266 if (sys.platform.startswith('win') and
267 not gclient_utils.FindExecutable('7z')):
hinoka@google.com776a2c32014-04-25 07:54:25 +0000268 python_fallback = True
269 elif sys.platform.startswith('darwin'):
270 # The OSX version of unzip doesn't support zip64.
271 python_fallback = True
sbc@chromium.org9d0644d2015-06-05 23:16:54 +0000272 elif not gclient_utils.FindExecutable('unzip'):
hinoka@google.com776a2c32014-04-25 07:54:25 +0000273 python_fallback = True
szager@chromium.org848fd492014-04-09 19:06:44 +0000274
275 gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir)
hinoka@chromium.org199bc5f2014-12-17 02:17:14 +0000276 gsutil = Gsutil(self.gsutil_exe, boto_path=None)
szager@chromium.org848fd492014-04-09 19:06:44 +0000277 # Get the most recent version of the zipfile.
278 _, ls_out, _ = gsutil.check_call('ls', gs_folder)
279 ls_out_sorted = sorted(ls_out.splitlines())
280 if not ls_out_sorted:
281 # This repo is not on Google Storage.
282 return False
283 latest_checkout = ls_out_sorted[-1]
284
285 # Download zip file to a temporary directory.
286 try:
szager@chromium.org1cbf1042014-06-17 18:26:24 +0000287 tempdir = tempfile.mkdtemp(prefix='_cache_tmp', dir=self.GetCachePath())
szager@chromium.org848fd492014-04-09 19:06:44 +0000288 self.print('Downloading %s' % latest_checkout)
hinoka@chromium.orgc58d11d2014-06-09 23:34:35 +0000289 code = gsutil.call('cp', latest_checkout, tempdir)
szager@chromium.org848fd492014-04-09 19:06:44 +0000290 if code:
szager@chromium.org848fd492014-04-09 19:06:44 +0000291 return False
292 filename = os.path.join(tempdir, latest_checkout.split('/')[-1])
293
hinoka@google.com776a2c32014-04-25 07:54:25 +0000294 # Unpack the file with 7z on Windows, unzip on linux, or fallback.
295 if not python_fallback:
296 if sys.platform.startswith('win'):
297 cmd = ['7z', 'x', '-o%s' % directory, '-tzip', filename]
298 else:
299 cmd = ['unzip', filename, '-d', directory]
300 retcode = subprocess.call(cmd)
szager@chromium.org848fd492014-04-09 19:06:44 +0000301 else:
hinoka@google.com776a2c32014-04-25 07:54:25 +0000302 try:
303 with zipfile.ZipFile(filename, 'r') as f:
304 f.printdir()
305 f.extractall(directory)
306 except Exception as e:
307 self.print('Encountered error: %s' % str(e), file=sys.stderr)
308 retcode = 1
309 else:
310 retcode = 0
szager@chromium.org848fd492014-04-09 19:06:44 +0000311 finally:
312 # Clean up the downloaded zipfile.
loislo@chromium.org67b59e92014-12-25 13:48:37 +0000313 gclient_utils.rm_file_or_tree(tempdir)
szager@chromium.org848fd492014-04-09 19:06:44 +0000314
315 if retcode:
316 self.print(
317 'Extracting bootstrap zipfile %s failed.\n'
318 'Resuming normal operations.' % filename)
319 return False
320 return True
321
322 def exists(self):
323 return os.path.isfile(os.path.join(self.mirror_path, 'config'))
324
szager@chromium.org66c8b852015-09-22 23:19:07 +0000325 def _preserve_fetchspec(self):
326 """Read and preserve remote.origin.fetch from an existing mirror.
327
328 This modifies self.fetch_specs.
329 """
330 if not self.exists():
331 return
332 try:
333 config_fetchspecs = subprocess.check_output(
334 [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'],
335 cwd=self.mirror_path)
336 for fetchspec in config_fetchspecs.splitlines():
337 self.fetch_specs.add(self.parse_fetch_spec(fetchspec))
338 except subprocess.CalledProcessError:
339 logging.warn('Tried and failed to preserve remote.origin.fetch from the '
340 'existing cache directory. You may need to manually edit '
341 '%s and "git cache fetch" again.'
342 % os.path.join(self.mirror_path, 'config'))
343
hinoka@chromium.orgaa1e1a42014-06-26 21:58:51 +0000344 def _ensure_bootstrapped(self, depth, bootstrap, force=False):
345 tempdir = None
hinoka@chromium.orgaa1e1a42014-06-26 21:58:51 +0000346 pack_dir = os.path.join(self.mirror_path, 'objects', 'pack')
347 pack_files = []
348
349 if os.path.isdir(pack_dir):
350 pack_files = [f for f in os.listdir(pack_dir) if f.endswith('.pack')]
351
352 should_bootstrap = (force or
szager@chromium.org66c8b852015-09-22 23:19:07 +0000353 not self.exists() or
hinoka@chromium.orgaa1e1a42014-06-26 21:58:51 +0000354 len(pack_files) > GC_AUTOPACKLIMIT)
355 if should_bootstrap:
szager@chromium.org66c8b852015-09-22 23:19:07 +0000356 if self.exists():
357 # Re-bootstrapping an existing mirror; preserve existing fetch spec.
358 self._preserve_fetchspec()
hinoka@chromium.orgaa1e1a42014-06-26 21:58:51 +0000359 tempdir = tempfile.mkdtemp(
360 prefix='_cache_tmp', suffix=self.basedir, dir=self.GetCachePath())
361 bootstrapped = not depth and bootstrap and self.bootstrap_repo(tempdir)
362 if bootstrapped:
363 # Bootstrap succeeded; delete previous cache, if any.
hinoka@chromium.org42f9adf2014-09-05 11:10:35 +0000364 gclient_utils.rmtree(self.mirror_path)
szager@chromium.org66c8b852015-09-22 23:19:07 +0000365 elif not self.exists():
hinoka@chromium.orgaa1e1a42014-06-26 21:58:51 +0000366 # Bootstrap failed, no previous cache; start with a bare git dir.
367 self.RunGit(['init', '--bare'], cwd=tempdir)
368 else:
369 # Bootstrap failed, previous cache exists; warn and continue.
370 logging.warn(
371 'Git cache has a lot of pack files (%d). Tried to re-bootstrap '
372 'but failed. Continuing with non-optimized repository.'
373 % len(pack_files))
374 gclient_utils.rmtree(tempdir)
375 tempdir = None
376 else:
377 if depth and os.path.exists(os.path.join(self.mirror_path, 'shallow')):
378 logging.warn(
379 'Shallow fetch requested, but repo cache already exists.')
380 return tempdir
381
382 def _fetch(self, rundir, verbose, depth):
383 self.config(rundir)
384 v = []
385 d = []
386 if verbose:
387 v = ['-v', '--progress']
388 if depth:
389 d = ['--depth', str(depth)]
390 fetch_cmd = ['fetch'] + v + d + ['origin']
391 fetch_specs = subprocess.check_output(
392 [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'],
393 cwd=rundir).strip().splitlines()
394 for spec in fetch_specs:
395 try:
396 self.print('Fetching %s' % spec)
397 self.RunGit(fetch_cmd + [spec], cwd=rundir, retry=True)
398 except subprocess.CalledProcessError:
399 if spec == '+refs/heads/*:refs/heads/*':
400 raise RefsHeadsFailedToFetch
401 logging.warn('Fetch of %s failed' % spec)
402
szager@chromium.org848fd492014-04-09 19:06:44 +0000403 def populate(self, depth=None, shallow=False, bootstrap=False,
szager@chromium.org108eced2014-06-19 21:22:43 +0000404 verbose=False, ignore_lock=False):
szager@chromium.orgb0a13a22014-06-18 00:52:25 +0000405 assert self.GetCachePath()
szager@chromium.org848fd492014-04-09 19:06:44 +0000406 if shallow and not depth:
407 depth = 10000
408 gclient_utils.safe_makedirs(self.GetCachePath())
409
szager@chromium.org108eced2014-06-19 21:22:43 +0000410 lockfile = Lockfile(self.mirror_path)
411 if not ignore_lock:
412 lockfile.lock()
413
hinoka@chromium.orgaa1e1a42014-06-26 21:58:51 +0000414 tempdir = None
szager@chromium.org108eced2014-06-19 21:22:43 +0000415 try:
hinoka@chromium.orgaa1e1a42014-06-26 21:58:51 +0000416 tempdir = self._ensure_bootstrapped(depth, bootstrap)
szager@chromium.org848fd492014-04-09 19:06:44 +0000417 rundir = tempdir or self.mirror_path
hinoka@chromium.orgaa1e1a42014-06-26 21:58:51 +0000418 self._fetch(rundir, verbose, depth)
419 except RefsHeadsFailedToFetch:
420 # This is a major failure, we need to clean and force a bootstrap.
421 gclient_utils.rmtree(rundir)
422 self.print(GIT_CACHE_CORRUPT_MESSAGE)
423 tempdir = self._ensure_bootstrapped(depth, bootstrap, force=True)
424 assert tempdir
425 self._fetch(tempdir or self.mirror_path, verbose, depth)
426 finally:
szager@chromium.org848fd492014-04-09 19:06:44 +0000427 if tempdir:
hinoka@chromium.org4e2ad842014-07-19 01:23:45 +0000428 try:
hinoka@chromium.orga5cda1e2014-08-24 12:09:13 +0000429 if os.path.exists(self.mirror_path):
430 gclient_utils.rmtree(self.mirror_path)
hinoka@chromium.org4e2ad842014-07-19 01:23:45 +0000431 os.rename(tempdir, self.mirror_path)
432 except OSError as e:
433 # This is somehow racy on Windows.
434 # Catching OSError because WindowsError isn't portable and
435 # pylint complains.
436 self.print('Error moving %s to %s: %s' % (tempdir, self.mirror_path,
437 str(e)))
szager@chromium.org108eced2014-06-19 21:22:43 +0000438 if not ignore_lock:
439 lockfile.unlock()
szager@chromium.org848fd492014-04-09 19:06:44 +0000440
hinoka@chromium.orgc8444f32014-06-18 23:18:17 +0000441 def update_bootstrap(self, prune=False):
szager@chromium.org848fd492014-04-09 19:06:44 +0000442 # The files are named <git number>.zip
443 gen_number = subprocess.check_output(
444 [self.git_exe, 'number', 'master'], cwd=self.mirror_path).strip()
hinoka@chromium.org7b1cb6f2014-09-08 21:40:50 +0000445 # Run Garbage Collect to compress packfile.
446 self.RunGit(['gc', '--prune=all'])
szager@chromium.org848fd492014-04-09 19:06:44 +0000447 # Creating a temp file and then deleting it ensures we can use this name.
448 _, tmp_zipfile = tempfile.mkstemp(suffix='.zip')
449 os.remove(tmp_zipfile)
450 subprocess.call(['zip', '-r', tmp_zipfile, '.'], cwd=self.mirror_path)
451 gsutil = Gsutil(path=self.gsutil_exe, boto_path=None)
hinoka@chromium.orgc8444f32014-06-18 23:18:17 +0000452 gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir)
453 dest_name = '%s/%s.zip' % (gs_folder, gen_number)
szager@chromium.org848fd492014-04-09 19:06:44 +0000454 gsutil.call('cp', tmp_zipfile, dest_name)
455 os.remove(tmp_zipfile)
456
hinoka@chromium.orgc8444f32014-06-18 23:18:17 +0000457 # Remove all other files in the same directory.
458 if prune:
459 _, ls_out, _ = gsutil.check_call('ls', gs_folder)
460 for filename in ls_out.splitlines():
461 if filename == dest_name:
462 continue
463 gsutil.call('rm', filename)
464
szager@chromium.orgcdfcd7c2014-06-10 23:40:46 +0000465 @staticmethod
466 def DeleteTmpPackFiles(path):
467 pack_dir = os.path.join(path, 'objects', 'pack')
szager@chromium.org33418492014-06-18 19:03:39 +0000468 if not os.path.isdir(pack_dir):
469 return
szager@chromium.orgcdfcd7c2014-06-10 23:40:46 +0000470 pack_files = [f for f in os.listdir(pack_dir) if
471 f.startswith('.tmp-') or f.startswith('tmp_pack_')]
472 for f in pack_files:
473 f = os.path.join(pack_dir, f)
474 try:
475 os.remove(f)
476 logging.warn('Deleted stale temporary pack file %s' % f)
477 except OSError:
478 logging.warn('Unable to delete temporary pack file %s' % f)
szager@chromium.org174766f2014-05-13 21:27:46 +0000479
szager@chromium.org1cbf1042014-06-17 18:26:24 +0000480 @classmethod
481 def BreakLocks(cls, path):
szager@chromium.org174766f2014-05-13 21:27:46 +0000482 did_unlock = False
483 lf = Lockfile(path)
484 if lf.break_lock():
485 did_unlock = True
486 # Look for lock files that might have been left behind by an interrupted
487 # git process.
488 lf = os.path.join(path, 'config.lock')
489 if os.path.exists(lf):
490 os.remove(lf)
491 did_unlock = True
szager@chromium.org1cbf1042014-06-17 18:26:24 +0000492 cls.DeleteTmpPackFiles(path)
szager@chromium.org174766f2014-05-13 21:27:46 +0000493 return did_unlock
494
szager@chromium.org848fd492014-04-09 19:06:44 +0000495 def unlock(self):
szager@chromium.org174766f2014-05-13 21:27:46 +0000496 return self.BreakLocks(self.mirror_path)
497
498 @classmethod
499 def UnlockAll(cls):
500 cachepath = cls.GetCachePath()
szager@chromium.orgb0a13a22014-06-18 00:52:25 +0000501 if not cachepath:
502 return
szager@chromium.org174766f2014-05-13 21:27:46 +0000503 dirlist = os.listdir(cachepath)
504 repo_dirs = set([os.path.join(cachepath, path) for path in dirlist
505 if os.path.isdir(os.path.join(cachepath, path))])
506 for dirent in dirlist:
szager@chromium.org1cbf1042014-06-17 18:26:24 +0000507 if dirent.startswith('_cache_tmp') or dirent.startswith('tmp'):
loislo@chromium.org67b59e92014-12-25 13:48:37 +0000508 gclient_utils.rm_file_or_tree(os.path.join(cachepath, dirent))
szager@chromium.org1cbf1042014-06-17 18:26:24 +0000509 elif (dirent.endswith('.lock') and
szager@chromium.org174766f2014-05-13 21:27:46 +0000510 os.path.isfile(os.path.join(cachepath, dirent))):
511 repo_dirs.add(os.path.join(cachepath, dirent[:-5]))
512
513 unlocked_repos = []
514 for repo_dir in repo_dirs:
515 if cls.BreakLocks(repo_dir):
516 unlocked_repos.append(repo_dir)
517
518 return unlocked_repos
szager@chromium.org848fd492014-04-09 19:06:44 +0000519
agable@chromium.org5a306a22014-02-24 22:13:59 +0000520@subcommand.usage('[url of repo to check for caching]')
521def CMDexists(parser, args):
522 """Check to see if there already is a cache of the given repo."""
szager@chromium.org848fd492014-04-09 19:06:44 +0000523 _, args = parser.parse_args(args)
agable@chromium.org5a306a22014-02-24 22:13:59 +0000524 if not len(args) == 1:
525 parser.error('git cache exists only takes exactly one repo url.')
526 url = args[0]
szager@chromium.org848fd492014-04-09 19:06:44 +0000527 mirror = Mirror(url)
528 if mirror.exists():
529 print(mirror.mirror_path)
agable@chromium.org5a306a22014-02-24 22:13:59 +0000530 return 0
531 return 1
532
533
hinoka@google.com563559c2014-04-02 00:36:24 +0000534@subcommand.usage('[url of repo to create a bootstrap zip file]')
535def CMDupdate_bootstrap(parser, args):
536 """Create and uploads a bootstrap tarball."""
537 # Lets just assert we can't do this on Windows.
538 if sys.platform.startswith('win'):
szager@chromium.org848fd492014-04-09 19:06:44 +0000539 print('Sorry, update bootstrap will not work on Windows.', file=sys.stderr)
hinoka@google.com563559c2014-04-02 00:36:24 +0000540 return 1
541
hinoka@chromium.orgc8444f32014-06-18 23:18:17 +0000542 parser.add_option('--prune', action='store_true',
543 help='Prune all other cached zipballs of the same repo.')
544
hinoka@google.com563559c2014-04-02 00:36:24 +0000545 # First, we need to ensure the cache is populated.
546 populate_args = args[:]
pgervais@chromium.orgb9f27512014-08-08 15:52:33 +0000547 populate_args.append('--no-bootstrap')
hinoka@google.com563559c2014-04-02 00:36:24 +0000548 CMDpopulate(parser, populate_args)
549
550 # Get the repo directory.
hinoka@chromium.orgc8444f32014-06-18 23:18:17 +0000551 options, args = parser.parse_args(args)
hinoka@google.com563559c2014-04-02 00:36:24 +0000552 url = args[0]
szager@chromium.org848fd492014-04-09 19:06:44 +0000553 mirror = Mirror(url)
hinoka@chromium.orgc8444f32014-06-18 23:18:17 +0000554 mirror.update_bootstrap(options.prune)
szager@chromium.org848fd492014-04-09 19:06:44 +0000555 return 0
hinoka@google.com563559c2014-04-02 00:36:24 +0000556
557
agable@chromium.org5a306a22014-02-24 22:13:59 +0000558@subcommand.usage('[url of repo to add to or update in cache]')
559def CMDpopulate(parser, args):
560 """Ensure that the cache has all up-to-date objects for the given repo."""
561 parser.add_option('--depth', type='int',
562 help='Only cache DEPTH commits of history')
563 parser.add_option('--shallow', '-s', action='store_true',
564 help='Only cache 10000 commits of history')
565 parser.add_option('--ref', action='append',
566 help='Specify additional refs to be fetched')
pgervais@chromium.orgb9f27512014-08-08 15:52:33 +0000567 parser.add_option('--no_bootstrap', '--no-bootstrap',
568 action='store_true',
hinoka@google.com563559c2014-04-02 00:36:24 +0000569 help='Don\'t bootstrap from Google Storage')
pgervais@chromium.orgb9f27512014-08-08 15:52:33 +0000570 parser.add_option('--ignore_locks', '--ignore-locks',
571 action='store_true',
szager@chromium.org108eced2014-06-19 21:22:43 +0000572 help='Don\'t try to lock repository')
hinoka@google.com563559c2014-04-02 00:36:24 +0000573
agable@chromium.org5a306a22014-02-24 22:13:59 +0000574 options, args = parser.parse_args(args)
agable@chromium.org5a306a22014-02-24 22:13:59 +0000575 if not len(args) == 1:
576 parser.error('git cache populate only takes exactly one repo url.')
577 url = args[0]
578
szager@chromium.org848fd492014-04-09 19:06:44 +0000579 mirror = Mirror(url, refs=options.ref)
580 kwargs = {
581 'verbose': options.verbose,
582 'shallow': options.shallow,
583 'bootstrap': not options.no_bootstrap,
szager@chromium.org108eced2014-06-19 21:22:43 +0000584 'ignore_lock': options.ignore_locks,
szager@chromium.org848fd492014-04-09 19:06:44 +0000585 }
agable@chromium.org5a306a22014-02-24 22:13:59 +0000586 if options.depth:
szager@chromium.org848fd492014-04-09 19:06:44 +0000587 kwargs['depth'] = options.depth
588 mirror.populate(**kwargs)
agable@chromium.org5a306a22014-02-24 22:13:59 +0000589
590
szager@chromium.orgf3145112014-08-07 21:02:36 +0000591@subcommand.usage('Fetch new commits into cache and current checkout')
592def CMDfetch(parser, args):
593 """Update mirror, and fetch in cwd."""
594 parser.add_option('--all', action='store_true', help='Fetch all remotes')
szager@chromium.org66c8b852015-09-22 23:19:07 +0000595 parser.add_option('--no_bootstrap', '--no-bootstrap',
596 action='store_true',
597 help='Don\'t (re)bootstrap from Google Storage')
szager@chromium.orgf3145112014-08-07 21:02:36 +0000598 options, args = parser.parse_args(args)
599
600 # Figure out which remotes to fetch. This mimics the behavior of regular
601 # 'git fetch'. Note that in the case of "stacked" or "pipelined" branches,
602 # this will NOT try to traverse up the branching structure to find the
603 # ultimate remote to update.
604 remotes = []
605 if options.all:
606 assert not args, 'fatal: fetch --all does not take a repository argument'
607 remotes = subprocess.check_output([Mirror.git_exe, 'remote']).splitlines()
608 elif args:
609 remotes = args
610 else:
611 current_branch = subprocess.check_output(
612 [Mirror.git_exe, 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
613 if current_branch != 'HEAD':
614 upstream = subprocess.check_output(
615 [Mirror.git_exe, 'config', 'branch.%s.remote' % current_branch]
616 ).strip()
617 if upstream and upstream != '.':
618 remotes = [upstream]
619 if not remotes:
620 remotes = ['origin']
621
622 cachepath = Mirror.GetCachePath()
623 git_dir = os.path.abspath(subprocess.check_output(
624 [Mirror.git_exe, 'rev-parse', '--git-dir']))
625 git_dir = os.path.abspath(git_dir)
626 if git_dir.startswith(cachepath):
627 mirror = Mirror.FromPath(git_dir)
szager@chromium.org66c8b852015-09-22 23:19:07 +0000628 mirror.populate(bootstrap=not options.no_bootstrap)
szager@chromium.orgf3145112014-08-07 21:02:36 +0000629 return 0
630 for remote in remotes:
631 remote_url = subprocess.check_output(
632 [Mirror.git_exe, 'config', 'remote.%s.url' % remote]).strip()
633 if remote_url.startswith(cachepath):
634 mirror = Mirror.FromPath(remote_url)
635 mirror.print = lambda *args: None
636 print('Updating git cache...')
szager@chromium.org66c8b852015-09-22 23:19:07 +0000637 mirror.populate(bootstrap=not options.no_bootstrap)
szager@chromium.orgf3145112014-08-07 21:02:36 +0000638 subprocess.check_call([Mirror.git_exe, 'fetch', remote])
639 return 0
640
641
agable@chromium.org5a306a22014-02-24 22:13:59 +0000642@subcommand.usage('[url of repo to unlock, or -a|--all]')
643def CMDunlock(parser, args):
644 """Unlock one or all repos if their lock files are still around."""
645 parser.add_option('--force', '-f', action='store_true',
646 help='Actually perform the action')
647 parser.add_option('--all', '-a', action='store_true',
648 help='Unlock all repository caches')
649 options, args = parser.parse_args(args)
650 if len(args) > 1 or (len(args) == 0 and not options.all):
651 parser.error('git cache unlock takes exactly one repo url, or --all')
652
agable@chromium.org5a306a22014-02-24 22:13:59 +0000653 if not options.force:
szager@chromium.org174766f2014-05-13 21:27:46 +0000654 cachepath = Mirror.GetCachePath()
655 lockfiles = [os.path.join(cachepath, path)
656 for path in os.listdir(cachepath)
657 if path.endswith('.lock') and os.path.isfile(path)]
agable@chromium.org5a306a22014-02-24 22:13:59 +0000658 parser.error('git cache unlock requires -f|--force to do anything. '
659 'Refusing to unlock the following repo caches: '
660 ', '.join(lockfiles))
661
szager@chromium.org848fd492014-04-09 19:06:44 +0000662 unlocked_repos = []
szager@chromium.org174766f2014-05-13 21:27:46 +0000663 if options.all:
664 unlocked_repos.extend(Mirror.UnlockAll())
665 else:
666 m = Mirror(args[0])
667 if m.unlock():
668 unlocked_repos.append(m.mirror_path)
agable@chromium.org5a306a22014-02-24 22:13:59 +0000669
szager@chromium.org848fd492014-04-09 19:06:44 +0000670 if unlocked_repos:
671 logging.info('Broke locks on these caches:\n %s' % '\n '.join(
672 unlocked_repos))
agable@chromium.org5a306a22014-02-24 22:13:59 +0000673
674
675class OptionParser(optparse.OptionParser):
676 """Wrapper class for OptionParser to handle global options."""
677
678 def __init__(self, *args, **kwargs):
679 optparse.OptionParser.__init__(self, *args, prog='git cache', **kwargs)
680 self.add_option('-c', '--cache-dir',
681 help='Path to the directory containing the cache')
szager@chromium.org2c391af2014-05-23 09:07:15 +0000682 self.add_option('-v', '--verbose', action='count', default=1,
agable@chromium.org5a306a22014-02-24 22:13:59 +0000683 help='Increase verbosity (can be passed multiple times)')
szager@chromium.org2c391af2014-05-23 09:07:15 +0000684 self.add_option('-q', '--quiet', action='store_true',
685 help='Suppress all extraneous output')
agable@chromium.org5a306a22014-02-24 22:13:59 +0000686
687 def parse_args(self, args=None, values=None):
688 options, args = optparse.OptionParser.parse_args(self, args, values)
szager@chromium.org2c391af2014-05-23 09:07:15 +0000689 if options.quiet:
690 options.verbose = 0
691
692 levels = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG]
693 logging.basicConfig(level=levels[min(options.verbose, len(levels) - 1)])
agable@chromium.org5a306a22014-02-24 22:13:59 +0000694
695 try:
szager@chromium.org848fd492014-04-09 19:06:44 +0000696 global_cache_dir = Mirror.GetCachePath()
697 except RuntimeError:
698 global_cache_dir = None
699 if options.cache_dir:
700 if global_cache_dir and (
701 os.path.abspath(options.cache_dir) !=
702 os.path.abspath(global_cache_dir)):
703 logging.warn('Overriding globally-configured cache directory.')
704 Mirror.SetCachePath(options.cache_dir)
agable@chromium.org5a306a22014-02-24 22:13:59 +0000705
agable@chromium.org5a306a22014-02-24 22:13:59 +0000706 return options, args
707
708
709def main(argv):
710 dispatcher = subcommand.CommandDispatcher(__name__)
711 return dispatcher.execute(OptionParser(), argv)
712
713
714if __name__ == '__main__':
sbc@chromium.org013731e2015-02-26 18:28:43 +0000715 try:
716 sys.exit(main(sys.argv[1:]))
717 except KeyboardInterrupt:
718 sys.stderr.write('interrupted\n')
719 sys.exit(1)