agable@chromium.org | 5a306a2 | 2014-02-24 22:13:59 +0000 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # Copyright 2014 The Chromium Authors. All rights reserved. |
| 3 | # Use of this source code is governed by a BSD-style license that can be |
| 4 | # found in the LICENSE file. |
| 5 | |
| 6 | """A git command for managing a local cache of git repositories.""" |
| 7 | |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 8 | from __future__ import print_function |
agable@chromium.org | 5a306a2 | 2014-02-24 22:13:59 +0000 | [diff] [blame] | 9 | import errno |
| 10 | import logging |
| 11 | import optparse |
| 12 | import os |
szager@chromium.org | 174766f | 2014-05-13 21:27:46 +0000 | [diff] [blame] | 13 | import re |
agable@chromium.org | 5a306a2 | 2014-02-24 22:13:59 +0000 | [diff] [blame] | 14 | import tempfile |
szager@chromium.org | 1132f5f | 2014-08-23 01:57:59 +0000 | [diff] [blame] | 15 | import threading |
pgervais@chromium.org | f372610 | 2014-04-17 17:24:15 +0000 | [diff] [blame] | 16 | import time |
agable@chromium.org | 5a306a2 | 2014-02-24 22:13:59 +0000 | [diff] [blame] | 17 | import subprocess |
| 18 | import sys |
| 19 | import urlparse |
hinoka@google.com | 776a2c3 | 2014-04-25 07:54:25 +0000 | [diff] [blame] | 20 | import zipfile |
agable@chromium.org | 5a306a2 | 2014-02-24 22:13:59 +0000 | [diff] [blame] | 21 | |
hinoka@google.com | 563559c | 2014-04-02 00:36:24 +0000 | [diff] [blame] | 22 | from download_from_google_storage import Gsutil |
agable@chromium.org | 5a306a2 | 2014-02-24 22:13:59 +0000 | [diff] [blame] | 23 | import gclient_utils |
| 24 | import subcommand |
| 25 | |
szager@chromium.org | 301a7c3 | 2014-06-16 17:13:50 +0000 | [diff] [blame] | 26 | # Analogous to gc.autopacklimit git config. |
| 27 | GC_AUTOPACKLIMIT = 50 |
| 28 | |
hinoka@chromium.org | aa1e1a4 | 2014-06-26 21:58:51 +0000 | [diff] [blame] | 29 | GIT_CACHE_CORRUPT_MESSAGE = 'WARNING: The Git cache is corrupt.' |
| 30 | |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 31 | try: |
Quinten Yearsley | b2cc4a9 | 2016-12-15 13:53:26 -0800 | [diff] [blame] | 32 | # pylint: disable=undefined-variable |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 33 | WinErr = WindowsError |
| 34 | except NameError: |
| 35 | class WinErr(Exception): |
| 36 | pass |
agable@chromium.org | 5a306a2 | 2014-02-24 22:13:59 +0000 | [diff] [blame] | 37 | |
hinoka | dcd8404 | 2016-06-09 14:26:17 -0700 | [diff] [blame] | 38 | class ClobberNeeded(Exception): |
hinoka@chromium.org | aa1e1a4 | 2014-06-26 21:58:51 +0000 | [diff] [blame] | 39 | pass |
agable@chromium.org | 5a306a2 | 2014-02-24 22:13:59 +0000 | [diff] [blame] | 40 | |
dnj | 4625b5a | 2016-11-10 18:23:26 -0800 | [diff] [blame] | 41 | |
| 42 | def exponential_backoff_retry(fn, excs=(Exception,), name=None, count=10, |
| 43 | sleep_time=0.25, printerr=None): |
| 44 | """Executes |fn| up to |count| times, backing off exponentially. |
| 45 | |
| 46 | Args: |
| 47 | fn (callable): The function to execute. If this raises a handled |
| 48 | exception, the function will retry with exponential backoff. |
| 49 | excs (tuple): A tuple of Exception types to handle. If one of these is |
| 50 | raised by |fn|, a retry will be attempted. If |fn| raises an Exception |
| 51 | that is not in this list, it will immediately pass through. If |excs| |
| 52 | is empty, the Exception base class will be used. |
| 53 | name (str): Optional operation name to print in the retry string. |
| 54 | count (int): The number of times to try before allowing the exception to |
| 55 | pass through. |
| 56 | sleep_time (float): The initial number of seconds to sleep in between |
| 57 | retries. This will be doubled each retry. |
| 58 | printerr (callable): Function that will be called with the error string upon |
| 59 | failures. If None, |logging.warning| will be used. |
| 60 | |
| 61 | Returns: The return value of the successful fn. |
| 62 | """ |
| 63 | printerr = printerr or logging.warning |
| 64 | for i in xrange(count): |
| 65 | try: |
| 66 | return fn() |
| 67 | except excs as e: |
| 68 | if (i+1) >= count: |
| 69 | raise |
| 70 | |
| 71 | printerr('Retrying %s in %.2f second(s) (%d / %d attempts): %s' % ( |
| 72 | (name or 'operation'), sleep_time, (i+1), count, e)) |
| 73 | time.sleep(sleep_time) |
| 74 | sleep_time *= 2 |
| 75 | |
| 76 | |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 77 | class Mirror(object): |
| 78 | |
| 79 | git_exe = 'git.bat' if sys.platform.startswith('win') else 'git' |
| 80 | gsutil_exe = os.path.join( |
hinoka@chromium.org | b091aa5 | 2014-12-20 01:47:31 +0000 | [diff] [blame] | 81 | os.path.dirname(os.path.abspath(__file__)), 'gsutil.py') |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 82 | |
szager@chromium.org | 66c8b85 | 2015-09-22 23:19:07 +0000 | [diff] [blame] | 83 | @staticmethod |
| 84 | def parse_fetch_spec(spec): |
| 85 | """Parses and canonicalizes a fetch spec. |
| 86 | |
| 87 | Returns (fetchspec, value_regex), where value_regex can be used |
| 88 | with 'git config --replace-all'. |
| 89 | """ |
| 90 | parts = spec.split(':', 1) |
| 91 | src = parts[0].lstrip('+').rstrip('/') |
| 92 | if not src.startswith('refs/'): |
| 93 | src = 'refs/heads/%s' % src |
| 94 | dest = parts[1].rstrip('/') if len(parts) > 1 else src |
| 95 | regex = r'\+%s:.*' % src.replace('*', r'\*') |
| 96 | return ('+%s:%s' % (src, dest), regex) |
| 97 | |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 98 | def __init__(self, url, refs=None, print_func=None): |
| 99 | self.url = url |
szager@chromium.org | 66c8b85 | 2015-09-22 23:19:07 +0000 | [diff] [blame] | 100 | self.fetch_specs = set([self.parse_fetch_spec(ref) for ref in (refs or [])]) |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 101 | self.basedir = self.UrlToCacheDir(url) |
| 102 | self.mirror_path = os.path.join(self.GetCachePath(), self.basedir) |
loislo@chromium.org | 0fb693f | 2014-12-25 15:28:22 +0000 | [diff] [blame] | 103 | if print_func: |
| 104 | self.print = self.print_without_file |
| 105 | self.print_func = print_func |
| 106 | else: |
| 107 | self.print = print |
| 108 | |
dnj | 4625b5a | 2016-11-10 18:23:26 -0800 | [diff] [blame] | 109 | def print_without_file(self, message, **_kwargs): |
loislo@chromium.org | 0fb693f | 2014-12-25 15:28:22 +0000 | [diff] [blame] | 110 | self.print_func(message) |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 111 | |
hinoka@chromium.org | f8fa23d | 2014-06-05 01:00:04 +0000 | [diff] [blame] | 112 | @property |
| 113 | def bootstrap_bucket(self): |
Ryan Tseng | 3beabd0 | 2017-03-15 13:57:58 -0700 | [diff] [blame] | 114 | u = urlparse.urlparse(self.url) |
| 115 | if u.netloc == 'chromium.googlesource.com': |
hinoka@chromium.org | f8fa23d | 2014-06-05 01:00:04 +0000 | [diff] [blame] | 116 | return 'chromium-git-cache' |
Ryan Tseng | 3beabd0 | 2017-03-15 13:57:58 -0700 | [diff] [blame] | 117 | elif u.netloc == 'chrome-internal.googlesource.com': |
| 118 | return 'chrome-git-cache' |
| 119 | # Not recognized. |
| 120 | return None |
hinoka@chromium.org | f8fa23d | 2014-06-05 01:00:04 +0000 | [diff] [blame] | 121 | |
szager@chromium.org | 174766f | 2014-05-13 21:27:46 +0000 | [diff] [blame] | 122 | @classmethod |
| 123 | def FromPath(cls, path): |
| 124 | return cls(cls.CacheDirToUrl(path)) |
| 125 | |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 126 | @staticmethod |
| 127 | def UrlToCacheDir(url): |
| 128 | """Convert a git url to a normalized form for the cache dir path.""" |
| 129 | parsed = urlparse.urlparse(url) |
| 130 | norm_url = parsed.netloc + parsed.path |
| 131 | if norm_url.endswith('.git'): |
| 132 | norm_url = norm_url[:-len('.git')] |
| 133 | return norm_url.replace('-', '--').replace('/', '-').lower() |
| 134 | |
| 135 | @staticmethod |
szager@chromium.org | 174766f | 2014-05-13 21:27:46 +0000 | [diff] [blame] | 136 | def CacheDirToUrl(path): |
| 137 | """Convert a cache dir path to its corresponding url.""" |
| 138 | netpath = re.sub(r'\b-\b', '/', os.path.basename(path)).replace('--', '-') |
| 139 | return 'https://%s' % netpath |
| 140 | |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 141 | @classmethod |
| 142 | def SetCachePath(cls, cachepath): |
Ryan Tseng | c3eb3fa | 2017-10-09 17:15:42 -0700 | [diff] [blame^] | 143 | setattr(cls, 'cachepath', cachepath) |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 144 | |
| 145 | @classmethod |
| 146 | def GetCachePath(cls): |
Ryan Tseng | c3eb3fa | 2017-10-09 17:15:42 -0700 | [diff] [blame^] | 147 | if not hasattr(cls, 'cachepath'): |
| 148 | try: |
| 149 | cachepath = subprocess.check_output( |
| 150 | [cls.git_exe, 'config', '--global', 'cache.cachepath']).strip() |
| 151 | except subprocess.CalledProcessError: |
| 152 | cachepath = None |
| 153 | if not cachepath: |
| 154 | raise RuntimeError( |
| 155 | 'No global cache.cachepath git configuration found.') |
| 156 | setattr(cls, 'cachepath', cachepath) |
| 157 | return getattr(cls, 'cachepath') |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 158 | |
dnj | 4625b5a | 2016-11-10 18:23:26 -0800 | [diff] [blame] | 159 | def Rename(self, src, dst): |
| 160 | # This is somehow racy on Windows. |
| 161 | # Catching OSError because WindowsError isn't portable and |
| 162 | # pylint complains. |
| 163 | exponential_backoff_retry( |
| 164 | lambda: os.rename(src, dst), |
| 165 | excs=(OSError,), |
| 166 | name='rename [%s] => [%s]' % (src, dst), |
| 167 | printerr=self.print) |
| 168 | |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 169 | def RunGit(self, cmd, **kwargs): |
| 170 | """Run git in a subprocess.""" |
| 171 | cwd = kwargs.setdefault('cwd', self.mirror_path) |
| 172 | kwargs.setdefault('print_stdout', False) |
| 173 | kwargs.setdefault('filter_fn', self.print) |
| 174 | env = kwargs.get('env') or kwargs.setdefault('env', os.environ.copy()) |
| 175 | env.setdefault('GIT_ASKPASS', 'true') |
| 176 | env.setdefault('SSH_ASKPASS', 'true') |
| 177 | self.print('running "git %s" in "%s"' % (' '.join(cmd), cwd)) |
| 178 | gclient_utils.CheckCallAndFilter([self.git_exe] + cmd, **kwargs) |
| 179 | |
| 180 | def config(self, cwd=None): |
| 181 | if cwd is None: |
| 182 | cwd = self.mirror_path |
szager@chromium.org | 301a7c3 | 2014-06-16 17:13:50 +0000 | [diff] [blame] | 183 | |
| 184 | # Don't run git-gc in a daemon. Bad things can happen if it gets killed. |
hinoka | dcd8404 | 2016-06-09 14:26:17 -0700 | [diff] [blame] | 185 | try: |
| 186 | self.RunGit(['config', 'gc.autodetach', '0'], cwd=cwd) |
| 187 | except subprocess.CalledProcessError: |
| 188 | # Hard error, need to clobber. |
| 189 | raise ClobberNeeded() |
szager@chromium.org | 301a7c3 | 2014-06-16 17:13:50 +0000 | [diff] [blame] | 190 | |
| 191 | # Don't combine pack files into one big pack file. It's really slow for |
| 192 | # repositories, and there's no way to track progress and make sure it's |
| 193 | # not stuck. |
Ryan Tseng | 3beabd0 | 2017-03-15 13:57:58 -0700 | [diff] [blame] | 194 | if self.supported_project(): |
| 195 | self.RunGit(['config', 'gc.autopacklimit', '0'], cwd=cwd) |
szager@chromium.org | 301a7c3 | 2014-06-16 17:13:50 +0000 | [diff] [blame] | 196 | |
| 197 | # Allocate more RAM for cache-ing delta chains, for better performance |
| 198 | # of "Resolving deltas". |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 199 | self.RunGit(['config', 'core.deltaBaseCacheLimit', |
hinoka@chromium.org | 8e095af | 2015-06-10 19:19:07 +0000 | [diff] [blame] | 200 | gclient_utils.DefaultDeltaBaseCacheLimit()], cwd=cwd) |
szager@chromium.org | 301a7c3 | 2014-06-16 17:13:50 +0000 | [diff] [blame] | 201 | |
hinoka@chromium.org | 8e095af | 2015-06-10 19:19:07 +0000 | [diff] [blame] | 202 | self.RunGit(['config', 'remote.origin.url', self.url], cwd=cwd) |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 203 | self.RunGit(['config', '--replace-all', 'remote.origin.fetch', |
hinoka@chromium.org | 8e095af | 2015-06-10 19:19:07 +0000 | [diff] [blame] | 204 | '+refs/heads/*:refs/heads/*', r'\+refs/heads/\*:.*'], cwd=cwd) |
szager@chromium.org | 66c8b85 | 2015-09-22 23:19:07 +0000 | [diff] [blame] | 205 | for spec, value_regex in self.fetch_specs: |
szager@chromium.org | 965c44f | 2014-08-19 21:19:19 +0000 | [diff] [blame] | 206 | self.RunGit( |
szager@chromium.org | 66c8b85 | 2015-09-22 23:19:07 +0000 | [diff] [blame] | 207 | ['config', '--replace-all', 'remote.origin.fetch', spec, value_regex], |
hinoka@chromium.org | 8e095af | 2015-06-10 19:19:07 +0000 | [diff] [blame] | 208 | cwd=cwd) |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 209 | |
| 210 | def bootstrap_repo(self, directory): |
hinoka@chromium.org | aa1e1a4 | 2014-06-26 21:58:51 +0000 | [diff] [blame] | 211 | """Bootstrap the repo from Google Stroage if possible. |
| 212 | |
| 213 | More apt-ly named bootstrap_repo_from_cloud_if_possible_else_do_nothing(). |
| 214 | """ |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 215 | |
Ryan Tseng | 3beabd0 | 2017-03-15 13:57:58 -0700 | [diff] [blame] | 216 | if not self.bootstrap_bucket: |
| 217 | return False |
hinoka@google.com | 776a2c3 | 2014-04-25 07:54:25 +0000 | [diff] [blame] | 218 | python_fallback = False |
sbc@chromium.org | 9d0644d | 2015-06-05 23:16:54 +0000 | [diff] [blame] | 219 | if (sys.platform.startswith('win') and |
| 220 | not gclient_utils.FindExecutable('7z')): |
hinoka@google.com | 776a2c3 | 2014-04-25 07:54:25 +0000 | [diff] [blame] | 221 | python_fallback = True |
| 222 | elif sys.platform.startswith('darwin'): |
| 223 | # The OSX version of unzip doesn't support zip64. |
| 224 | python_fallback = True |
sbc@chromium.org | 9d0644d | 2015-06-05 23:16:54 +0000 | [diff] [blame] | 225 | elif not gclient_utils.FindExecutable('unzip'): |
hinoka@google.com | 776a2c3 | 2014-04-25 07:54:25 +0000 | [diff] [blame] | 226 | python_fallback = True |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 227 | |
| 228 | gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir) |
hinoka@chromium.org | 199bc5f | 2014-12-17 02:17:14 +0000 | [diff] [blame] | 229 | gsutil = Gsutil(self.gsutil_exe, boto_path=None) |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 230 | # Get the most recent version of the zipfile. |
| 231 | _, ls_out, _ = gsutil.check_call('ls', gs_folder) |
| 232 | ls_out_sorted = sorted(ls_out.splitlines()) |
| 233 | if not ls_out_sorted: |
| 234 | # This repo is not on Google Storage. |
| 235 | return False |
| 236 | latest_checkout = ls_out_sorted[-1] |
| 237 | |
| 238 | # Download zip file to a temporary directory. |
| 239 | try: |
szager@chromium.org | 1cbf104 | 2014-06-17 18:26:24 +0000 | [diff] [blame] | 240 | tempdir = tempfile.mkdtemp(prefix='_cache_tmp', dir=self.GetCachePath()) |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 241 | self.print('Downloading %s' % latest_checkout) |
hinoka@chromium.org | c58d11d | 2014-06-09 23:34:35 +0000 | [diff] [blame] | 242 | code = gsutil.call('cp', latest_checkout, tempdir) |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 243 | if code: |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 244 | return False |
| 245 | filename = os.path.join(tempdir, latest_checkout.split('/')[-1]) |
| 246 | |
hinoka@google.com | 776a2c3 | 2014-04-25 07:54:25 +0000 | [diff] [blame] | 247 | # Unpack the file with 7z on Windows, unzip on linux, or fallback. |
| 248 | if not python_fallback: |
| 249 | if sys.platform.startswith('win'): |
| 250 | cmd = ['7z', 'x', '-o%s' % directory, '-tzip', filename] |
| 251 | else: |
| 252 | cmd = ['unzip', filename, '-d', directory] |
| 253 | retcode = subprocess.call(cmd) |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 254 | else: |
hinoka@google.com | 776a2c3 | 2014-04-25 07:54:25 +0000 | [diff] [blame] | 255 | try: |
| 256 | with zipfile.ZipFile(filename, 'r') as f: |
| 257 | f.printdir() |
| 258 | f.extractall(directory) |
| 259 | except Exception as e: |
| 260 | self.print('Encountered error: %s' % str(e), file=sys.stderr) |
| 261 | retcode = 1 |
| 262 | else: |
| 263 | retcode = 0 |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 264 | finally: |
| 265 | # Clean up the downloaded zipfile. |
dnj | 4625b5a | 2016-11-10 18:23:26 -0800 | [diff] [blame] | 266 | # |
| 267 | # This is somehow racy on Windows. |
| 268 | # Catching OSError because WindowsError isn't portable and |
| 269 | # pylint complains. |
| 270 | exponential_backoff_retry( |
| 271 | lambda: gclient_utils.rm_file_or_tree(tempdir), |
| 272 | excs=(OSError,), |
| 273 | name='rmtree [%s]' % (tempdir,), |
| 274 | printerr=self.print) |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 275 | |
| 276 | if retcode: |
| 277 | self.print( |
| 278 | 'Extracting bootstrap zipfile %s failed.\n' |
| 279 | 'Resuming normal operations.' % filename) |
| 280 | return False |
| 281 | return True |
| 282 | |
| 283 | def exists(self): |
| 284 | return os.path.isfile(os.path.join(self.mirror_path, 'config')) |
| 285 | |
Ryan Tseng | 3beabd0 | 2017-03-15 13:57:58 -0700 | [diff] [blame] | 286 | def supported_project(self): |
| 287 | """Returns true if this repo is known to have a bootstrap zip file.""" |
| 288 | u = urlparse.urlparse(self.url) |
| 289 | return u.netloc in [ |
| 290 | 'chromium.googlesource.com', |
| 291 | 'chrome-internal.googlesource.com'] |
| 292 | |
szager@chromium.org | 66c8b85 | 2015-09-22 23:19:07 +0000 | [diff] [blame] | 293 | def _preserve_fetchspec(self): |
| 294 | """Read and preserve remote.origin.fetch from an existing mirror. |
| 295 | |
| 296 | This modifies self.fetch_specs. |
| 297 | """ |
| 298 | if not self.exists(): |
| 299 | return |
| 300 | try: |
| 301 | config_fetchspecs = subprocess.check_output( |
| 302 | [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'], |
| 303 | cwd=self.mirror_path) |
| 304 | for fetchspec in config_fetchspecs.splitlines(): |
| 305 | self.fetch_specs.add(self.parse_fetch_spec(fetchspec)) |
| 306 | except subprocess.CalledProcessError: |
| 307 | logging.warn('Tried and failed to preserve remote.origin.fetch from the ' |
| 308 | 'existing cache directory. You may need to manually edit ' |
| 309 | '%s and "git cache fetch" again.' |
| 310 | % os.path.join(self.mirror_path, 'config')) |
| 311 | |
hinoka@chromium.org | aa1e1a4 | 2014-06-26 21:58:51 +0000 | [diff] [blame] | 312 | def _ensure_bootstrapped(self, depth, bootstrap, force=False): |
| 313 | tempdir = None |
hinoka@chromium.org | aa1e1a4 | 2014-06-26 21:58:51 +0000 | [diff] [blame] | 314 | pack_dir = os.path.join(self.mirror_path, 'objects', 'pack') |
| 315 | pack_files = [] |
| 316 | |
| 317 | if os.path.isdir(pack_dir): |
| 318 | pack_files = [f for f in os.listdir(pack_dir) if f.endswith('.pack')] |
| 319 | |
| 320 | should_bootstrap = (force or |
szager@chromium.org | 66c8b85 | 2015-09-22 23:19:07 +0000 | [diff] [blame] | 321 | not self.exists() or |
hinoka@chromium.org | aa1e1a4 | 2014-06-26 21:58:51 +0000 | [diff] [blame] | 322 | len(pack_files) > GC_AUTOPACKLIMIT) |
| 323 | if should_bootstrap: |
szager@chromium.org | 66c8b85 | 2015-09-22 23:19:07 +0000 | [diff] [blame] | 324 | if self.exists(): |
| 325 | # Re-bootstrapping an existing mirror; preserve existing fetch spec. |
| 326 | self._preserve_fetchspec() |
hinoka@chromium.org | aa1e1a4 | 2014-06-26 21:58:51 +0000 | [diff] [blame] | 327 | tempdir = tempfile.mkdtemp( |
| 328 | prefix='_cache_tmp', suffix=self.basedir, dir=self.GetCachePath()) |
| 329 | bootstrapped = not depth and bootstrap and self.bootstrap_repo(tempdir) |
| 330 | if bootstrapped: |
| 331 | # Bootstrap succeeded; delete previous cache, if any. |
hinoka@chromium.org | 42f9adf | 2014-09-05 11:10:35 +0000 | [diff] [blame] | 332 | gclient_utils.rmtree(self.mirror_path) |
Ryan Tseng | 3beabd0 | 2017-03-15 13:57:58 -0700 | [diff] [blame] | 333 | elif not self.exists() or not self.supported_project(): |
| 334 | # Bootstrap failed due to either |
| 335 | # 1. No previous cache |
| 336 | # 2. Project doesn't have a bootstrap zip file |
| 337 | # Start with a bare git dir. |
hinoka@chromium.org | aa1e1a4 | 2014-06-26 21:58:51 +0000 | [diff] [blame] | 338 | self.RunGit(['init', '--bare'], cwd=tempdir) |
| 339 | else: |
| 340 | # Bootstrap failed, previous cache exists; warn and continue. |
| 341 | logging.warn( |
| 342 | 'Git cache has a lot of pack files (%d). Tried to re-bootstrap ' |
| 343 | 'but failed. Continuing with non-optimized repository.' |
| 344 | % len(pack_files)) |
| 345 | gclient_utils.rmtree(tempdir) |
| 346 | tempdir = None |
| 347 | else: |
| 348 | if depth and os.path.exists(os.path.join(self.mirror_path, 'shallow')): |
| 349 | logging.warn( |
| 350 | 'Shallow fetch requested, but repo cache already exists.') |
| 351 | return tempdir |
| 352 | |
| 353 | def _fetch(self, rundir, verbose, depth): |
| 354 | self.config(rundir) |
| 355 | v = [] |
| 356 | d = [] |
| 357 | if verbose: |
| 358 | v = ['-v', '--progress'] |
| 359 | if depth: |
| 360 | d = ['--depth', str(depth)] |
| 361 | fetch_cmd = ['fetch'] + v + d + ['origin'] |
| 362 | fetch_specs = subprocess.check_output( |
| 363 | [self.git_exe, 'config', '--get-all', 'remote.origin.fetch'], |
| 364 | cwd=rundir).strip().splitlines() |
| 365 | for spec in fetch_specs: |
| 366 | try: |
| 367 | self.print('Fetching %s' % spec) |
| 368 | self.RunGit(fetch_cmd + [spec], cwd=rundir, retry=True) |
| 369 | except subprocess.CalledProcessError: |
| 370 | if spec == '+refs/heads/*:refs/heads/*': |
hinoka | dcd8404 | 2016-06-09 14:26:17 -0700 | [diff] [blame] | 371 | raise ClobberNeeded() # Corrupted cache. |
hinoka@chromium.org | aa1e1a4 | 2014-06-26 21:58:51 +0000 | [diff] [blame] | 372 | logging.warn('Fetch of %s failed' % spec) |
| 373 | |
Ryan Tseng | c3eb3fa | 2017-10-09 17:15:42 -0700 | [diff] [blame^] | 374 | def populate(self, depth=None, shallow=False, bootstrap=False, verbose=False): |
szager@chromium.org | b0a13a2 | 2014-06-18 00:52:25 +0000 | [diff] [blame] | 375 | assert self.GetCachePath() |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 376 | if shallow and not depth: |
| 377 | depth = 10000 |
| 378 | gclient_utils.safe_makedirs(self.GetCachePath()) |
| 379 | |
hinoka@chromium.org | aa1e1a4 | 2014-06-26 21:58:51 +0000 | [diff] [blame] | 380 | tempdir = None |
szager@chromium.org | 108eced | 2014-06-19 21:22:43 +0000 | [diff] [blame] | 381 | try: |
hinoka@chromium.org | aa1e1a4 | 2014-06-26 21:58:51 +0000 | [diff] [blame] | 382 | tempdir = self._ensure_bootstrapped(depth, bootstrap) |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 383 | rundir = tempdir or self.mirror_path |
hinoka@chromium.org | aa1e1a4 | 2014-06-26 21:58:51 +0000 | [diff] [blame] | 384 | self._fetch(rundir, verbose, depth) |
hinoka | dcd8404 | 2016-06-09 14:26:17 -0700 | [diff] [blame] | 385 | except ClobberNeeded: |
hinoka@chromium.org | aa1e1a4 | 2014-06-26 21:58:51 +0000 | [diff] [blame] | 386 | # This is a major failure, we need to clean and force a bootstrap. |
| 387 | gclient_utils.rmtree(rundir) |
| 388 | self.print(GIT_CACHE_CORRUPT_MESSAGE) |
| 389 | tempdir = self._ensure_bootstrapped(depth, bootstrap, force=True) |
| 390 | assert tempdir |
| 391 | self._fetch(tempdir or self.mirror_path, verbose, depth) |
| 392 | finally: |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 393 | if tempdir: |
dnj | b445ef5 | 2016-11-10 15:51:39 -0800 | [diff] [blame] | 394 | if os.path.exists(self.mirror_path): |
| 395 | gclient_utils.rmtree(self.mirror_path) |
dnj | 4625b5a | 2016-11-10 18:23:26 -0800 | [diff] [blame] | 396 | self.Rename(tempdir, self.mirror_path) |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 397 | |
hinoka@chromium.org | c8444f3 | 2014-06-18 23:18:17 +0000 | [diff] [blame] | 398 | def update_bootstrap(self, prune=False): |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 399 | # The files are named <git number>.zip |
| 400 | gen_number = subprocess.check_output( |
| 401 | [self.git_exe, 'number', 'master'], cwd=self.mirror_path).strip() |
hinoka@chromium.org | 7b1cb6f | 2014-09-08 21:40:50 +0000 | [diff] [blame] | 402 | # Run Garbage Collect to compress packfile. |
| 403 | self.RunGit(['gc', '--prune=all']) |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 404 | # Creating a temp file and then deleting it ensures we can use this name. |
| 405 | _, tmp_zipfile = tempfile.mkstemp(suffix='.zip') |
| 406 | os.remove(tmp_zipfile) |
| 407 | subprocess.call(['zip', '-r', tmp_zipfile, '.'], cwd=self.mirror_path) |
| 408 | gsutil = Gsutil(path=self.gsutil_exe, boto_path=None) |
hinoka@chromium.org | c8444f3 | 2014-06-18 23:18:17 +0000 | [diff] [blame] | 409 | gs_folder = 'gs://%s/%s' % (self.bootstrap_bucket, self.basedir) |
| 410 | dest_name = '%s/%s.zip' % (gs_folder, gen_number) |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 411 | gsutil.call('cp', tmp_zipfile, dest_name) |
| 412 | os.remove(tmp_zipfile) |
| 413 | |
hinoka@chromium.org | c8444f3 | 2014-06-18 23:18:17 +0000 | [diff] [blame] | 414 | # Remove all other files in the same directory. |
| 415 | if prune: |
| 416 | _, ls_out, _ = gsutil.check_call('ls', gs_folder) |
| 417 | for filename in ls_out.splitlines(): |
| 418 | if filename == dest_name: |
| 419 | continue |
| 420 | gsutil.call('rm', filename) |
| 421 | |
szager@chromium.org | cdfcd7c | 2014-06-10 23:40:46 +0000 | [diff] [blame] | 422 | @staticmethod |
| 423 | def DeleteTmpPackFiles(path): |
| 424 | pack_dir = os.path.join(path, 'objects', 'pack') |
szager@chromium.org | 3341849 | 2014-06-18 19:03:39 +0000 | [diff] [blame] | 425 | if not os.path.isdir(pack_dir): |
| 426 | return |
szager@chromium.org | cdfcd7c | 2014-06-10 23:40:46 +0000 | [diff] [blame] | 427 | pack_files = [f for f in os.listdir(pack_dir) if |
| 428 | f.startswith('.tmp-') or f.startswith('tmp_pack_')] |
| 429 | for f in pack_files: |
| 430 | f = os.path.join(pack_dir, f) |
| 431 | try: |
| 432 | os.remove(f) |
| 433 | logging.warn('Deleted stale temporary pack file %s' % f) |
| 434 | except OSError: |
| 435 | logging.warn('Unable to delete temporary pack file %s' % f) |
szager@chromium.org | 174766f | 2014-05-13 21:27:46 +0000 | [diff] [blame] | 436 | |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 437 | |
agable@chromium.org | 5a306a2 | 2014-02-24 22:13:59 +0000 | [diff] [blame] | 438 | @subcommand.usage('[url of repo to check for caching]') |
| 439 | def CMDexists(parser, args): |
| 440 | """Check to see if there already is a cache of the given repo.""" |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 441 | _, args = parser.parse_args(args) |
agable@chromium.org | 5a306a2 | 2014-02-24 22:13:59 +0000 | [diff] [blame] | 442 | if not len(args) == 1: |
| 443 | parser.error('git cache exists only takes exactly one repo url.') |
| 444 | url = args[0] |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 445 | mirror = Mirror(url) |
| 446 | if mirror.exists(): |
| 447 | print(mirror.mirror_path) |
agable@chromium.org | 5a306a2 | 2014-02-24 22:13:59 +0000 | [diff] [blame] | 448 | return 0 |
| 449 | return 1 |
| 450 | |
| 451 | |
hinoka@google.com | 563559c | 2014-04-02 00:36:24 +0000 | [diff] [blame] | 452 | @subcommand.usage('[url of repo to create a bootstrap zip file]') |
| 453 | def CMDupdate_bootstrap(parser, args): |
| 454 | """Create and uploads a bootstrap tarball.""" |
| 455 | # Lets just assert we can't do this on Windows. |
| 456 | if sys.platform.startswith('win'): |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 457 | print('Sorry, update bootstrap will not work on Windows.', file=sys.stderr) |
hinoka@google.com | 563559c | 2014-04-02 00:36:24 +0000 | [diff] [blame] | 458 | return 1 |
| 459 | |
hinoka@chromium.org | c8444f3 | 2014-06-18 23:18:17 +0000 | [diff] [blame] | 460 | parser.add_option('--prune', action='store_true', |
| 461 | help='Prune all other cached zipballs of the same repo.') |
| 462 | |
hinoka@google.com | 563559c | 2014-04-02 00:36:24 +0000 | [diff] [blame] | 463 | # First, we need to ensure the cache is populated. |
| 464 | populate_args = args[:] |
pgervais@chromium.org | b9f2751 | 2014-08-08 15:52:33 +0000 | [diff] [blame] | 465 | populate_args.append('--no-bootstrap') |
hinoka@google.com | 563559c | 2014-04-02 00:36:24 +0000 | [diff] [blame] | 466 | CMDpopulate(parser, populate_args) |
| 467 | |
| 468 | # Get the repo directory. |
hinoka@chromium.org | c8444f3 | 2014-06-18 23:18:17 +0000 | [diff] [blame] | 469 | options, args = parser.parse_args(args) |
hinoka@google.com | 563559c | 2014-04-02 00:36:24 +0000 | [diff] [blame] | 470 | url = args[0] |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 471 | mirror = Mirror(url) |
hinoka@chromium.org | c8444f3 | 2014-06-18 23:18:17 +0000 | [diff] [blame] | 472 | mirror.update_bootstrap(options.prune) |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 473 | return 0 |
hinoka@google.com | 563559c | 2014-04-02 00:36:24 +0000 | [diff] [blame] | 474 | |
| 475 | |
agable@chromium.org | 5a306a2 | 2014-02-24 22:13:59 +0000 | [diff] [blame] | 476 | @subcommand.usage('[url of repo to add to or update in cache]') |
| 477 | def CMDpopulate(parser, args): |
| 478 | """Ensure that the cache has all up-to-date objects for the given repo.""" |
| 479 | parser.add_option('--depth', type='int', |
| 480 | help='Only cache DEPTH commits of history') |
| 481 | parser.add_option('--shallow', '-s', action='store_true', |
| 482 | help='Only cache 10000 commits of history') |
| 483 | parser.add_option('--ref', action='append', |
| 484 | help='Specify additional refs to be fetched') |
pgervais@chromium.org | b9f2751 | 2014-08-08 15:52:33 +0000 | [diff] [blame] | 485 | parser.add_option('--no_bootstrap', '--no-bootstrap', |
| 486 | action='store_true', |
hinoka@google.com | 563559c | 2014-04-02 00:36:24 +0000 | [diff] [blame] | 487 | help='Don\'t bootstrap from Google Storage') |
| 488 | |
agable@chromium.org | 5a306a2 | 2014-02-24 22:13:59 +0000 | [diff] [blame] | 489 | options, args = parser.parse_args(args) |
agable@chromium.org | 5a306a2 | 2014-02-24 22:13:59 +0000 | [diff] [blame] | 490 | if not len(args) == 1: |
| 491 | parser.error('git cache populate only takes exactly one repo url.') |
| 492 | url = args[0] |
| 493 | |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 494 | mirror = Mirror(url, refs=options.ref) |
| 495 | kwargs = { |
| 496 | 'verbose': options.verbose, |
| 497 | 'shallow': options.shallow, |
| 498 | 'bootstrap': not options.no_bootstrap, |
| 499 | } |
agable@chromium.org | 5a306a2 | 2014-02-24 22:13:59 +0000 | [diff] [blame] | 500 | if options.depth: |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 501 | kwargs['depth'] = options.depth |
| 502 | mirror.populate(**kwargs) |
agable@chromium.org | 5a306a2 | 2014-02-24 22:13:59 +0000 | [diff] [blame] | 503 | |
| 504 | |
szager@chromium.org | f314511 | 2014-08-07 21:02:36 +0000 | [diff] [blame] | 505 | @subcommand.usage('Fetch new commits into cache and current checkout') |
| 506 | def CMDfetch(parser, args): |
| 507 | """Update mirror, and fetch in cwd.""" |
| 508 | parser.add_option('--all', action='store_true', help='Fetch all remotes') |
szager@chromium.org | 66c8b85 | 2015-09-22 23:19:07 +0000 | [diff] [blame] | 509 | parser.add_option('--no_bootstrap', '--no-bootstrap', |
| 510 | action='store_true', |
| 511 | help='Don\'t (re)bootstrap from Google Storage') |
szager@chromium.org | f314511 | 2014-08-07 21:02:36 +0000 | [diff] [blame] | 512 | options, args = parser.parse_args(args) |
| 513 | |
| 514 | # Figure out which remotes to fetch. This mimics the behavior of regular |
| 515 | # 'git fetch'. Note that in the case of "stacked" or "pipelined" branches, |
| 516 | # this will NOT try to traverse up the branching structure to find the |
| 517 | # ultimate remote to update. |
| 518 | remotes = [] |
| 519 | if options.all: |
| 520 | assert not args, 'fatal: fetch --all does not take a repository argument' |
| 521 | remotes = subprocess.check_output([Mirror.git_exe, 'remote']).splitlines() |
| 522 | elif args: |
| 523 | remotes = args |
| 524 | else: |
| 525 | current_branch = subprocess.check_output( |
| 526 | [Mirror.git_exe, 'rev-parse', '--abbrev-ref', 'HEAD']).strip() |
| 527 | if current_branch != 'HEAD': |
| 528 | upstream = subprocess.check_output( |
| 529 | [Mirror.git_exe, 'config', 'branch.%s.remote' % current_branch] |
| 530 | ).strip() |
| 531 | if upstream and upstream != '.': |
| 532 | remotes = [upstream] |
| 533 | if not remotes: |
| 534 | remotes = ['origin'] |
| 535 | |
| 536 | cachepath = Mirror.GetCachePath() |
| 537 | git_dir = os.path.abspath(subprocess.check_output( |
| 538 | [Mirror.git_exe, 'rev-parse', '--git-dir'])) |
| 539 | git_dir = os.path.abspath(git_dir) |
| 540 | if git_dir.startswith(cachepath): |
| 541 | mirror = Mirror.FromPath(git_dir) |
szager@chromium.org | dbb6f82 | 2016-02-02 22:59:30 +0000 | [diff] [blame] | 542 | mirror.populate( |
Ryan Tseng | c3eb3fa | 2017-10-09 17:15:42 -0700 | [diff] [blame^] | 543 | bootstrap=not options.no_bootstrap,) |
szager@chromium.org | f314511 | 2014-08-07 21:02:36 +0000 | [diff] [blame] | 544 | return 0 |
| 545 | for remote in remotes: |
| 546 | remote_url = subprocess.check_output( |
| 547 | [Mirror.git_exe, 'config', 'remote.%s.url' % remote]).strip() |
| 548 | if remote_url.startswith(cachepath): |
| 549 | mirror = Mirror.FromPath(remote_url) |
| 550 | mirror.print = lambda *args: None |
| 551 | print('Updating git cache...') |
szager@chromium.org | dbb6f82 | 2016-02-02 22:59:30 +0000 | [diff] [blame] | 552 | mirror.populate( |
Ryan Tseng | c3eb3fa | 2017-10-09 17:15:42 -0700 | [diff] [blame^] | 553 | bootstrap=not options.no_bootstrap) |
szager@chromium.org | f314511 | 2014-08-07 21:02:36 +0000 | [diff] [blame] | 554 | subprocess.check_call([Mirror.git_exe, 'fetch', remote]) |
| 555 | return 0 |
| 556 | |
| 557 | |
agable@chromium.org | 5a306a2 | 2014-02-24 22:13:59 +0000 | [diff] [blame] | 558 | class OptionParser(optparse.OptionParser): |
| 559 | """Wrapper class for OptionParser to handle global options.""" |
| 560 | |
| 561 | def __init__(self, *args, **kwargs): |
| 562 | optparse.OptionParser.__init__(self, *args, prog='git cache', **kwargs) |
| 563 | self.add_option('-c', '--cache-dir', |
| 564 | help='Path to the directory containing the cache') |
szager@chromium.org | 2c391af | 2014-05-23 09:07:15 +0000 | [diff] [blame] | 565 | self.add_option('-v', '--verbose', action='count', default=1, |
agable@chromium.org | 5a306a2 | 2014-02-24 22:13:59 +0000 | [diff] [blame] | 566 | help='Increase verbosity (can be passed multiple times)') |
szager@chromium.org | 2c391af | 2014-05-23 09:07:15 +0000 | [diff] [blame] | 567 | self.add_option('-q', '--quiet', action='store_true', |
| 568 | help='Suppress all extraneous output') |
agable@chromium.org | 5a306a2 | 2014-02-24 22:13:59 +0000 | [diff] [blame] | 569 | |
| 570 | def parse_args(self, args=None, values=None): |
| 571 | options, args = optparse.OptionParser.parse_args(self, args, values) |
szager@chromium.org | 2c391af | 2014-05-23 09:07:15 +0000 | [diff] [blame] | 572 | if options.quiet: |
| 573 | options.verbose = 0 |
| 574 | |
| 575 | levels = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG] |
| 576 | logging.basicConfig(level=levels[min(options.verbose, len(levels) - 1)]) |
agable@chromium.org | 5a306a2 | 2014-02-24 22:13:59 +0000 | [diff] [blame] | 577 | |
| 578 | try: |
szager@chromium.org | 848fd49 | 2014-04-09 19:06:44 +0000 | [diff] [blame] | 579 | global_cache_dir = Mirror.GetCachePath() |
| 580 | except RuntimeError: |
| 581 | global_cache_dir = None |
| 582 | if options.cache_dir: |
| 583 | if global_cache_dir and ( |
| 584 | os.path.abspath(options.cache_dir) != |
| 585 | os.path.abspath(global_cache_dir)): |
| 586 | logging.warn('Overriding globally-configured cache directory.') |
| 587 | Mirror.SetCachePath(options.cache_dir) |
agable@chromium.org | 5a306a2 | 2014-02-24 22:13:59 +0000 | [diff] [blame] | 588 | |
agable@chromium.org | 5a306a2 | 2014-02-24 22:13:59 +0000 | [diff] [blame] | 589 | return options, args |
| 590 | |
| 591 | |
| 592 | def main(argv): |
| 593 | dispatcher = subcommand.CommandDispatcher(__name__) |
| 594 | return dispatcher.execute(OptionParser(), argv) |
| 595 | |
| 596 | |
| 597 | if __name__ == '__main__': |
sbc@chromium.org | 013731e | 2015-02-26 18:28:43 +0000 | [diff] [blame] | 598 | try: |
| 599 | sys.exit(main(sys.argv[1:])) |
| 600 | except KeyboardInterrupt: |
| 601 | sys.stderr.write('interrupted\n') |
| 602 | sys.exit(1) |