Make gclient_scm.py use cache_dir
Instead of having custom logic for dealing with cache directories, use
git_cache.py to populate caches.
Also fixes a bug in git_cache.py where it was looking for lockfiles in cwd rather than the cache dir.
Other changes:
* _Run now returns output.
* Always print to stdout in CheckCallAndFilterOutput, even if it gets a carriage return. This is done because git progress report are carriage returns and not newlines and we don't want everything on the same line and not strip out the CRs.
* Removed members changed tests, its not very useful to know a new import is added.
BUG=339171
Review URL: https://codereview.chromium.org/180243006
git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@254248 0039d316-1c4b-4281-b951-d872f2087c98
diff --git a/gclient_scm.py b/gclient_scm.py
index 0389f46..238122f 100644
--- a/gclient_scm.py
+++ b/gclient_scm.py
@@ -4,14 +4,12 @@
"""Gclient-specific SCM-specific operations."""
-import collections
import logging
import os
import posixpath
import re
import sys
import tempfile
-import threading
import traceback
import urlparse
@@ -148,9 +146,6 @@
remote = 'origin'
cache_dir = None
- # If a given cache is used in a solution more than once, prevent multiple
- # threads from updating it simultaneously.
- cache_locks = collections.defaultdict(threading.Lock)
def __init__(self, url=None, root_dir=None, relpath=None):
"""Removes 'git+' fake prefix from git URL."""
@@ -352,10 +347,6 @@
self._FetchAndReset(revision, file_list, options)
return_early = True
- # Need to do this in the normal path as well as in the post-remote-switch
- # path.
- self._PossiblySwitchCache(url, options)
-
if return_early:
return self._Capture(['rev-parse', '--verify', 'HEAD'])
@@ -681,54 +672,6 @@
base_url = self.url
return base_url[:base_url.rfind('/')] + url
- @staticmethod
- def _NormalizeGitURL(url):
- '''Takes a git url, strips the scheme, and ensures it ends with '.git'.'''
- idx = url.find('://')
- if idx != -1:
- url = url[idx+3:]
- if not url.endswith('.git'):
- url += '.git'
- return url
-
- def _PossiblySwitchCache(self, url, options):
- """Handles switching a repo from with-cache to direct, or vice versa.
-
- When we go from direct to with-cache, the remote url changes from the
- 'real' url to the local file url (in cache_dir). Therefore, this function
- assumes that |url| points to the correctly-switched-over local file url, if
- we're in cache_mode.
-
- When we go from with-cache to direct, assume that the normal url-switching
- code already flipped the remote over, and we just need to repack and break
- the dependency to the cache.
- """
-
- altfile = os.path.join(
- self.checkout_path, '.git', 'objects', 'info', 'alternates')
- if self.cache_dir:
- if not os.path.exists(altfile):
- try:
- with open(altfile, 'w') as f:
- f.write(os.path.join(url, 'objects'))
- # pylint: disable=C0301
- # This dance is necessary according to emperical evidence, also at:
- # http://lists-archives.com/git/713652-retrospectively-add-alternates-to-a-repository.html
- self._Run(['repack', '-ad'], options)
- self._Run(['repack', '-adl'], options)
- except Exception:
- # If something goes wrong, try to remove the altfile so we'll go down
- # this path again next time.
- try:
- os.remove(altfile)
- except OSError as e:
- print >> sys.stderr, "FAILED: os.remove('%s') -> %s" % (altfile, e)
- raise
- else:
- if os.path.exists(altfile):
- self._Run(['repack', '-a'], options)
- os.remove(altfile)
-
def _CreateOrUpdateCache(self, url, options):
"""Make a new git mirror or update existing mirror for |url|, and return the
mirror URI to clone from.
@@ -737,60 +680,12 @@
"""
if not self.cache_dir:
return url
-
- # Replace - with -- to avoid ambiguity. / with - to flatten folder structure
- folder = os.path.join(
- self.cache_dir,
- self._NormalizeGitURL(url).replace('-', '--').replace('/', '-'))
- altfile = os.path.join(folder, 'objects', 'info', 'alternates')
-
- # If we're bringing an old cache up to date or cloning a new cache, and the
- # existing repo is currently a direct clone, use its objects to help out
- # the fetch here.
- checkout_objects = os.path.join(self.checkout_path, '.git', 'objects')
- checkout_altfile = os.path.join(checkout_objects, 'info', 'alternates')
- use_reference = (
- os.path.exists(checkout_objects) and
- not os.path.exists(checkout_altfile))
-
v = ['-v'] if options.verbose else []
- filter_fn = lambda l: '[up to date]' not in l
- with self.cache_locks[folder]:
- gclient_utils.safe_makedirs(self.cache_dir)
- if not os.path.exists(os.path.join(folder, 'config')):
- gclient_utils.rmtree(folder)
- cmd = ['clone'] + v + ['-c', 'core.deltaBaseCacheLimit=2g',
- '--progress', '--bare']
-
- if use_reference:
- cmd += ['--reference', os.path.abspath(self.checkout_path)]
-
- self._Run(cmd + [url, folder],
- options, filter_fn=filter_fn, cwd=self.cache_dir, retry=True)
- else:
- # For now, assert that host/path/to/repo.git is identical. We may want
- # to relax this restriction in the future to allow for smarter cache
- # repo update schemes (such as pulling the same repo, but from a
- # different host).
- existing_url = self._Capture(['config', 'remote.%s.url' % self.remote],
- cwd=folder)
- assert self._NormalizeGitURL(existing_url) == self._NormalizeGitURL(url)
-
- if use_reference:
- with open(altfile, 'w') as f:
- f.write(os.path.abspath(checkout_objects))
-
- # Would normally use `git remote update`, but it doesn't support
- # --progress, so use fetch instead.
- self._Run(['fetch'] + v + ['--multiple', '--progress', '--all'],
- options, filter_fn=filter_fn, cwd=folder, retry=True)
-
- # If the clone has an object dependency on the existing repo, break it
- # with repack and remove the linkage.
- if os.path.exists(altfile):
- self._Run(['repack', '-a'], options, cwd=folder)
- os.remove(altfile)
- return folder
+ self._Run(['cache', 'populate'] + v +
+ ['--shallow', '--cache-dir', self.cache_dir, url],
+ options, cwd=self._root_dir, retry=True)
+ return self._Run(['cache', 'exists', '--cache-dir', self.cache_dir, url],
+ options).strip()
def _Clone(self, revision, url, options):
"""Clone a git repository from the given URL.
@@ -1044,7 +939,7 @@
stdout = kwargs.get('stdout', sys.stdout)
stdout.write('\n________ running \'git %s\' in \'%s\'\n' % (
' '.join(args), kwargs['cwd']))
- gclient_utils.CheckCallAndFilter(['git'] + args, **kwargs)
+ return gclient_utils.CheckCallAndFilter(['git'] + args, **kwargs)
class SVNWrapper(SCMWrapper):