Add cipd support to gclient.
Bug: 789809
Change-Id: I9942eaa613d51edd77ee5195603932a103f5e3cd
Reviewed-on: https://chromium-review.googlesource.com/829953
Commit-Queue: John Budorick <jbudorick@chromium.org>
Reviewed-by: Aaron Gable <agable@chromium.org>
diff --git a/gclient.py b/gclient.py
index cc072e8..fd46ece 100755
--- a/gclient.py
+++ b/gclient.py
@@ -277,8 +277,9 @@
('dependency url must be either string or None, '
'instead of %s') % self._url.__class__.__name__)
# Make any deps_file path platform-appropriate.
- for sep in ['/', '\\']:
- self._deps_file = self._deps_file.replace(sep, os.sep)
+ if self._deps_file:
+ for sep in ['/', '\\']:
+ self._deps_file = self._deps_file.replace(sep, os.sep)
@property
def deps_file(self):
@@ -422,6 +423,22 @@
if not self.name and self.parent:
raise gclient_utils.Error('Dependency without name')
+ def ToLines(self):
+ s = []
+ condition_part = ([' "condition": %r,' % self.condition]
+ if self.condition else [])
+ s.extend([
+ ' # %s' % self.hierarchy(include_url=False),
+ ' "%s": {' % (self.name,),
+ ' "url": "%s",' % (self.raw_url,),
+ ] + condition_part + [
+ ' },',
+ '',
+ ])
+ return s
+
+
+
@property
def requirements(self):
"""Calculate the list of requirements."""
@@ -534,7 +551,7 @@
'relative DEPS entry \'%s\' must begin with a slash' % url)
# Create a scm just to query the full url.
parent_url = self.parent.parsed_url
- scm = gclient_scm.CreateSCM(
+ scm = self.CreateSCM(
parent_url, self.root.root_dir, None, self.outbuf)
parsed_url = scm.FullUrlForRelativeUrl(url)
else:
@@ -623,6 +640,7 @@
def _deps_to_objects(self, deps, use_relative_paths):
"""Convert a deps dict to a dict of Dependency objects."""
deps_to_add = []
+ cipd_root = None
for name, dep_value in deps.iteritems():
should_process = self.recursion_limit and self.should_process
deps_file = self.deps_file
@@ -632,30 +650,58 @@
deps_file = ent['deps_file']
if dep_value is None:
continue
+
condition = None
condition_value = True
if isinstance(dep_value, basestring):
raw_url = dep_value
+ dep_type = None
else:
# This should be guaranteed by schema checking in gclient_eval.
assert isinstance(dep_value, collections.Mapping)
- raw_url = dep_value['url']
+ raw_url = dep_value.get('url')
# Take into account should_process metadata set by MergeWithOsDeps.
should_process = (should_process and
dep_value.get('should_process', True))
condition = dep_value.get('condition')
-
- url = raw_url.format(**self.get_vars())
+ dep_type = dep_value.get('dep_type')
if condition:
condition_value = gclient_eval.EvaluateCondition(
condition, self.get_vars())
if not self._get_option('process_all_deps', False):
should_process = should_process and condition_value
- deps_to_add.append(Dependency(
- self, name, raw_url, url, None, None, self.custom_vars, None,
- deps_file, should_process, use_relative_paths, condition,
- condition_value))
+
+ if dep_type == 'cipd':
+ if not cipd_root:
+ cipd_root = gclient_scm.CipdRoot(
+ os.path.join(self.root.root_dir, self.name),
+ # TODO(jbudorick): Support other service URLs as necessary.
+ # Service URLs should be constant over the scope of a cipd
+ # root, so a var per DEPS file specifying the service URL
+ # should suffice.
+ 'https://chrome-infra-packages.appspot.com')
+ for package in dep_value.get('packages', []):
+ deps_to_add.append(
+ CipdDependency(
+ self, name, package, cipd_root,
+ self.custom_vars, should_process, use_relative_paths,
+ condition, condition_value))
+ elif dep_type == 'git':
+ url = raw_url.format(**self.get_vars())
+ deps_to_add.append(
+ GitDependency(
+ self, name, raw_url, url, None, None, self.custom_vars, None,
+ deps_file, should_process, use_relative_paths, condition,
+ condition_value))
+ else:
+ url = raw_url.format(**self.get_vars())
+ deps_to_add.append(
+ Dependency(
+ self, name, raw_url, url, None, None, self.custom_vars, None,
+ deps_file, should_process, use_relative_paths, condition,
+ condition_value))
+
deps_to_add.sort(key=lambda x: x.name)
return deps_to_add
@@ -695,7 +741,8 @@
# Eval the content.
try:
if self._get_option('validate_syntax', False):
- gclient_eval.Exec(deps_content, global_scope, local_scope, filepath)
+ local_scope = gclient_eval.Exec(
+ deps_content, global_scope, local_scope, filepath)
else:
exec(deps_content, global_scope, local_scope)
except SyntaxError as e:
@@ -878,7 +925,7 @@
options = copy.copy(options)
options.revision = revision_override
self._used_revision = options.revision
- self._used_scm = gclient_scm.CreateSCM(
+ self._used_scm = self.CreateSCM(
parsed_url, self.root.root_dir, self.name, self.outbuf,
out_cb=work_queue.out_cb)
self._got_revision = self._used_scm.RunCommand(command, options, args,
@@ -913,7 +960,7 @@
if command == 'recurse':
# Skip file only checkout.
- scm = gclient_scm.GetScmName(parsed_url)
+ scm = self.GetScmName(parsed_url)
if not options.scm or scm in options.scm:
cwd = os.path.normpath(os.path.join(self.root.root_dir, self.name))
# Pass in the SCM type as an env variable. Make sure we don't put
@@ -967,6 +1014,40 @@
else:
print('Skipped missing %s' % cwd, file=sys.stderr)
+ def GetScmName(self, url):
+ """Get the name of the SCM for the given URL.
+
+ While we currently support both git and cipd as SCM implementations,
+ this currently cannot return 'cipd', regardless of the URL, as CIPD
+ has no canonical URL format. If you want to use CIPD as an SCM, you
+ must currently do so by explicitly using a CipdDependency.
+ """
+ if not url:
+ return None
+ url, _ = gclient_utils.SplitUrlRevision(url)
+ if url.endswith('.git'):
+ return 'git'
+ protocol = url.split('://')[0]
+ if protocol in (
+ 'file', 'git', 'git+http', 'git+https', 'http', 'https', 'ssh', 'sso'):
+ return 'git'
+ return None
+
+ def CreateSCM(self, url, root_dir=None, relpath=None, out_fh=None,
+ out_cb=None):
+ SCM_MAP = {
+ 'cipd': gclient_scm.CipdWrapper,
+ 'git': gclient_scm.GitWrapper,
+ }
+
+ scm_name = self.GetScmName(url)
+ if not scm_name in SCM_MAP:
+ raise gclient_utils.Error('No SCM found for url %s' % url)
+ scm_class = SCM_MAP[scm_name]
+ if not scm_class.BinaryExists():
+ raise gclient_utils.Error('%s command not found' % scm_name)
+ return scm_class(url, root_dir, relpath, out_fh, out_cb)
+
def HasGNArgsFile(self):
return self._gn_args_file is not None
@@ -1004,7 +1085,7 @@
# what files have changed so we always run all hooks. It'd be nice to fix
# that.
if (options.force or
- gclient_scm.GetScmName(self.parsed_url) in ('git', None) or
+ self.GetScmName(self.parsed_url) in ('git', None) or
os.path.isdir(os.path.join(self.root.root_dir, self.name, '.git'))):
result.extend(self.deps_hooks)
else:
@@ -1280,7 +1361,7 @@
solutions."""
for dep in self.dependencies:
if dep.managed and dep.url:
- scm = gclient_scm.CreateSCM(
+ scm = self.CreateSCM(
dep.url, self.root_dir, dep.name, self.outbuf)
actual_url = scm.GetActualRemoteURL(self._options)
if actual_url and not scm.DoesRemoteURLMatch(self._options):
@@ -1305,10 +1386,10 @@
it or fix the checkout.
''' % {'checkout_path': os.path.join(self.root_dir, dep.name),
'expected_url': dep.url,
- 'expected_scm': gclient_scm.GetScmName(dep.url),
+ 'expected_scm': self.GetScmName(dep.url),
'mirror_string' : mirror_string,
'actual_url': actual_url,
- 'actual_scm': gclient_scm.GetScmName(actual_url)})
+ 'actual_scm': self.GetScmName(actual_url)})
def SetConfig(self, content):
assert not self.dependencies
@@ -1529,7 +1610,7 @@
(not any(path.startswith(entry + '/') for path in entries)) and
os.path.exists(e_dir)):
# The entry has been removed from DEPS.
- scm = gclient_scm.CreateSCM(
+ scm = self.CreateSCM(
prev_url, self.root_dir, entry_fixed, self.outbuf)
# Check to see if this directory is now part of a higher-up checkout.
@@ -1619,7 +1700,7 @@
if dep.parsed_url is None:
return None
url, _ = gclient_utils.SplitUrlRevision(dep.parsed_url)
- scm = gclient_scm.CreateSCM(
+ scm = dep.CreateSCM(
dep.parsed_url, self.root_dir, dep.name, self.outbuf)
if not os.path.isdir(scm.checkout_path):
return None
@@ -1699,6 +1780,91 @@
return self._enforced_os
+class GitDependency(Dependency):
+ """A Dependency object that represents a single git checkout."""
+
+ #override
+ def GetScmName(self, url):
+ """Always 'git'."""
+ del url
+ return 'git'
+
+ #override
+ def CreateSCM(self, url, root_dir=None, relpath=None, out_fh=None,
+ out_cb=None):
+ """Create a Wrapper instance suitable for handling this git dependency."""
+ return gclient_scm.GitWrapper(url, root_dir, relpath, out_fh, out_cb)
+
+
+class CipdDependency(Dependency):
+ """A Dependency object that represents a single CIPD package."""
+
+ def __init__(
+ self, parent, name, dep_value, cipd_root,
+ custom_vars, should_process, relative, condition, condition_value):
+ package = dep_value['package']
+ version = dep_value['version']
+ url = urlparse.urljoin(
+ cipd_root.service_url, '%s@%s' % (package, version))
+ super(CipdDependency, self).__init__(
+ parent, name, url, url, None, None, custom_vars,
+ None, None, should_process, relative, condition, condition_value)
+ if relative:
+ # TODO(jbudorick): Implement relative if necessary.
+ raise gclient_utils.Error(
+ 'Relative CIPD dependencies are not currently supported.')
+ self._cipd_root = cipd_root
+
+ self._cipd_subdir = os.path.relpath(
+ os.path.join(self.root.root_dir, self.name), cipd_root.root_dir)
+ self._cipd_package = self._cipd_root.add_package(
+ self._cipd_subdir, package, version)
+
+ def ParseDepsFile(self):
+ """CIPD dependencies are not currently allowed to have nested deps."""
+ self.add_dependencies_and_close([], [])
+
+ #override
+ def GetScmName(self, url):
+ """Always 'cipd'."""
+ del url
+ return 'cipd'
+
+ #override
+ def CreateSCM(self, url, root_dir=None, relpath=None, out_fh=None,
+ out_cb=None):
+ """Create a Wrapper instance suitable for handling this CIPD dependency."""
+ return gclient_scm.CipdWrapper(
+ url, root_dir, relpath, out_fh, out_cb,
+ root=self._cipd_root,
+ package=self._cipd_package)
+
+ def ToLines(self):
+ """Return a list of lines representing this in a DEPS file."""
+ s = []
+ if self._cipd_package.authority_for_subdir:
+ condition_part = ([' "condition": %r,' % self.condition]
+ if self.condition else [])
+ s.extend([
+ ' # %s' % self.hierarchy(include_url=False),
+ ' "%s": {' % (self.name,),
+ ' "packages": [',
+ ])
+ for p in self._cipd_root.packages(self._cipd_subdir):
+ s.extend([
+ ' "package": "%s",' % p.name,
+ ' "version": "%s",' % p.version,
+ ])
+ s.extend([
+ ' ],',
+ ' "dep_type": "cipd",',
+ ] + condition_part + [
+ ' },',
+ '',
+ ])
+ return s
+
+
#### gclient commands.
@@ -1809,7 +1975,7 @@
if revision and gclient_utils.IsFullGitSha(revision):
return
- scm = gclient_scm.CreateSCM(
+ scm = dep.CreateSCM(
dep.parsed_url, self._client.root_dir, dep.name, dep.outbuf)
revinfo = scm.revinfo(self._client._options, [], None)
@@ -2028,17 +2194,8 @@
if not deps:
return []
s = ['deps = {']
- for name, dep in sorted(deps.iteritems()):
- condition_part = ([' "condition": %r,' % dep.condition]
- if dep.condition else [])
- s.extend([
- ' # %s' % dep.hierarchy(include_url=False),
- ' "%s": {' % (name,),
- ' "url": "%s",' % (dep.raw_url,),
- ] + condition_part + [
- ' },',
- '',
- ])
+ for _, dep in sorted(deps.iteritems()):
+ s.extend(dep.ToLines())
s.extend(['}', ''])
return s