blob: 723b22b9b1882b0ef347bde017e18d87352cb7d1 [file] [log] [blame]
nodirbe642ff2016-06-09 15:51:51 -07001# Copyright 2016 The LUCI Authors. All rights reserved.
2# Use of this source code is governed under the Apache License, Version 2.0
3# that can be found in the LICENSE file.
4
5"""Fetches CIPD client and installs packages."""
6
nodir90bc8dc2016-06-15 13:35:21 -07007__version__ = '0.2'
nodirbe642ff2016-06-09 15:51:51 -07008
nodir90bc8dc2016-06-15 13:35:21 -07009import collections
nodirbe642ff2016-06-09 15:51:51 -070010import contextlib
11import hashlib
12import logging
13import optparse
14import os
15import platform
16import sys
17import tempfile
18import time
19import urllib
20
21from utils import file_path
22from utils import fs
23from utils import net
24from utils import subprocess42
25from utils import tools
26import isolated_format
27import isolateserver
28
29
30# .exe on Windows.
31EXECUTABLE_SUFFIX = '.exe' if sys.platform == 'win32' else ''
32
33
34class Error(Exception):
35 """Raised on CIPD errors."""
36
37
38def add_cipd_options(parser):
39 group = optparse.OptionGroup(parser, 'CIPD')
40 group.add_option(
41 '--cipd-server',
nodirff531b42016-06-23 13:05:06 -070042 help='URL of the CIPD server. Only relevant with --cipd-package.')
nodirbe642ff2016-06-09 15:51:51 -070043 group.add_option(
44 '--cipd-client-package',
nodir90bc8dc2016-06-15 13:35:21 -070045 help='Package name of CIPD client with optional parameters described in '
nodirff531b42016-06-23 13:05:06 -070046 '--cipd-package help. '
47 'Only relevant with --cipd-package. '
nodirbe642ff2016-06-09 15:51:51 -070048 'Default: "%default"',
nodir90bc8dc2016-06-15 13:35:21 -070049 default='infra/tools/cipd/${platform}')
nodirbe642ff2016-06-09 15:51:51 -070050 group.add_option(
nodir90bc8dc2016-06-15 13:35:21 -070051 '--cipd-client-version',
52 help='Version of CIPD client. '
nodirff531b42016-06-23 13:05:06 -070053 'Only relevant with --cipd-package. '
nodir90bc8dc2016-06-15 13:35:21 -070054 'Default: "%default"',
55 default='latest')
56 group.add_option(
nodirff531b42016-06-23 13:05:06 -070057 '--cipd-package',
58 dest='cipd_packages',
59 help='A CIPD package to install. '
60 'Format is "<path>:<package_name>:<version>". '
61 '"path" is installation directory relative to run_dir, '
62 'defaults to ".". '
nodir90bc8dc2016-06-15 13:35:21 -070063 '"package_name" may have ${platform} and/or ${os_ver} parameters. '
nodirbe642ff2016-06-09 15:51:51 -070064 '${platform} will be expanded to "<os>-<architecture>" and '
65 '${os_ver} will be expanded to OS version name. '
nodirff531b42016-06-23 13:05:06 -070066 'The option can be specified multiple times.',
67 action='append',
68 default=[])
nodirbe642ff2016-06-09 15:51:51 -070069 group.add_option(
70 '--cipd-cache',
71 help='CIPD cache directory, separate from isolate cache. '
72 'Only relevant with --cipd-package. '
73 'Default: "%default".',
74 default='')
75 parser.add_option_group(group)
76
77
78def validate_cipd_options(parser, options):
79 """Calls parser.error on first found error among cipd options."""
nodirff531b42016-06-23 13:05:06 -070080 if not options.cipd_packages:
nodirbe642ff2016-06-09 15:51:51 -070081 return
nodirff531b42016-06-23 13:05:06 -070082
83 for pkg in options.cipd_packages:
84 parts = pkg.split(':', 2)
85 if len(parts) != 3:
86 parser.error('invalid package "%s": must have at least 2 colons' % pkg)
87 _path, name, version = parts
88 if not name:
89 parser.error('invalid package "%s": package name is not specified' % pkg)
90 if not version:
91 parser.error('invalid package "%s": version is not specified' % pkg)
92
nodirbe642ff2016-06-09 15:51:51 -070093 if not options.cipd_server:
nodirff531b42016-06-23 13:05:06 -070094 parser.error('--cipd-package requires non-empty --cipd-server')
nodirbe642ff2016-06-09 15:51:51 -070095
96 if not options.cipd_client_package:
nodirbe642ff2016-06-09 15:51:51 -070097 parser.error(
nodirff531b42016-06-23 13:05:06 -070098 '--cipd-package requires non-empty --cipd-client-package')
nodir90bc8dc2016-06-15 13:35:21 -070099 if not options.cipd_client_version:
100 parser.error(
nodirff531b42016-06-23 13:05:06 -0700101 '--cipd-package requires non-empty --cipd-client-version')
nodirbe642ff2016-06-09 15:51:51 -0700102
103
104class CipdClient(object):
105 """Installs packages."""
106
107 def __init__(self, binary_path, service_url=None):
108 """Initializes CipdClient.
109
110 Args:
111 binary_path (str): path to the CIPD client binary.
112 service_url (str): if not None, URL of the CIPD backend that overrides
113 the default one.
114 """
115 self.binary_path = binary_path
116 self.service_url = service_url
117
118 def ensure(
119 self, site_root, packages, cache_dir=None, tmp_dir=None, timeout=None):
120 """Ensures that packages installed in |site_root| equals |packages| set.
121
122 Blocking call.
123
124 Args:
125 site_root (str): where to install packages.
nodir90bc8dc2016-06-15 13:35:21 -0700126 packages: list of (package_template, version) tuples.
nodirbe642ff2016-06-09 15:51:51 -0700127 cache_dir (str): if set, cache dir for cipd binary own cache.
128 Typically contains packages and tags.
129 tmp_dir (str): if not None, dir for temp files.
130 timeout (int): if not None, timeout in seconds for this function to run.
131
132 Raises:
133 Error if could not install packages or timed out.
134 """
135 timeoutfn = tools.sliding_timeout(timeout)
136 logging.info('Installing packages %r into %s', packages, site_root)
137
138 list_file_handle, list_file_path = tempfile.mkstemp(
139 dir=tmp_dir, prefix=u'cipd-ensure-list-', suffix='.txt')
140 try:
141 try:
nodir90bc8dc2016-06-15 13:35:21 -0700142 for pkg, version in packages:
nodirbe642ff2016-06-09 15:51:51 -0700143 pkg = render_package_name_template(pkg)
144 os.write(list_file_handle, '%s %s\n' % (pkg, version))
145 finally:
146 os.close(list_file_handle)
147
148 cmd = [
149 self.binary_path, 'ensure',
150 '-root', site_root,
151 '-list', list_file_path,
152 '-verbose', # this is safe because cipd-ensure does not print a lot
153 ]
154 if cache_dir:
155 cmd += ['-cache-dir', cache_dir]
156 if self.service_url:
157 cmd += ['-service-url', self.service_url]
158
159 logging.debug('Running %r', cmd)
160 process = subprocess42.Popen(
161 cmd, stdout=subprocess42.PIPE, stderr=subprocess42.PIPE)
162 output = []
163 for pipe_name, line in process.yield_any_line(timeout=0.1):
164 to = timeoutfn()
165 if to is not None and to <= 0:
166 raise Error(
167 'Could not install packages; took more than %d seconds' % timeout)
168 if not pipe_name:
169 # stdout or stderr was closed, but yield_any_line still may have
170 # something to yield.
171 continue
172 output.append(line)
173 if pipe_name == 'stderr':
174 logging.debug('cipd client: %s', line)
175 else:
176 logging.info('cipd client: %s', line)
177
178 exit_code = process.wait(timeout=timeoutfn())
179 if exit_code != 0:
180 raise Error(
181 'Could not install packages; exit code %d\noutput:%s' % (
182 exit_code, '\n'.join(output)))
183 finally:
184 fs.remove(list_file_path)
185
186
187def get_platform():
188 """Returns ${platform} parameter value.
189
190 Borrowed from
191 https://chromium.googlesource.com/infra/infra/+/aaf9586/build/build.py#204
192 """
193 # linux, mac or windows.
194 platform_variant = {
195 'darwin': 'mac',
196 'linux2': 'linux',
197 'win32': 'windows',
198 }.get(sys.platform)
199 if not platform_variant:
200 raise Error('Unknown OS: %s' % sys.platform)
201
202 # amd64, 386, etc.
203 machine = platform.machine().lower()
204 platform_arch = {
205 'amd64': 'amd64',
206 'i386': '386',
207 'i686': '386',
208 'x86': '386',
209 'x86_64': 'amd64',
210 }.get(machine)
211 if not platform_arch:
212 if machine.startswith('arm'):
213 platform_arch = 'armv6l'
214 else:
215 platform_arch = 'amd64' if sys.maxsize > 2**32 else '386'
216 return '%s-%s' % (platform_variant, platform_arch)
217
218
219def get_os_ver():
220 """Returns ${os_ver} parameter value.
221
222 Examples: 'ubuntu14_04' or 'mac10_9' or 'win6_1'.
223
224 Borrowed from
225 https://chromium.googlesource.com/infra/infra/+/aaf9586/build/build.py#204
226 """
227 if sys.platform == 'darwin':
228 # platform.mac_ver()[0] is '10.9.5'.
229 dist = platform.mac_ver()[0].split('.')
230 return 'mac%s_%s' % (dist[0], dist[1])
231
232 if sys.platform == 'linux2':
233 # platform.linux_distribution() is ('Ubuntu', '14.04', ...).
234 dist = platform.linux_distribution()
235 return '%s%s' % (dist[0].lower(), dist[1].replace('.', '_'))
236
237 if sys.platform == 'win32':
238 # platform.version() is '6.1.7601'.
239 dist = platform.version().split('.')
240 return 'win%s_%s' % (dist[0], dist[1])
241 raise Error('Unknown OS: %s' % sys.platform)
242
243
244def render_package_name_template(template):
245 """Expands template variables in a CIPD package name template."""
246 return (template
247 .lower() # Package names are always lower case
248 .replace('${platform}', get_platform())
249 .replace('${os_ver}', get_os_ver()))
250
251
nodirbe642ff2016-06-09 15:51:51 -0700252def _check_response(res, fmt, *args):
253 """Raises Error if response is bad."""
254 if not res:
255 raise Error('%s: no response' % (fmt % args))
256
257 if res.get('status') != 'SUCCESS':
258 raise Error('%s: %s' % (
259 fmt % args,
260 res.get('error_message') or 'status is %s' % res.get('status')))
261
262
263def resolve_version(cipd_server, package_name, version, timeout=None):
264 """Resolves a package instance version (e.g. a tag) to an instance id."""
265 url = '%s/_ah/api/repo/v1/instance/resolve?%s' % (
266 cipd_server,
267 urllib.urlencode({
268 'package_name': package_name,
269 'version': version,
270 }))
271 res = net.url_read_json(url, timeout=timeout)
272 _check_response(res, 'Could not resolve version %s:%s', package_name, version)
273 instance_id = res.get('instance_id')
274 if not instance_id:
275 raise Error('Invalid resolveVersion response: no instance id')
276 return instance_id
277
278
279def get_client_fetch_url(service_url, package_name, instance_id, timeout=None):
280 """Returns a fetch URL of CIPD client binary contents.
281
282 Raises:
283 Error if cannot retrieve fetch URL.
284 """
285 # Fetch the URL of the binary from CIPD backend.
286 package_name = render_package_name_template(package_name)
287 url = '%s/_ah/api/repo/v1/client?%s' % (service_url, urllib.urlencode({
288 'package_name': package_name,
289 'instance_id': instance_id,
290 }))
291 res = net.url_read_json(url, timeout=timeout)
292 _check_response(
293 res, 'Could not fetch CIPD client %s:%s',package_name, instance_id)
294 fetch_url = res.get('client_binary', {}).get('fetch_url')
295 if not fetch_url:
296 raise Error('Invalid fetchClientBinary response: no fetch_url')
297 return fetch_url
298
299
300def _fetch_cipd_client(disk_cache, instance_id, fetch_url, timeoutfn):
301 """Fetches cipd binary to |disk_cache|.
302
303 Retries requests with exponential back-off.
304
305 Raises:
306 Error if could not fetch content.
307 """
308 sleep_time = 1
309 for attempt in xrange(5):
310 if attempt > 0:
311 if timeoutfn() is not None and timeoutfn() < sleep_time:
312 raise Error('Could not fetch CIPD client: timeout')
313 logging.warning('Will retry to fetch CIPD client in %ds', sleep_time)
314 time.sleep(sleep_time)
315 sleep_time *= 2
316
317 try:
318 res = net.url_open(fetch_url, timeout=timeoutfn())
319 if res:
320 disk_cache.write(instance_id, res.iter_content(64 * 1024))
321 return
322 except net.TimeoutError as ex:
323 raise Error('Could not fetch CIPD client: %s', ex)
324 except net.NetError as ex:
325 logging.warning(
326 'Could not fetch CIPD client on attempt #%d: %s', attempt + 1, ex)
327
328 raise Error('Could not fetch CIPD client after 5 retries')
329
330
331@contextlib.contextmanager
332def get_client(
333 service_url, package_name, version, cache_dir, timeout=None):
334 """Returns a context manager that yields a CipdClient. A blocking call.
335
336 Args:
337 service_url (str): URL of the CIPD backend.
338 package_name (str): package name template of the CIPD client.
339 version (str): version of CIPD client package.
340 cache_dir: directory to store instance cache, version cache
341 and a hardlink to the client binary.
342 timeout (int): if not None, timeout in seconds for this function.
343
344 Yields:
345 CipdClient.
346
347 Raises:
348 Error if CIPD client version cannot be resolved or client cannot be fetched.
349 """
350 timeoutfn = tools.sliding_timeout(timeout)
351
352 package_name = render_package_name_template(package_name)
353
354 # Resolve version to instance id.
355 # Is it an instance id already? They look like HEX SHA1.
356 if isolated_format.is_valid_hash(version, hashlib.sha1):
357 instance_id = version
358 else:
359 # version_cache is {version_digest -> instance id} mapping.
360 # It does not take a lot of disk space.
361 version_cache = isolateserver.DiskCache(
362 unicode(os.path.join(cache_dir, 'versions')),
363 isolateserver.CachePolicies(0, 0, 300),
364 hashlib.sha1)
365 with version_cache:
maruel2e8d0f52016-07-16 07:51:29 -0700366 version_cache.cleanup()
nodirbe642ff2016-06-09 15:51:51 -0700367 # Convert |version| to a string that may be used as a filename in disk
368 # cache by hashing it.
369 version_digest = hashlib.sha1(version).hexdigest()
370 try:
371 instance_id = version_cache.read(version_digest)
372 except isolateserver.CacheMiss:
373 instance_id = resolve_version(
374 service_url, package_name, version, timeout=timeoutfn())
375 version_cache.write(version_digest, instance_id)
376
377 # instance_cache is {instance_id -> client binary} mapping.
378 # It is bounded by 5 client versions.
379 instance_cache = isolateserver.DiskCache(
380 unicode(os.path.join(cache_dir, 'clients')),
381 isolateserver.CachePolicies(0, 0, 5),
382 hashlib.sha1)
383 with instance_cache:
maruel2e8d0f52016-07-16 07:51:29 -0700384 instance_cache.cleanup()
nodirbe642ff2016-06-09 15:51:51 -0700385 if instance_id not in instance_cache:
386 logging.info('Fetching CIPD client %s:%s', package_name, instance_id)
387 fetch_url = get_client_fetch_url(
388 service_url, package_name, instance_id, timeout=timeoutfn())
389 _fetch_cipd_client(instance_cache, instance_id, fetch_url, timeoutfn)
390
391 # A single host can run multiple swarming bots, but ATM they do not share
392 # same root bot directory. Thus, it is safe to use the same name for the
393 # binary.
394 binary_path = unicode(os.path.join(cache_dir, 'cipd' + EXECUTABLE_SUFFIX))
395 if fs.isfile(binary_path):
nodir6dfdb2d2016-06-14 20:14:08 -0700396 file_path.remove(binary_path)
nodirbe642ff2016-06-09 15:51:51 -0700397 instance_cache.hardlink(instance_id, binary_path, 0511) # -r-x--x--x
398
399 yield CipdClient(binary_path)
nodir90bc8dc2016-06-15 13:35:21 -0700400
401
nodirff531b42016-06-23 13:05:06 -0700402def parse_package_args(packages):
403 """Parses --cipd-package arguments.
nodir90bc8dc2016-06-15 13:35:21 -0700404
nodirff531b42016-06-23 13:05:06 -0700405 Assumes |packages| were validated by validate_cipd_options.
406
407 Returns:
408 A map {path: [(package, version)]}.
nodir90bc8dc2016-06-15 13:35:21 -0700409 """
nodirff531b42016-06-23 13:05:06 -0700410 result = collections.defaultdict(list)
411 for pkg in packages:
412 path, name, version = pkg.split(':', 2)
nodir90bc8dc2016-06-15 13:35:21 -0700413 path = path.replace('/', os.path.sep)
nodir90bc8dc2016-06-15 13:35:21 -0700414 if not name:
nodirff531b42016-06-23 13:05:06 -0700415 raise Error('Invalid package "%s": package name is not specified' % pkg)
nodir90bc8dc2016-06-15 13:35:21 -0700416 if not version:
nodirff531b42016-06-23 13:05:06 -0700417 raise Error('Invalid package "%s": version is not specified' % pkg)
418 result[path].append((name, version))
419 return result