nodir | be642ff | 2016-06-09 15:51:51 -0700 | [diff] [blame] | 1 | # Copyright 2016 The LUCI Authors. All rights reserved. |
| 2 | # Use of this source code is governed under the Apache License, Version 2.0 |
| 3 | # that can be found in the LICENSE file. |
| 4 | |
| 5 | """Fetches CIPD client and installs packages.""" |
| 6 | |
nodir | 90bc8dc | 2016-06-15 13:35:21 -0700 | [diff] [blame] | 7 | __version__ = '0.2' |
nodir | be642ff | 2016-06-09 15:51:51 -0700 | [diff] [blame] | 8 | |
nodir | 90bc8dc | 2016-06-15 13:35:21 -0700 | [diff] [blame] | 9 | import collections |
nodir | be642ff | 2016-06-09 15:51:51 -0700 | [diff] [blame] | 10 | import contextlib |
| 11 | import hashlib |
| 12 | import logging |
| 13 | import optparse |
| 14 | import os |
| 15 | import platform |
| 16 | import sys |
| 17 | import tempfile |
| 18 | import time |
| 19 | import urllib |
| 20 | |
| 21 | from utils import file_path |
| 22 | from utils import fs |
| 23 | from utils import net |
| 24 | from utils import subprocess42 |
| 25 | from utils import tools |
| 26 | import isolated_format |
| 27 | import isolateserver |
| 28 | |
| 29 | |
| 30 | # .exe on Windows. |
| 31 | EXECUTABLE_SUFFIX = '.exe' if sys.platform == 'win32' else '' |
| 32 | |
| 33 | |
| 34 | class Error(Exception): |
| 35 | """Raised on CIPD errors.""" |
| 36 | |
| 37 | |
| 38 | def add_cipd_options(parser): |
| 39 | group = optparse.OptionGroup(parser, 'CIPD') |
| 40 | group.add_option( |
| 41 | '--cipd-server', |
nodir | ff531b4 | 2016-06-23 13:05:06 -0700 | [diff] [blame] | 42 | help='URL of the CIPD server. Only relevant with --cipd-package.') |
nodir | be642ff | 2016-06-09 15:51:51 -0700 | [diff] [blame] | 43 | group.add_option( |
| 44 | '--cipd-client-package', |
nodir | 90bc8dc | 2016-06-15 13:35:21 -0700 | [diff] [blame] | 45 | help='Package name of CIPD client with optional parameters described in ' |
nodir | ff531b4 | 2016-06-23 13:05:06 -0700 | [diff] [blame] | 46 | '--cipd-package help. ' |
| 47 | 'Only relevant with --cipd-package. ' |
nodir | be642ff | 2016-06-09 15:51:51 -0700 | [diff] [blame] | 48 | 'Default: "%default"', |
nodir | 90bc8dc | 2016-06-15 13:35:21 -0700 | [diff] [blame] | 49 | default='infra/tools/cipd/${platform}') |
nodir | be642ff | 2016-06-09 15:51:51 -0700 | [diff] [blame] | 50 | group.add_option( |
nodir | 90bc8dc | 2016-06-15 13:35:21 -0700 | [diff] [blame] | 51 | '--cipd-client-version', |
| 52 | help='Version of CIPD client. ' |
nodir | ff531b4 | 2016-06-23 13:05:06 -0700 | [diff] [blame] | 53 | 'Only relevant with --cipd-package. ' |
nodir | 90bc8dc | 2016-06-15 13:35:21 -0700 | [diff] [blame] | 54 | 'Default: "%default"', |
| 55 | default='latest') |
| 56 | group.add_option( |
nodir | ff531b4 | 2016-06-23 13:05:06 -0700 | [diff] [blame] | 57 | '--cipd-package', |
| 58 | dest='cipd_packages', |
| 59 | help='A CIPD package to install. ' |
| 60 | 'Format is "<path>:<package_name>:<version>". ' |
| 61 | '"path" is installation directory relative to run_dir, ' |
| 62 | 'defaults to ".". ' |
nodir | 90bc8dc | 2016-06-15 13:35:21 -0700 | [diff] [blame] | 63 | '"package_name" may have ${platform} and/or ${os_ver} parameters. ' |
nodir | be642ff | 2016-06-09 15:51:51 -0700 | [diff] [blame] | 64 | '${platform} will be expanded to "<os>-<architecture>" and ' |
| 65 | '${os_ver} will be expanded to OS version name. ' |
nodir | ff531b4 | 2016-06-23 13:05:06 -0700 | [diff] [blame] | 66 | 'The option can be specified multiple times.', |
| 67 | action='append', |
| 68 | default=[]) |
nodir | be642ff | 2016-06-09 15:51:51 -0700 | [diff] [blame] | 69 | group.add_option( |
| 70 | '--cipd-cache', |
| 71 | help='CIPD cache directory, separate from isolate cache. ' |
| 72 | 'Only relevant with --cipd-package. ' |
| 73 | 'Default: "%default".', |
| 74 | default='') |
| 75 | parser.add_option_group(group) |
| 76 | |
| 77 | |
| 78 | def validate_cipd_options(parser, options): |
| 79 | """Calls parser.error on first found error among cipd options.""" |
nodir | ff531b4 | 2016-06-23 13:05:06 -0700 | [diff] [blame] | 80 | if not options.cipd_packages: |
nodir | be642ff | 2016-06-09 15:51:51 -0700 | [diff] [blame] | 81 | return |
nodir | ff531b4 | 2016-06-23 13:05:06 -0700 | [diff] [blame] | 82 | |
| 83 | for pkg in options.cipd_packages: |
| 84 | parts = pkg.split(':', 2) |
| 85 | if len(parts) != 3: |
| 86 | parser.error('invalid package "%s": must have at least 2 colons' % pkg) |
| 87 | _path, name, version = parts |
| 88 | if not name: |
| 89 | parser.error('invalid package "%s": package name is not specified' % pkg) |
| 90 | if not version: |
| 91 | parser.error('invalid package "%s": version is not specified' % pkg) |
| 92 | |
nodir | be642ff | 2016-06-09 15:51:51 -0700 | [diff] [blame] | 93 | if not options.cipd_server: |
nodir | ff531b4 | 2016-06-23 13:05:06 -0700 | [diff] [blame] | 94 | parser.error('--cipd-package requires non-empty --cipd-server') |
nodir | be642ff | 2016-06-09 15:51:51 -0700 | [diff] [blame] | 95 | |
| 96 | if not options.cipd_client_package: |
nodir | be642ff | 2016-06-09 15:51:51 -0700 | [diff] [blame] | 97 | parser.error( |
nodir | ff531b4 | 2016-06-23 13:05:06 -0700 | [diff] [blame] | 98 | '--cipd-package requires non-empty --cipd-client-package') |
nodir | 90bc8dc | 2016-06-15 13:35:21 -0700 | [diff] [blame] | 99 | if not options.cipd_client_version: |
| 100 | parser.error( |
nodir | ff531b4 | 2016-06-23 13:05:06 -0700 | [diff] [blame] | 101 | '--cipd-package requires non-empty --cipd-client-version') |
nodir | be642ff | 2016-06-09 15:51:51 -0700 | [diff] [blame] | 102 | |
| 103 | |
| 104 | class CipdClient(object): |
| 105 | """Installs packages.""" |
| 106 | |
| 107 | def __init__(self, binary_path, service_url=None): |
| 108 | """Initializes CipdClient. |
| 109 | |
| 110 | Args: |
| 111 | binary_path (str): path to the CIPD client binary. |
| 112 | service_url (str): if not None, URL of the CIPD backend that overrides |
| 113 | the default one. |
| 114 | """ |
| 115 | self.binary_path = binary_path |
| 116 | self.service_url = service_url |
| 117 | |
| 118 | def ensure( |
| 119 | self, site_root, packages, cache_dir=None, tmp_dir=None, timeout=None): |
| 120 | """Ensures that packages installed in |site_root| equals |packages| set. |
| 121 | |
| 122 | Blocking call. |
| 123 | |
| 124 | Args: |
| 125 | site_root (str): where to install packages. |
nodir | 90bc8dc | 2016-06-15 13:35:21 -0700 | [diff] [blame] | 126 | packages: list of (package_template, version) tuples. |
nodir | be642ff | 2016-06-09 15:51:51 -0700 | [diff] [blame] | 127 | cache_dir (str): if set, cache dir for cipd binary own cache. |
| 128 | Typically contains packages and tags. |
| 129 | tmp_dir (str): if not None, dir for temp files. |
| 130 | timeout (int): if not None, timeout in seconds for this function to run. |
| 131 | |
| 132 | Raises: |
| 133 | Error if could not install packages or timed out. |
| 134 | """ |
| 135 | timeoutfn = tools.sliding_timeout(timeout) |
| 136 | logging.info('Installing packages %r into %s', packages, site_root) |
| 137 | |
| 138 | list_file_handle, list_file_path = tempfile.mkstemp( |
| 139 | dir=tmp_dir, prefix=u'cipd-ensure-list-', suffix='.txt') |
| 140 | try: |
| 141 | try: |
nodir | 90bc8dc | 2016-06-15 13:35:21 -0700 | [diff] [blame] | 142 | for pkg, version in packages: |
nodir | be642ff | 2016-06-09 15:51:51 -0700 | [diff] [blame] | 143 | pkg = render_package_name_template(pkg) |
| 144 | os.write(list_file_handle, '%s %s\n' % (pkg, version)) |
| 145 | finally: |
| 146 | os.close(list_file_handle) |
| 147 | |
| 148 | cmd = [ |
| 149 | self.binary_path, 'ensure', |
| 150 | '-root', site_root, |
| 151 | '-list', list_file_path, |
| 152 | '-verbose', # this is safe because cipd-ensure does not print a lot |
| 153 | ] |
| 154 | if cache_dir: |
| 155 | cmd += ['-cache-dir', cache_dir] |
| 156 | if self.service_url: |
| 157 | cmd += ['-service-url', self.service_url] |
| 158 | |
| 159 | logging.debug('Running %r', cmd) |
| 160 | process = subprocess42.Popen( |
| 161 | cmd, stdout=subprocess42.PIPE, stderr=subprocess42.PIPE) |
| 162 | output = [] |
| 163 | for pipe_name, line in process.yield_any_line(timeout=0.1): |
| 164 | to = timeoutfn() |
| 165 | if to is not None and to <= 0: |
| 166 | raise Error( |
| 167 | 'Could not install packages; took more than %d seconds' % timeout) |
| 168 | if not pipe_name: |
| 169 | # stdout or stderr was closed, but yield_any_line still may have |
| 170 | # something to yield. |
| 171 | continue |
| 172 | output.append(line) |
| 173 | if pipe_name == 'stderr': |
| 174 | logging.debug('cipd client: %s', line) |
| 175 | else: |
| 176 | logging.info('cipd client: %s', line) |
| 177 | |
| 178 | exit_code = process.wait(timeout=timeoutfn()) |
| 179 | if exit_code != 0: |
| 180 | raise Error( |
| 181 | 'Could not install packages; exit code %d\noutput:%s' % ( |
| 182 | exit_code, '\n'.join(output))) |
| 183 | finally: |
| 184 | fs.remove(list_file_path) |
| 185 | |
| 186 | |
| 187 | def get_platform(): |
| 188 | """Returns ${platform} parameter value. |
| 189 | |
| 190 | Borrowed from |
| 191 | https://chromium.googlesource.com/infra/infra/+/aaf9586/build/build.py#204 |
| 192 | """ |
| 193 | # linux, mac or windows. |
| 194 | platform_variant = { |
| 195 | 'darwin': 'mac', |
| 196 | 'linux2': 'linux', |
| 197 | 'win32': 'windows', |
| 198 | }.get(sys.platform) |
| 199 | if not platform_variant: |
| 200 | raise Error('Unknown OS: %s' % sys.platform) |
| 201 | |
| 202 | # amd64, 386, etc. |
| 203 | machine = platform.machine().lower() |
| 204 | platform_arch = { |
| 205 | 'amd64': 'amd64', |
| 206 | 'i386': '386', |
| 207 | 'i686': '386', |
| 208 | 'x86': '386', |
| 209 | 'x86_64': 'amd64', |
| 210 | }.get(machine) |
| 211 | if not platform_arch: |
| 212 | if machine.startswith('arm'): |
| 213 | platform_arch = 'armv6l' |
| 214 | else: |
| 215 | platform_arch = 'amd64' if sys.maxsize > 2**32 else '386' |
| 216 | return '%s-%s' % (platform_variant, platform_arch) |
| 217 | |
| 218 | |
| 219 | def get_os_ver(): |
| 220 | """Returns ${os_ver} parameter value. |
| 221 | |
| 222 | Examples: 'ubuntu14_04' or 'mac10_9' or 'win6_1'. |
| 223 | |
| 224 | Borrowed from |
| 225 | https://chromium.googlesource.com/infra/infra/+/aaf9586/build/build.py#204 |
| 226 | """ |
| 227 | if sys.platform == 'darwin': |
| 228 | # platform.mac_ver()[0] is '10.9.5'. |
| 229 | dist = platform.mac_ver()[0].split('.') |
| 230 | return 'mac%s_%s' % (dist[0], dist[1]) |
| 231 | |
| 232 | if sys.platform == 'linux2': |
| 233 | # platform.linux_distribution() is ('Ubuntu', '14.04', ...). |
| 234 | dist = platform.linux_distribution() |
| 235 | return '%s%s' % (dist[0].lower(), dist[1].replace('.', '_')) |
| 236 | |
| 237 | if sys.platform == 'win32': |
| 238 | # platform.version() is '6.1.7601'. |
| 239 | dist = platform.version().split('.') |
| 240 | return 'win%s_%s' % (dist[0], dist[1]) |
| 241 | raise Error('Unknown OS: %s' % sys.platform) |
| 242 | |
| 243 | |
| 244 | def render_package_name_template(template): |
| 245 | """Expands template variables in a CIPD package name template.""" |
| 246 | return (template |
| 247 | .lower() # Package names are always lower case |
| 248 | .replace('${platform}', get_platform()) |
| 249 | .replace('${os_ver}', get_os_ver())) |
| 250 | |
| 251 | |
nodir | be642ff | 2016-06-09 15:51:51 -0700 | [diff] [blame] | 252 | def _check_response(res, fmt, *args): |
| 253 | """Raises Error if response is bad.""" |
| 254 | if not res: |
| 255 | raise Error('%s: no response' % (fmt % args)) |
| 256 | |
| 257 | if res.get('status') != 'SUCCESS': |
| 258 | raise Error('%s: %s' % ( |
| 259 | fmt % args, |
| 260 | res.get('error_message') or 'status is %s' % res.get('status'))) |
| 261 | |
| 262 | |
| 263 | def resolve_version(cipd_server, package_name, version, timeout=None): |
| 264 | """Resolves a package instance version (e.g. a tag) to an instance id.""" |
| 265 | url = '%s/_ah/api/repo/v1/instance/resolve?%s' % ( |
| 266 | cipd_server, |
| 267 | urllib.urlencode({ |
| 268 | 'package_name': package_name, |
| 269 | 'version': version, |
| 270 | })) |
| 271 | res = net.url_read_json(url, timeout=timeout) |
| 272 | _check_response(res, 'Could not resolve version %s:%s', package_name, version) |
| 273 | instance_id = res.get('instance_id') |
| 274 | if not instance_id: |
| 275 | raise Error('Invalid resolveVersion response: no instance id') |
| 276 | return instance_id |
| 277 | |
| 278 | |
| 279 | def get_client_fetch_url(service_url, package_name, instance_id, timeout=None): |
| 280 | """Returns a fetch URL of CIPD client binary contents. |
| 281 | |
| 282 | Raises: |
| 283 | Error if cannot retrieve fetch URL. |
| 284 | """ |
| 285 | # Fetch the URL of the binary from CIPD backend. |
| 286 | package_name = render_package_name_template(package_name) |
| 287 | url = '%s/_ah/api/repo/v1/client?%s' % (service_url, urllib.urlencode({ |
| 288 | 'package_name': package_name, |
| 289 | 'instance_id': instance_id, |
| 290 | })) |
| 291 | res = net.url_read_json(url, timeout=timeout) |
| 292 | _check_response( |
| 293 | res, 'Could not fetch CIPD client %s:%s',package_name, instance_id) |
| 294 | fetch_url = res.get('client_binary', {}).get('fetch_url') |
| 295 | if not fetch_url: |
| 296 | raise Error('Invalid fetchClientBinary response: no fetch_url') |
| 297 | return fetch_url |
| 298 | |
| 299 | |
| 300 | def _fetch_cipd_client(disk_cache, instance_id, fetch_url, timeoutfn): |
| 301 | """Fetches cipd binary to |disk_cache|. |
| 302 | |
| 303 | Retries requests with exponential back-off. |
| 304 | |
| 305 | Raises: |
| 306 | Error if could not fetch content. |
| 307 | """ |
| 308 | sleep_time = 1 |
| 309 | for attempt in xrange(5): |
| 310 | if attempt > 0: |
| 311 | if timeoutfn() is not None and timeoutfn() < sleep_time: |
| 312 | raise Error('Could not fetch CIPD client: timeout') |
| 313 | logging.warning('Will retry to fetch CIPD client in %ds', sleep_time) |
| 314 | time.sleep(sleep_time) |
| 315 | sleep_time *= 2 |
| 316 | |
| 317 | try: |
| 318 | res = net.url_open(fetch_url, timeout=timeoutfn()) |
| 319 | if res: |
| 320 | disk_cache.write(instance_id, res.iter_content(64 * 1024)) |
| 321 | return |
| 322 | except net.TimeoutError as ex: |
| 323 | raise Error('Could not fetch CIPD client: %s', ex) |
| 324 | except net.NetError as ex: |
| 325 | logging.warning( |
| 326 | 'Could not fetch CIPD client on attempt #%d: %s', attempt + 1, ex) |
| 327 | |
| 328 | raise Error('Could not fetch CIPD client after 5 retries') |
| 329 | |
| 330 | |
| 331 | @contextlib.contextmanager |
| 332 | def get_client( |
| 333 | service_url, package_name, version, cache_dir, timeout=None): |
| 334 | """Returns a context manager that yields a CipdClient. A blocking call. |
| 335 | |
| 336 | Args: |
| 337 | service_url (str): URL of the CIPD backend. |
| 338 | package_name (str): package name template of the CIPD client. |
| 339 | version (str): version of CIPD client package. |
| 340 | cache_dir: directory to store instance cache, version cache |
| 341 | and a hardlink to the client binary. |
| 342 | timeout (int): if not None, timeout in seconds for this function. |
| 343 | |
| 344 | Yields: |
| 345 | CipdClient. |
| 346 | |
| 347 | Raises: |
| 348 | Error if CIPD client version cannot be resolved or client cannot be fetched. |
| 349 | """ |
| 350 | timeoutfn = tools.sliding_timeout(timeout) |
| 351 | |
| 352 | package_name = render_package_name_template(package_name) |
| 353 | |
| 354 | # Resolve version to instance id. |
| 355 | # Is it an instance id already? They look like HEX SHA1. |
| 356 | if isolated_format.is_valid_hash(version, hashlib.sha1): |
| 357 | instance_id = version |
| 358 | else: |
| 359 | # version_cache is {version_digest -> instance id} mapping. |
| 360 | # It does not take a lot of disk space. |
| 361 | version_cache = isolateserver.DiskCache( |
| 362 | unicode(os.path.join(cache_dir, 'versions')), |
| 363 | isolateserver.CachePolicies(0, 0, 300), |
| 364 | hashlib.sha1) |
| 365 | with version_cache: |
maruel | 2e8d0f5 | 2016-07-16 07:51:29 -0700 | [diff] [blame^] | 366 | version_cache.cleanup() |
nodir | be642ff | 2016-06-09 15:51:51 -0700 | [diff] [blame] | 367 | # Convert |version| to a string that may be used as a filename in disk |
| 368 | # cache by hashing it. |
| 369 | version_digest = hashlib.sha1(version).hexdigest() |
| 370 | try: |
| 371 | instance_id = version_cache.read(version_digest) |
| 372 | except isolateserver.CacheMiss: |
| 373 | instance_id = resolve_version( |
| 374 | service_url, package_name, version, timeout=timeoutfn()) |
| 375 | version_cache.write(version_digest, instance_id) |
| 376 | |
| 377 | # instance_cache is {instance_id -> client binary} mapping. |
| 378 | # It is bounded by 5 client versions. |
| 379 | instance_cache = isolateserver.DiskCache( |
| 380 | unicode(os.path.join(cache_dir, 'clients')), |
| 381 | isolateserver.CachePolicies(0, 0, 5), |
| 382 | hashlib.sha1) |
| 383 | with instance_cache: |
maruel | 2e8d0f5 | 2016-07-16 07:51:29 -0700 | [diff] [blame^] | 384 | instance_cache.cleanup() |
nodir | be642ff | 2016-06-09 15:51:51 -0700 | [diff] [blame] | 385 | if instance_id not in instance_cache: |
| 386 | logging.info('Fetching CIPD client %s:%s', package_name, instance_id) |
| 387 | fetch_url = get_client_fetch_url( |
| 388 | service_url, package_name, instance_id, timeout=timeoutfn()) |
| 389 | _fetch_cipd_client(instance_cache, instance_id, fetch_url, timeoutfn) |
| 390 | |
| 391 | # A single host can run multiple swarming bots, but ATM they do not share |
| 392 | # same root bot directory. Thus, it is safe to use the same name for the |
| 393 | # binary. |
| 394 | binary_path = unicode(os.path.join(cache_dir, 'cipd' + EXECUTABLE_SUFFIX)) |
| 395 | if fs.isfile(binary_path): |
nodir | 6dfdb2d | 2016-06-14 20:14:08 -0700 | [diff] [blame] | 396 | file_path.remove(binary_path) |
nodir | be642ff | 2016-06-09 15:51:51 -0700 | [diff] [blame] | 397 | instance_cache.hardlink(instance_id, binary_path, 0511) # -r-x--x--x |
| 398 | |
| 399 | yield CipdClient(binary_path) |
nodir | 90bc8dc | 2016-06-15 13:35:21 -0700 | [diff] [blame] | 400 | |
| 401 | |
nodir | ff531b4 | 2016-06-23 13:05:06 -0700 | [diff] [blame] | 402 | def parse_package_args(packages): |
| 403 | """Parses --cipd-package arguments. |
nodir | 90bc8dc | 2016-06-15 13:35:21 -0700 | [diff] [blame] | 404 | |
nodir | ff531b4 | 2016-06-23 13:05:06 -0700 | [diff] [blame] | 405 | Assumes |packages| were validated by validate_cipd_options. |
| 406 | |
| 407 | Returns: |
| 408 | A map {path: [(package, version)]}. |
nodir | 90bc8dc | 2016-06-15 13:35:21 -0700 | [diff] [blame] | 409 | """ |
nodir | ff531b4 | 2016-06-23 13:05:06 -0700 | [diff] [blame] | 410 | result = collections.defaultdict(list) |
| 411 | for pkg in packages: |
| 412 | path, name, version = pkg.split(':', 2) |
nodir | 90bc8dc | 2016-06-15 13:35:21 -0700 | [diff] [blame] | 413 | path = path.replace('/', os.path.sep) |
nodir | 90bc8dc | 2016-06-15 13:35:21 -0700 | [diff] [blame] | 414 | if not name: |
nodir | ff531b4 | 2016-06-23 13:05:06 -0700 | [diff] [blame] | 415 | raise Error('Invalid package "%s": package name is not specified' % pkg) |
nodir | 90bc8dc | 2016-06-15 13:35:21 -0700 | [diff] [blame] | 416 | if not version: |
nodir | ff531b4 | 2016-06-23 13:05:06 -0700 | [diff] [blame] | 417 | raise Error('Invalid package "%s": version is not specified' % pkg) |
| 418 | result[path].append((name, version)) |
| 419 | return result |