Mike Frysinger | e58c0e2 | 2017-10-04 15:43:30 -0400 | [diff] [blame] | 1 | # -*- coding: utf-8 -*- |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 2 | # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. |
| 3 | # Use of this source code is governed by a BSD-style license that can be |
| 4 | # found in the LICENSE file. |
| 5 | |
Mike Frysinger | 750c5f5 | 2014-09-16 16:16:57 -0400 | [diff] [blame] | 6 | """This script manages the installed toolchains in the chroot.""" |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 7 | |
Mike Frysinger | 383367e | 2014-09-16 15:06:17 -0400 | [diff] [blame] | 8 | from __future__ import print_function |
| 9 | |
Mike Frysinger | 3ed4772 | 2017-08-08 14:59:08 -0400 | [diff] [blame] | 10 | import errno |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 11 | import glob |
Mike Frysinger | 3ed4772 | 2017-08-08 14:59:08 -0400 | [diff] [blame] | 12 | import hashlib |
Mike Frysinger | 7ccee99 | 2012-06-01 21:27:59 -0400 | [diff] [blame] | 13 | import json |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 14 | import os |
Rahul Chaudhry | a8127bb | 2016-07-22 15:53:17 -0700 | [diff] [blame] | 15 | import re |
Tobias Bosch | ddd1649 | 2019-08-14 09:29:54 -0700 | [diff] [blame] | 16 | import shutil |
Mike Frysinger | 499ca39 | 2020-04-28 07:35:54 -0400 | [diff] [blame] | 17 | import sys |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 18 | |
Aviv Keshet | b7519e1 | 2016-10-04 00:50:00 -0700 | [diff] [blame] | 19 | from chromite.lib import constants |
Mike Frysinger | 506e75f | 2012-12-17 14:21:13 -0500 | [diff] [blame] | 20 | from chromite.lib import commandline |
Brian Harring | 503f3ab | 2012-03-09 21:39:41 -0800 | [diff] [blame] | 21 | from chromite.lib import cros_build_lib |
Ralph Nathan | 0304728 | 2015-03-23 11:09:32 -0700 | [diff] [blame] | 22 | from chromite.lib import cros_logging as logging |
Brian Harring | af019fb | 2012-05-10 15:06:13 -0700 | [diff] [blame] | 23 | from chromite.lib import osutils |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 24 | from chromite.lib import parallel |
David James | 27ac4ae | 2012-12-03 23:16:15 -0800 | [diff] [blame] | 25 | from chromite.lib import toolchain |
Mike Frysinger | e652ba1 | 2019-09-08 00:57:43 -0400 | [diff] [blame] | 26 | from chromite.utils import key_value_store |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 27 | |
| 28 | # Needs to be after chromite imports. |
Mike Frysinger | abb7d81 | 2020-05-15 00:13:10 -0400 | [diff] [blame] | 29 | import lddtree # pylint: disable=wrong-import-order |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 30 | |
Mike Frysinger | 3159600 | 2012-12-03 23:54:24 -0500 | [diff] [blame] | 31 | if cros_build_lib.IsInsideChroot(): |
| 32 | # Only import portage after we've checked that we're inside the chroot. |
| 33 | # Outside may not have portage, in which case the above may not happen. |
| 34 | # We'll check in main() if the operation needs portage. |
Mike Frysinger | 27e21b7 | 2018-07-12 14:20:21 -0400 | [diff] [blame] | 35 | # pylint: disable=import-error |
Mike Frysinger | 3159600 | 2012-12-03 23:54:24 -0500 | [diff] [blame] | 36 | import portage |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 37 | |
| 38 | |
Mike Frysinger | 499ca39 | 2020-04-28 07:35:54 -0400 | [diff] [blame] | 39 | assert sys.version_info >= (3, 6), 'This module requires Python 3.6+' |
| 40 | |
| 41 | |
Matt Tennant | f1e3097 | 2012-03-02 16:30:07 -0800 | [diff] [blame] | 42 | EMERGE_CMD = os.path.join(constants.CHROMITE_BIN_DIR, 'parallel_emerge') |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 43 | PACKAGE_STABLE = '[stable]' |
Zdenek Behan | 4eb6fd2 | 2012-03-12 17:00:56 +0100 | [diff] [blame] | 44 | |
| 45 | CHROMIUMOS_OVERLAY = '/usr/local/portage/chromiumos' |
Christopher Wiley | b22c071 | 2015-06-02 10:37:03 -0700 | [diff] [blame] | 46 | ECLASS_OVERLAY = '/usr/local/portage/eclass-overlay' |
Zdenek Behan | 4eb6fd2 | 2012-03-12 17:00:56 +0100 | [diff] [blame] | 47 | STABLE_OVERLAY = '/usr/local/portage/stable' |
| 48 | CROSSDEV_OVERLAY = '/usr/local/portage/crossdev' |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 49 | |
| 50 | |
Mike Frysinger | 66bfde5 | 2017-09-12 16:42:57 -0400 | [diff] [blame] | 51 | # The exact list of host toolchain packages we care about. These are the |
| 52 | # packages that bots/devs install only from binpkgs and rely on the SDK bot |
| 53 | # (chromiumos-sdk) to validate+uprev. |
| 54 | # |
Mike Frysinger | 66bfde5 | 2017-09-12 16:42:57 -0400 | [diff] [blame] | 55 | # We don't use crossdev to manage the host toolchain for us, especially since |
| 56 | # we diverge significantly now (with llvm/clang/etc...), and we don't need or |
| 57 | # want crossdev managing /etc/portage config files for the sdk |
| 58 | HOST_PACKAGES = ( |
| 59 | 'dev-lang/go', |
Yunlian Jiang | 2cb91dc | 2018-03-08 10:56:27 -0800 | [diff] [blame] | 60 | 'dev-libs/elfutils', |
Mike Frysinger | 66bfde5 | 2017-09-12 16:42:57 -0400 | [diff] [blame] | 61 | 'sys-devel/binutils', |
Mike Frysinger | 66bfde5 | 2017-09-12 16:42:57 -0400 | [diff] [blame] | 62 | 'sys-devel/gcc', |
| 63 | 'sys-devel/llvm', |
| 64 | 'sys-kernel/linux-headers', |
| 65 | 'sys-libs/glibc', |
| 66 | 'sys-libs/libcxx', |
| 67 | 'sys-libs/libcxxabi', |
Manoj Gupta | de64cb2 | 2019-03-31 18:48:58 -0700 | [diff] [blame] | 68 | 'sys-libs/llvm-libunwind', |
Mike Frysinger | 66bfde5 | 2017-09-12 16:42:57 -0400 | [diff] [blame] | 69 | ) |
| 70 | |
Mike Frysinger | 785b0c3 | 2017-09-13 01:35:59 -0400 | [diff] [blame] | 71 | # These packages are also installed into the host SDK. However, they require |
| 72 | # the cross-compilers to be installed first (because they need them to actually |
| 73 | # build), so we have to delay their installation. |
| 74 | HOST_POST_CROSS_PACKAGES = ( |
Manoj Gupta | 65f8844 | 2018-04-12 22:42:19 -0700 | [diff] [blame] | 75 | 'dev-lang/rust', |
George Burgess IV | 9d1e232 | 2021-04-30 09:54:45 -0700 | [diff] [blame] | 76 | 'dev-lang/rust-bootstrap', |
Mike Frysinger | 61a2439 | 2017-10-17 17:14:27 -0400 | [diff] [blame] | 77 | 'virtual/target-sdk-post-cross', |
Patrick Georgi | 043ce6e | 2019-02-20 22:27:09 +0100 | [diff] [blame] | 78 | 'dev-embedded/coreboot-sdk', |
George Burgess IV | 288ecc8 | 2021-04-21 07:52:45 -0700 | [diff] [blame] | 79 | 'dev-embedded/ti50-sdk', |
Mike Frysinger | 785b0c3 | 2017-09-13 01:35:59 -0400 | [diff] [blame] | 80 | ) |
| 81 | |
| 82 | # New packages that we're in the process of adding to the SDK. Since the SDK |
| 83 | # bot hasn't had a chance to run yet, there are no binary packages available, |
| 84 | # so we have to list them here and wait. Once it completes, entries here can |
| 85 | # be removed so they'll end up on bots & dev's systems. |
George Burgess IV | 66e199c | 2021-05-05 15:38:40 -0700 | [diff] [blame] | 86 | NEW_PACKAGES = () |
Mike Frysinger | 785b0c3 | 2017-09-13 01:35:59 -0400 | [diff] [blame] | 87 | |
Rahul Chaudhry | 4b80305 | 2015-05-13 15:25:56 -0700 | [diff] [blame] | 88 | # Enable the Go compiler for these targets. |
| 89 | TARGET_GO_ENABLED = ( |
| 90 | 'x86_64-cros-linux-gnu', |
Rahul Chaudhry | 4b80305 | 2015-05-13 15:25:56 -0700 | [diff] [blame] | 91 | 'armv7a-cros-linux-gnueabi', |
Yunlian Jiang | 16c1a42 | 2017-10-04 10:33:22 -0700 | [diff] [blame] | 92 | 'armv7a-cros-linux-gnueabihf', |
Rahul Chaudhry | 4d41658 | 2017-10-25 12:31:58 -0700 | [diff] [blame] | 93 | 'aarch64-cros-linux-gnu', |
Rahul Chaudhry | 4b80305 | 2015-05-13 15:25:56 -0700 | [diff] [blame] | 94 | ) |
| 95 | CROSSDEV_GO_ARGS = ['--ex-pkg', 'dev-lang/go'] |
| 96 | |
Manoj Gupta | 1b5642e | 2017-03-08 16:44:12 -0800 | [diff] [blame] | 97 | # Enable llvm's compiler-rt for these targets. |
| 98 | TARGET_COMPILER_RT_ENABLED = ( |
| 99 | 'armv7a-cros-linux-gnueabi', |
Yunlian Jiang | 1b77ee4 | 2017-10-06 13:44:29 -0700 | [diff] [blame] | 100 | 'armv7a-cros-linux-gnueabihf', |
Manoj Gupta | 946abb4 | 2017-04-12 14:27:19 -0700 | [diff] [blame] | 101 | 'aarch64-cros-linux-gnu', |
Manoj Gupta | 21f3a08 | 2018-03-06 21:25:39 -0800 | [diff] [blame] | 102 | 'armv7m-cros-eabi', |
Manoj Gupta | 1b5642e | 2017-03-08 16:44:12 -0800 | [diff] [blame] | 103 | ) |
| 104 | CROSSDEV_COMPILER_RT_ARGS = ['--ex-pkg', 'sys-libs/compiler-rt'] |
| 105 | |
Manoj Gupta | 946abb4 | 2017-04-12 14:27:19 -0700 | [diff] [blame] | 106 | TARGET_LLVM_PKGS_ENABLED = ( |
| 107 | 'armv7a-cros-linux-gnueabi', |
Yunlian Jiang | 16c1a42 | 2017-10-04 10:33:22 -0700 | [diff] [blame] | 108 | 'armv7a-cros-linux-gnueabihf', |
Manoj Gupta | 946abb4 | 2017-04-12 14:27:19 -0700 | [diff] [blame] | 109 | 'aarch64-cros-linux-gnu', |
Manoj Gupta | 9371312 | 2020-10-29 17:52:16 -0700 | [diff] [blame] | 110 | 'i686-pc-linux-gnu', |
Manoj Gupta | 946abb4 | 2017-04-12 14:27:19 -0700 | [diff] [blame] | 111 | 'x86_64-cros-linux-gnu', |
| 112 | ) |
| 113 | |
| 114 | LLVM_PKGS_TABLE = { |
Yunlian Jiang | bb2a3d3 | 2017-04-26 14:44:09 -0700 | [diff] [blame] | 115 | 'ex_libcxxabi' : ['--ex-pkg', 'sys-libs/libcxxabi'], |
| 116 | 'ex_libcxx' : ['--ex-pkg', 'sys-libs/libcxx'], |
Chirantan Ekbote | e694cad | 2018-07-12 15:07:24 -0700 | [diff] [blame] | 117 | 'ex_llvm-libunwind' : ['--ex-pkg', 'sys-libs/llvm-libunwind'], |
Manoj Gupta | 946abb4 | 2017-04-12 14:27:19 -0700 | [diff] [blame] | 118 | } |
| 119 | |
David James | 66a09c4 | 2012-11-05 13:31:38 -0800 | [diff] [blame] | 120 | class Crossdev(object): |
| 121 | """Class for interacting with crossdev and caching its output.""" |
| 122 | |
| 123 | _CACHE_FILE = os.path.join(CROSSDEV_OVERLAY, '.configured.json') |
| 124 | _CACHE = {} |
Han Shen | e23782f | 2016-02-18 12:20:00 -0800 | [diff] [blame] | 125 | # Packages that needs separate handling, in addition to what we have from |
| 126 | # crossdev. |
| 127 | MANUAL_PKGS = { |
George Burgess IV | ca1d761 | 2020-10-01 00:38:32 -0700 | [diff] [blame] | 128 | 'rust': 'dev-lang', |
Han Shen | e23782f | 2016-02-18 12:20:00 -0800 | [diff] [blame] | 129 | 'llvm': 'sys-devel', |
Manoj Gupta | 20cfc6c | 2017-07-19 15:49:55 -0700 | [diff] [blame] | 130 | 'libcxxabi': 'sys-libs', |
| 131 | 'libcxx': 'sys-libs', |
Yunlian Jiang | da3ce5f | 2018-04-25 14:10:01 -0700 | [diff] [blame] | 132 | 'elfutils': 'dev-libs', |
Chirantan Ekbote | e694cad | 2018-07-12 15:07:24 -0700 | [diff] [blame] | 133 | 'llvm-libunwind': 'sys-libs', |
Han Shen | e23782f | 2016-02-18 12:20:00 -0800 | [diff] [blame] | 134 | } |
David James | 66a09c4 | 2012-11-05 13:31:38 -0800 | [diff] [blame] | 135 | |
| 136 | @classmethod |
| 137 | def Load(cls, reconfig): |
Mike Frysinger | 3ed4772 | 2017-08-08 14:59:08 -0400 | [diff] [blame] | 138 | """Load crossdev cache from disk. |
| 139 | |
| 140 | We invalidate the cache when crossdev updates or this script changes. |
| 141 | """ |
David James | 90239b9 | 2012-11-05 15:31:34 -0800 | [diff] [blame] | 142 | crossdev_version = GetStablePackageVersion('sys-devel/crossdev', True) |
Mike Frysinger | 3ed4772 | 2017-08-08 14:59:08 -0400 | [diff] [blame] | 143 | # If we run the compiled/cached .pyc file, we'll read/hash that when we |
| 144 | # really always want to track the source .py file. |
| 145 | script = os.path.abspath(__file__) |
| 146 | if script.endswith('.pyc'): |
| 147 | script = script[:-1] |
Mike Frysinger | b3202be | 2019-11-15 22:25:59 -0500 | [diff] [blame] | 148 | setup_toolchains_hash = hashlib.md5( |
| 149 | osutils.ReadFile(script, mode='rb')).hexdigest() |
Mike Frysinger | 3ed4772 | 2017-08-08 14:59:08 -0400 | [diff] [blame] | 150 | |
| 151 | cls._CACHE = { |
| 152 | 'crossdev_version': crossdev_version, |
| 153 | 'setup_toolchains_hash': setup_toolchains_hash, |
| 154 | } |
| 155 | |
| 156 | logging.debug('cache: checking file: %s', cls._CACHE_FILE) |
| 157 | if reconfig: |
| 158 | logging.debug('cache: forcing regen due to reconfig') |
| 159 | return |
| 160 | |
| 161 | try: |
| 162 | file_data = osutils.ReadFile(cls._CACHE_FILE) |
| 163 | except IOError as e: |
| 164 | if e.errno != errno.ENOENT: |
| 165 | logging.warning('cache: reading failed: %s', e) |
| 166 | osutils.SafeUnlink(cls._CACHE_FILE) |
| 167 | return |
| 168 | |
| 169 | try: |
| 170 | data = json.loads(file_data) |
| 171 | except ValueError as e: |
| 172 | logging.warning('cache: ignoring invalid content: %s', e) |
| 173 | return |
| 174 | |
| 175 | if crossdev_version != data.get('crossdev_version'): |
| 176 | logging.debug('cache: rebuilding after crossdev upgrade') |
| 177 | elif setup_toolchains_hash != data.get('setup_toolchains_hash'): |
| 178 | logging.debug('cache: rebuilding after cros_setup_toolchains change') |
| 179 | else: |
| 180 | logging.debug('cache: content is up-to-date!') |
| 181 | cls._CACHE = data |
David James | 66a09c4 | 2012-11-05 13:31:38 -0800 | [diff] [blame] | 182 | |
| 183 | @classmethod |
| 184 | def Save(cls): |
| 185 | """Store crossdev cache on disk.""" |
| 186 | # Save the cache from the successful run. |
| 187 | with open(cls._CACHE_FILE, 'w') as f: |
| 188 | json.dump(cls._CACHE, f) |
| 189 | |
| 190 | @classmethod |
| 191 | def GetConfig(cls, target): |
| 192 | """Returns a map of crossdev provided variables about a tuple.""" |
| 193 | CACHE_ATTR = '_target_tuple_map' |
| 194 | |
| 195 | val = cls._CACHE.setdefault(CACHE_ATTR, {}) |
| 196 | if not target in val: |
Mike Frysinger | 785b0c3 | 2017-09-13 01:35:59 -0400 | [diff] [blame] | 197 | if target.startswith('host'): |
Mike Frysinger | 66bfde5 | 2017-09-12 16:42:57 -0400 | [diff] [blame] | 198 | conf = { |
| 199 | 'crosspkgs': [], |
| 200 | 'target': toolchain.GetHostTuple(), |
| 201 | } |
Mike Frysinger | 785b0c3 | 2017-09-13 01:35:59 -0400 | [diff] [blame] | 202 | if target == 'host': |
| 203 | packages_list = HOST_PACKAGES |
| 204 | else: |
| 205 | packages_list = HOST_POST_CROSS_PACKAGES |
Mike Frysinger | 66bfde5 | 2017-09-12 16:42:57 -0400 | [diff] [blame] | 206 | manual_pkgs = dict((pkg, cat) for cat, pkg in |
Mike Frysinger | 785b0c3 | 2017-09-13 01:35:59 -0400 | [diff] [blame] | 207 | [x.split('/') for x in packages_list]) |
Mike Frysinger | 66bfde5 | 2017-09-12 16:42:57 -0400 | [diff] [blame] | 208 | else: |
| 209 | # Build the crossdev command. |
| 210 | cmd = ['crossdev', '--show-target-cfg', '--ex-gdb'] |
| 211 | if target in TARGET_COMPILER_RT_ENABLED: |
| 212 | cmd.extend(CROSSDEV_COMPILER_RT_ARGS) |
| 213 | if target in TARGET_GO_ENABLED: |
| 214 | cmd.extend(CROSSDEV_GO_ARGS) |
| 215 | if target in TARGET_LLVM_PKGS_ENABLED: |
| 216 | for pkg in LLVM_PKGS_TABLE: |
| 217 | cmd.extend(LLVM_PKGS_TABLE[pkg]) |
| 218 | cmd.extend(['-t', target]) |
| 219 | # Catch output of crossdev. |
Mike Frysinger | 45602c7 | 2019-09-22 02:15:11 -0400 | [diff] [blame] | 220 | out = cros_build_lib.run( |
Mike Frysinger | 0282d22 | 2019-12-17 17:15:48 -0500 | [diff] [blame] | 221 | cmd, print_cmd=False, stdout=True, |
Mike Frysinger | b3202be | 2019-11-15 22:25:59 -0500 | [diff] [blame] | 222 | encoding='utf-8').stdout.splitlines() |
Mike Frysinger | 66bfde5 | 2017-09-12 16:42:57 -0400 | [diff] [blame] | 223 | # List of tuples split at the first '=', converted into dict. |
| 224 | conf = dict((k, cros_build_lib.ShellUnquote(v)) |
| 225 | for k, v in (x.split('=', 1) for x in out)) |
| 226 | conf['crosspkgs'] = conf['crosspkgs'].split() |
Han Shen | e23782f | 2016-02-18 12:20:00 -0800 | [diff] [blame] | 227 | |
Mike Frysinger | 66bfde5 | 2017-09-12 16:42:57 -0400 | [diff] [blame] | 228 | manual_pkgs = cls.MANUAL_PKGS |
| 229 | |
| 230 | for pkg, cat in manual_pkgs.items(): |
Mike Frysinger | 3bba503 | 2016-09-20 14:15:04 -0400 | [diff] [blame] | 231 | conf[pkg + '_pn'] = pkg |
| 232 | conf[pkg + '_category'] = cat |
| 233 | if pkg not in conf['crosspkgs']: |
| 234 | conf['crosspkgs'].append(pkg) |
Han Shen | e23782f | 2016-02-18 12:20:00 -0800 | [diff] [blame] | 235 | |
| 236 | val[target] = conf |
| 237 | |
David James | 66a09c4 | 2012-11-05 13:31:38 -0800 | [diff] [blame] | 238 | return val[target] |
| 239 | |
| 240 | @classmethod |
| 241 | def UpdateTargets(cls, targets, usepkg, config_only=False): |
| 242 | """Calls crossdev to initialize a cross target. |
| 243 | |
| 244 | Args: |
Nicolas Boichat | 3fecf39 | 2019-11-25 02:58:28 +0000 | [diff] [blame] | 245 | targets: The list of targets to initialize using crossdev. |
Don Garrett | 25f309a | 2014-03-19 14:02:12 -0700 | [diff] [blame] | 246 | usepkg: Copies the commandline opts. |
| 247 | config_only: Just update. |
David James | 66a09c4 | 2012-11-05 13:31:38 -0800 | [diff] [blame] | 248 | """ |
| 249 | configured_targets = cls._CACHE.setdefault('configured_targets', []) |
| 250 | |
| 251 | cmdbase = ['crossdev', '--show-fail-log'] |
| 252 | cmdbase.extend(['--env', 'FEATURES=splitdebug']) |
| 253 | # Pick stable by default, and override as necessary. |
| 254 | cmdbase.extend(['-P', '--oneshot']) |
| 255 | if usepkg: |
| 256 | cmdbase.extend(['-P', '--getbinpkg', |
| 257 | '-P', '--usepkgonly', |
| 258 | '--without-headers']) |
| 259 | |
Christopher Wiley | b22c071 | 2015-06-02 10:37:03 -0700 | [diff] [blame] | 260 | overlays = ' '.join((CHROMIUMOS_OVERLAY, ECLASS_OVERLAY, STABLE_OVERLAY)) |
David James | 66a09c4 | 2012-11-05 13:31:38 -0800 | [diff] [blame] | 261 | cmdbase.extend(['--overlays', overlays]) |
| 262 | cmdbase.extend(['--ov-output', CROSSDEV_OVERLAY]) |
| 263 | |
Nicolas Boichat | 3fecf39 | 2019-11-25 02:58:28 +0000 | [diff] [blame] | 264 | # Build target by the reversed alphabetical order to make sure |
| 265 | # armv7a-cros-linux-gnueabihf builds before armv7a-cros-linux-gnueabi |
| 266 | # because some dependency issue. This can be reverted once we |
| 267 | # migrated to armv7a-cros-linux-gnueabihf. crbug.com/711369 |
| 268 | for target in sorted(targets, reverse=True): |
| 269 | if config_only and target in configured_targets: |
| 270 | continue |
David James | 66a09c4 | 2012-11-05 13:31:38 -0800 | [diff] [blame] | 271 | |
Nicolas Boichat | 3fecf39 | 2019-11-25 02:58:28 +0000 | [diff] [blame] | 272 | cmd = cmdbase + ['-t', target] |
| 273 | |
| 274 | for pkg in GetTargetPackages(target): |
| 275 | if pkg == 'gdb': |
| 276 | # Gdb does not have selectable versions. |
| 277 | cmd.append('--ex-gdb') |
| 278 | elif pkg == 'ex_compiler-rt': |
| 279 | cmd.extend(CROSSDEV_COMPILER_RT_ARGS) |
| 280 | elif pkg == 'ex_go': |
| 281 | # Go does not have selectable versions. |
| 282 | cmd.extend(CROSSDEV_GO_ARGS) |
| 283 | elif pkg in LLVM_PKGS_TABLE: |
| 284 | cmd.extend(LLVM_PKGS_TABLE[pkg]) |
| 285 | elif pkg in cls.MANUAL_PKGS: |
| 286 | pass |
| 287 | else: |
| 288 | # The first of the desired versions is the "primary" one. |
| 289 | version = GetDesiredPackageVersions(target, pkg)[0] |
| 290 | cmd.extend(['--%s' % pkg, version]) |
| 291 | |
| 292 | cmd.extend(targets[target]['crossdev'].split()) |
| 293 | if config_only: |
| 294 | # In this case we want to just quietly reinit |
| 295 | cmd.append('--init-target') |
Mike Frysinger | 0282d22 | 2019-12-17 17:15:48 -0500 | [diff] [blame] | 296 | cros_build_lib.run(cmd, print_cmd=False, stdout=True) |
David James | 66a09c4 | 2012-11-05 13:31:38 -0800 | [diff] [blame] | 297 | else: |
Nicolas Boichat | 3fecf39 | 2019-11-25 02:58:28 +0000 | [diff] [blame] | 298 | cros_build_lib.run(cmd) |
David James | 66a09c4 | 2012-11-05 13:31:38 -0800 | [diff] [blame] | 299 | |
Nicolas Boichat | 3fecf39 | 2019-11-25 02:58:28 +0000 | [diff] [blame] | 300 | configured_targets.append(target) |
David James | 66a09c4 | 2012-11-05 13:31:38 -0800 | [diff] [blame] | 301 | |
Zdenek Behan | 4eb6fd2 | 2012-03-12 17:00:56 +0100 | [diff] [blame] | 302 | |
Zdenek Behan | f4d18a0 | 2012-03-22 15:45:05 +0100 | [diff] [blame] | 303 | def GetTargetPackages(target): |
| 304 | """Returns a list of packages for a given target.""" |
David James | 66a09c4 | 2012-11-05 13:31:38 -0800 | [diff] [blame] | 305 | conf = Crossdev.GetConfig(target) |
Zdenek Behan | f4d18a0 | 2012-03-22 15:45:05 +0100 | [diff] [blame] | 306 | # Undesired packages are denoted by empty ${pkg}_pn variable. |
Han Shen | e23782f | 2016-02-18 12:20:00 -0800 | [diff] [blame] | 307 | return [x for x in conf['crosspkgs'] if conf.get(x+'_pn')] |
Zdenek Behan | f4d18a0 | 2012-03-22 15:45:05 +0100 | [diff] [blame] | 308 | |
| 309 | |
Zdenek Behan | 4eb6fd2 | 2012-03-12 17:00:56 +0100 | [diff] [blame] | 310 | # Portage helper functions: |
| 311 | def GetPortagePackage(target, package): |
| 312 | """Returns a package name for the given target.""" |
David James | 66a09c4 | 2012-11-05 13:31:38 -0800 | [diff] [blame] | 313 | conf = Crossdev.GetConfig(target) |
Zdenek Behan | 4eb6fd2 | 2012-03-12 17:00:56 +0100 | [diff] [blame] | 314 | # Portage category: |
Mike Frysinger | 785b0c3 | 2017-09-13 01:35:59 -0400 | [diff] [blame] | 315 | if target.startswith('host') or package in Crossdev.MANUAL_PKGS: |
Zdenek Behan | 4eb6fd2 | 2012-03-12 17:00:56 +0100 | [diff] [blame] | 316 | category = conf[package + '_category'] |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 317 | else: |
Zdenek Behan | 4eb6fd2 | 2012-03-12 17:00:56 +0100 | [diff] [blame] | 318 | category = conf['category'] |
| 319 | # Portage package: |
| 320 | pn = conf[package + '_pn'] |
| 321 | # Final package name: |
Mike Frysinger | 422faf4 | 2014-11-12 23:16:48 -0500 | [diff] [blame] | 322 | assert category |
| 323 | assert pn |
Zdenek Behan | 4eb6fd2 | 2012-03-12 17:00:56 +0100 | [diff] [blame] | 324 | return '%s/%s' % (category, pn) |
| 325 | |
| 326 | |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 327 | def PortageTrees(root): |
| 328 | """Return the portage trees for a given root.""" |
| 329 | if root == '/': |
| 330 | return portage.db['/'] |
| 331 | # The portage logic requires the path always end in a slash. |
| 332 | root = root.rstrip('/') + '/' |
| 333 | return portage.create_trees(target_root=root, config_root=root)[root] |
| 334 | |
| 335 | |
| 336 | def GetInstalledPackageVersions(atom, root='/'): |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 337 | """Extracts the list of current versions of a target, package pair. |
| 338 | |
Mike Frysinger | dc649ae | 2014-11-26 19:29:24 -0500 | [diff] [blame] | 339 | Args: |
| 340 | atom: The atom to operate on (e.g. sys-devel/gcc) |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 341 | root: The root to check for installed packages. |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 342 | |
Mike Frysinger | dc649ae | 2014-11-26 19:29:24 -0500 | [diff] [blame] | 343 | Returns: |
| 344 | The list of versions of the package currently installed. |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 345 | """ |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 346 | versions = [] |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 347 | for pkg in PortageTrees(root)['vartree'].dbapi.match(atom, use_cache=0): |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 348 | version = portage.versions.cpv_getversion(pkg) |
| 349 | versions.append(version) |
| 350 | return versions |
| 351 | |
| 352 | |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 353 | def GetStablePackageVersion(atom, installed, root='/'): |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 354 | """Extracts the current stable version for a given package. |
| 355 | |
Mike Frysinger | dc649ae | 2014-11-26 19:29:24 -0500 | [diff] [blame] | 356 | Args: |
| 357 | atom: The target/package to operate on eg. i686-pc-linux-gnu,gcc |
| 358 | installed: Whether we want installed packages or ebuilds |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 359 | root: The root to use when querying packages. |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 360 | |
Mike Frysinger | dc649ae | 2014-11-26 19:29:24 -0500 | [diff] [blame] | 361 | Returns: |
| 362 | A string containing the latest version. |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 363 | """ |
David James | 90239b9 | 2012-11-05 15:31:34 -0800 | [diff] [blame] | 364 | pkgtype = 'vartree' if installed else 'porttree' |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 365 | cpv = portage.best(PortageTrees(root)[pkgtype].dbapi.match(atom, use_cache=0)) |
David James | 90239b9 | 2012-11-05 15:31:34 -0800 | [diff] [blame] | 366 | return portage.versions.cpv_getversion(cpv) if cpv else None |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 367 | |
| 368 | |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 369 | def VersionListToNumeric(target, package, versions, installed, root='/'): |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 370 | """Resolves keywords in a given version list for a particular package. |
| 371 | |
| 372 | Resolving means replacing PACKAGE_STABLE with the actual number. |
| 373 | |
Mike Frysinger | dc649ae | 2014-11-26 19:29:24 -0500 | [diff] [blame] | 374 | Args: |
| 375 | target: The target to operate on (e.g. i686-pc-linux-gnu) |
| 376 | package: The target/package to operate on (e.g. gcc) |
| 377 | versions: List of versions to resolve |
| 378 | installed: Query installed packages |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 379 | root: The install root to use; ignored if |installed| is False. |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 380 | |
Mike Frysinger | dc649ae | 2014-11-26 19:29:24 -0500 | [diff] [blame] | 381 | Returns: |
| 382 | List of purely numeric versions equivalent to argument |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 383 | """ |
| 384 | resolved = [] |
David James | 90239b9 | 2012-11-05 15:31:34 -0800 | [diff] [blame] | 385 | atom = GetPortagePackage(target, package) |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 386 | if not installed: |
| 387 | root = '/' |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 388 | for version in versions: |
| 389 | if version == PACKAGE_STABLE: |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 390 | resolved.append(GetStablePackageVersion(atom, installed, root=root)) |
Mike Frysinger | 7f0e198 | 2017-09-12 17:08:50 -0400 | [diff] [blame] | 391 | else: |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 392 | resolved.append(version) |
| 393 | return resolved |
| 394 | |
| 395 | |
| 396 | def GetDesiredPackageVersions(target, package): |
| 397 | """Produces the list of desired versions for each target, package pair. |
| 398 | |
| 399 | The first version in the list is implicitly treated as primary, ie. |
| 400 | the version that will be initialized by crossdev and selected. |
| 401 | |
| 402 | If the version is PACKAGE_STABLE, it really means the current version which |
| 403 | is emerged by using the package atom with no particular version key. |
| 404 | Since crossdev unmasks all packages by default, this will actually |
| 405 | mean 'unstable' in most cases. |
| 406 | |
Mike Frysinger | dc649ae | 2014-11-26 19:29:24 -0500 | [diff] [blame] | 407 | Args: |
| 408 | target: The target to operate on (e.g. i686-pc-linux-gnu) |
| 409 | package: The target/package to operate on (e.g. gcc) |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 410 | |
Mike Frysinger | dc649ae | 2014-11-26 19:29:24 -0500 | [diff] [blame] | 411 | Returns: |
| 412 | A list composed of either a version string, PACKAGE_STABLE |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 413 | """ |
Mike Frysinger | d23be27 | 2017-09-12 17:18:44 -0400 | [diff] [blame] | 414 | if package in GetTargetPackages(target): |
| 415 | return [PACKAGE_STABLE] |
| 416 | else: |
| 417 | return [] |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 418 | |
| 419 | |
| 420 | def TargetIsInitialized(target): |
| 421 | """Verifies if the given list of targets has been correctly initialized. |
| 422 | |
| 423 | This determines whether we have to call crossdev while emerging |
| 424 | toolchain packages or can do it using emerge. Emerge is naturally |
| 425 | preferred, because all packages can be updated in a single pass. |
| 426 | |
Mike Frysinger | dc649ae | 2014-11-26 19:29:24 -0500 | [diff] [blame] | 427 | Args: |
| 428 | target: The target to operate on (e.g. i686-pc-linux-gnu) |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 429 | |
Mike Frysinger | dc649ae | 2014-11-26 19:29:24 -0500 | [diff] [blame] | 430 | Returns: |
| 431 | True if |target| is completely initialized, else False |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 432 | """ |
| 433 | # Check if packages for the given target all have a proper version. |
| 434 | try: |
Zdenek Behan | f4d18a0 | 2012-03-22 15:45:05 +0100 | [diff] [blame] | 435 | for package in GetTargetPackages(target): |
David James | 66a09c4 | 2012-11-05 13:31:38 -0800 | [diff] [blame] | 436 | atom = GetPortagePackage(target, package) |
Zdenek Behan | 4eb6fd2 | 2012-03-12 17:00:56 +0100 | [diff] [blame] | 437 | # Do we even want this package && is it initialized? |
Mike Frysinger | 7f0e198 | 2017-09-12 17:08:50 -0400 | [diff] [blame] | 438 | if not (GetStablePackageVersion(atom, True) and |
| 439 | GetStablePackageVersion(atom, False)): |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 440 | return False |
| 441 | return True |
| 442 | except cros_build_lib.RunCommandError: |
| 443 | # Fails - The target has likely never been initialized before. |
| 444 | return False |
| 445 | |
| 446 | |
| 447 | def RemovePackageMask(target): |
| 448 | """Removes a package.mask file for the given platform. |
| 449 | |
| 450 | The pre-existing package.mask files can mess with the keywords. |
| 451 | |
Mike Frysinger | dc649ae | 2014-11-26 19:29:24 -0500 | [diff] [blame] | 452 | Args: |
| 453 | target: The target to operate on (e.g. i686-pc-linux-gnu) |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 454 | """ |
| 455 | maskfile = os.path.join('/etc/portage/package.mask', 'cross-' + target) |
Brian Harring | af019fb | 2012-05-10 15:06:13 -0700 | [diff] [blame] | 456 | osutils.SafeUnlink(maskfile) |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 457 | |
| 458 | |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 459 | # Main functions performing the actual update steps. |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 460 | def RebuildLibtool(root='/'): |
Mike Frysinger | c880a96 | 2013-11-08 13:59:06 -0500 | [diff] [blame] | 461 | """Rebuild libtool as needed |
| 462 | |
| 463 | Libtool hardcodes full paths to internal gcc files, so whenever we upgrade |
| 464 | gcc, libtool will break. We can't use binary packages either as those will |
| 465 | most likely be compiled against the previous version of gcc. |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 466 | |
| 467 | Args: |
| 468 | root: The install root where we want libtool rebuilt. |
Mike Frysinger | c880a96 | 2013-11-08 13:59:06 -0500 | [diff] [blame] | 469 | """ |
| 470 | needs_update = False |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 471 | with open(os.path.join(root, 'usr/bin/libtool')) as f: |
Mike Frysinger | c880a96 | 2013-11-08 13:59:06 -0500 | [diff] [blame] | 472 | for line in f: |
| 473 | # Look for a line like: |
| 474 | # sys_lib_search_path_spec="..." |
| 475 | # It'll be a list of paths and gcc will be one of them. |
| 476 | if line.startswith('sys_lib_search_path_spec='): |
| 477 | line = line.rstrip() |
| 478 | for path in line.split('=', 1)[1].strip('"').split(): |
Mike Frysinger | 3bba503 | 2016-09-20 14:15:04 -0400 | [diff] [blame] | 479 | root_path = os.path.join(root, path.lstrip(os.path.sep)) |
| 480 | logging.debug('Libtool: checking %s', root_path) |
| 481 | if not os.path.exists(root_path): |
| 482 | logging.info('Rebuilding libtool after gcc upgrade') |
| 483 | logging.info(' %s', line) |
| 484 | logging.info(' missing path: %s', path) |
Mike Frysinger | c880a96 | 2013-11-08 13:59:06 -0500 | [diff] [blame] | 485 | needs_update = True |
| 486 | break |
| 487 | |
| 488 | if needs_update: |
| 489 | break |
| 490 | |
| 491 | if needs_update: |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 492 | cmd = [EMERGE_CMD, '--oneshot'] |
| 493 | if root != '/': |
| 494 | cmd.extend(['--sysroot=%s' % root, '--root=%s' % root]) |
| 495 | cmd.append('sys-devel/libtool') |
Mike Frysinger | 45602c7 | 2019-09-22 02:15:11 -0400 | [diff] [blame] | 496 | cros_build_lib.run(cmd) |
Mike Frysinger | 3bba503 | 2016-09-20 14:15:04 -0400 | [diff] [blame] | 497 | else: |
| 498 | logging.debug('Libtool is up-to-date; no need to rebuild') |
Mike Frysinger | c880a96 | 2013-11-08 13:59:06 -0500 | [diff] [blame] | 499 | |
| 500 | |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 501 | def UpdateTargets(targets, usepkg, root='/'): |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 502 | """Determines which packages need update/unmerge and defers to portage. |
| 503 | |
Mike Frysinger | dc649ae | 2014-11-26 19:29:24 -0500 | [diff] [blame] | 504 | Args: |
| 505 | targets: The list of targets to update |
| 506 | usepkg: Copies the commandline option |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 507 | root: The install root in which we want packages updated. |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 508 | """ |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 509 | # For each target, we do two things. Figure out the list of updates, |
| 510 | # and figure out the appropriate keywords/masks. Crossdev will initialize |
| 511 | # these, but they need to be regenerated on every update. |
Mike Frysinger | 3bba503 | 2016-09-20 14:15:04 -0400 | [diff] [blame] | 512 | logging.info('Determining required toolchain updates...') |
David James | 90239b9 | 2012-11-05 15:31:34 -0800 | [diff] [blame] | 513 | mergemap = {} |
ChromeOS Developer | eb8a5c4 | 2021-01-12 00:05:02 +0000 | [diff] [blame] | 514 | # Used to keep track of post-cross packages. These are allowed to have |
| 515 | # implicit dependencies on toolchain packages, and therefore need to |
| 516 | # be built last. |
| 517 | post_cross_pkgs = set() |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 518 | for target in targets: |
ChromeOS Developer | eb8a5c4 | 2021-01-12 00:05:02 +0000 | [diff] [blame] | 519 | is_post_cross_target = target.endswith('-post-cross') |
Mike Frysinger | 3bba503 | 2016-09-20 14:15:04 -0400 | [diff] [blame] | 520 | logging.debug('Updating target %s', target) |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 521 | # Record the highest needed version for each target, for masking purposes. |
| 522 | RemovePackageMask(target) |
Zdenek Behan | f4d18a0 | 2012-03-22 15:45:05 +0100 | [diff] [blame] | 523 | for package in GetTargetPackages(target): |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 524 | # Portage name for the package |
Mike Frysinger | 7f0e198 | 2017-09-12 17:08:50 -0400 | [diff] [blame] | 525 | logging.debug(' Checking package %s', package) |
Zdenek Behan | 4eb6fd2 | 2012-03-12 17:00:56 +0100 | [diff] [blame] | 526 | pkg = GetPortagePackage(target, package) |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 527 | current = GetInstalledPackageVersions(pkg, root=root) |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 528 | desired = GetDesiredPackageVersions(target, package) |
Zdenek Behan | 699ddd3 | 2012-04-13 07:14:08 +0200 | [diff] [blame] | 529 | desired_num = VersionListToNumeric(target, package, desired, False) |
Mike Frysinger | 785b0c3 | 2017-09-13 01:35:59 -0400 | [diff] [blame] | 530 | if pkg in NEW_PACKAGES and usepkg: |
| 531 | # Skip this binary package (for now). |
| 532 | continue |
Zdenek Behan | 4eb6fd2 | 2012-03-12 17:00:56 +0100 | [diff] [blame] | 533 | mergemap[pkg] = set(desired_num).difference(current) |
Mike Frysinger | 7f0e198 | 2017-09-12 17:08:50 -0400 | [diff] [blame] | 534 | logging.debug(' %s -> %s', current, desired_num) |
ChromeOS Developer | eb8a5c4 | 2021-01-12 00:05:02 +0000 | [diff] [blame] | 535 | if is_post_cross_target: |
| 536 | post_cross_pkgs.add(pkg) |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 537 | |
Mike Frysinger | 7f0e198 | 2017-09-12 17:08:50 -0400 | [diff] [blame] | 538 | packages = [pkg for pkg, vers in mergemap.items() if vers] |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 539 | if not packages: |
Mike Frysinger | 3bba503 | 2016-09-20 14:15:04 -0400 | [diff] [blame] | 540 | logging.info('Nothing to update!') |
David James | f8c672f | 2012-11-06 13:38:11 -0800 | [diff] [blame] | 541 | return False |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 542 | |
Mike Frysinger | 3bba503 | 2016-09-20 14:15:04 -0400 | [diff] [blame] | 543 | logging.info('Updating packages:') |
| 544 | logging.info('%s', packages) |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 545 | |
Zdenek Behan | 7e33b4e | 2012-03-12 17:00:56 +0100 | [diff] [blame] | 546 | cmd = [EMERGE_CMD, '--oneshot', '--update'] |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 547 | if usepkg: |
| 548 | cmd.extend(['--getbinpkg', '--usepkgonly']) |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 549 | if root != '/': |
| 550 | cmd.extend(['--sysroot=%s' % root, '--root=%s' % root]) |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 551 | |
ChromeOS Developer | eb8a5c4 | 2021-01-12 00:05:02 +0000 | [diff] [blame] | 552 | if usepkg: |
| 553 | # Since we are not building from source, we can handle |
| 554 | # all packages in one go. |
| 555 | cmd.extend(packages) |
| 556 | cros_build_lib.run(cmd) |
| 557 | else: |
George Burgess IV | d7762ab | 2021-04-29 10:54:55 -0700 | [diff] [blame] | 558 | pre_cross_items = [pkg for pkg in packages if pkg not in post_cross_pkgs] |
| 559 | if pre_cross_items: |
| 560 | cros_build_lib.run(cmd + pre_cross_items) |
ChromeOS Developer | eb8a5c4 | 2021-01-12 00:05:02 +0000 | [diff] [blame] | 561 | post_cross_items = [pkg for pkg in packages if pkg in post_cross_pkgs] |
George Burgess IV | d7762ab | 2021-04-29 10:54:55 -0700 | [diff] [blame] | 562 | if post_cross_items: |
| 563 | cros_build_lib.run(cmd + post_cross_items) |
David James | f8c672f | 2012-11-06 13:38:11 -0800 | [diff] [blame] | 564 | return True |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 565 | |
| 566 | |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 567 | def CleanTargets(targets, root='/'): |
| 568 | """Unmerges old packages that are assumed unnecessary. |
| 569 | |
| 570 | Args: |
| 571 | targets: The list of targets to clean up. |
| 572 | root: The install root in which we want packages cleaned up. |
| 573 | """ |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 574 | unmergemap = {} |
| 575 | for target in targets: |
Mike Frysinger | 3bba503 | 2016-09-20 14:15:04 -0400 | [diff] [blame] | 576 | logging.debug('Cleaning target %s', target) |
Zdenek Behan | f4d18a0 | 2012-03-22 15:45:05 +0100 | [diff] [blame] | 577 | for package in GetTargetPackages(target): |
Mike Frysinger | 3bba503 | 2016-09-20 14:15:04 -0400 | [diff] [blame] | 578 | logging.debug(' Cleaning package %s', package) |
Zdenek Behan | 4eb6fd2 | 2012-03-12 17:00:56 +0100 | [diff] [blame] | 579 | pkg = GetPortagePackage(target, package) |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 580 | current = GetInstalledPackageVersions(pkg, root=root) |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 581 | desired = GetDesiredPackageVersions(target, package) |
Gilad Arnold | 2fcb0aa | 2015-05-21 14:51:41 -0700 | [diff] [blame] | 582 | # NOTE: This refers to installed packages (vartree) rather than the |
| 583 | # Portage version (porttree and/or bintree) when determining the current |
| 584 | # version. While this isn't the most accurate thing to do, it is probably |
| 585 | # a good simple compromise, which should have the desired result of |
| 586 | # uninstalling everything but the latest installed version. In |
| 587 | # particular, using the bintree (--usebinpkg) requires a non-trivial |
| 588 | # binhost sync and is probably more complex than useful. |
Zdenek Behan | 699ddd3 | 2012-04-13 07:14:08 +0200 | [diff] [blame] | 589 | desired_num = VersionListToNumeric(target, package, desired, True) |
| 590 | if not set(desired_num).issubset(current): |
Mike Frysinger | 3bba503 | 2016-09-20 14:15:04 -0400 | [diff] [blame] | 591 | logging.warning('Error detecting stable version for %s, ' |
| 592 | 'skipping clean!', pkg) |
Zdenek Behan | 699ddd3 | 2012-04-13 07:14:08 +0200 | [diff] [blame] | 593 | return |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 594 | unmergemap[pkg] = set(current).difference(desired_num) |
| 595 | |
| 596 | # Cleaning doesn't care about consistency and rebuilding package.* files. |
| 597 | packages = [] |
Mike Frysinger | 0bdbc10 | 2019-06-13 15:27:29 -0400 | [diff] [blame] | 598 | for pkg, vers in unmergemap.items(): |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 599 | packages.extend('=%s-%s' % (pkg, ver) for ver in vers if ver != '9999') |
| 600 | |
| 601 | if packages: |
Mike Frysinger | 3bba503 | 2016-09-20 14:15:04 -0400 | [diff] [blame] | 602 | logging.info('Cleaning packages:') |
| 603 | logging.info('%s', packages) |
Zdenek Behan | 7e33b4e | 2012-03-12 17:00:56 +0100 | [diff] [blame] | 604 | cmd = [EMERGE_CMD, '--unmerge'] |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 605 | if root != '/': |
| 606 | cmd.extend(['--sysroot=%s' % root, '--root=%s' % root]) |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 607 | cmd.extend(packages) |
Mike Frysinger | 45602c7 | 2019-09-22 02:15:11 -0400 | [diff] [blame] | 608 | cros_build_lib.run(cmd) |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 609 | else: |
Mike Frysinger | 3bba503 | 2016-09-20 14:15:04 -0400 | [diff] [blame] | 610 | logging.info('Nothing to clean!') |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 611 | |
| 612 | |
Adrian Ratiu | 78e534d | 2021-01-06 19:58:38 +0200 | [diff] [blame] | 613 | def SelectActiveToolchains(targets, root='/'): |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 614 | """Runs gcc-config and binutils-config to select the desired. |
| 615 | |
Mike Frysinger | dc649ae | 2014-11-26 19:29:24 -0500 | [diff] [blame] | 616 | Args: |
| 617 | targets: The targets to select |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 618 | root: The root where we want to select toolchain versions. |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 619 | """ |
| 620 | for package in ['gcc', 'binutils']: |
| 621 | for target in targets: |
Mike Frysinger | 785b0c3 | 2017-09-13 01:35:59 -0400 | [diff] [blame] | 622 | # See if this package is part of this target. |
| 623 | if package not in GetTargetPackages(target): |
| 624 | logging.debug('%s: %s is not used', target, package) |
| 625 | continue |
| 626 | |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 627 | # Pick the first version in the numbered list as the selected one. |
| 628 | desired = GetDesiredPackageVersions(target, package) |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 629 | desired_num = VersionListToNumeric(target, package, desired, True, |
| 630 | root=root) |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 631 | desired = desired_num[0] |
| 632 | # *-config does not play revisions, strip them, keep just PV. |
| 633 | desired = portage.versions.pkgsplit('%s-%s' % (package, desired))[1] |
| 634 | |
Mike Frysinger | 785b0c3 | 2017-09-13 01:35:59 -0400 | [diff] [blame] | 635 | if target.startswith('host'): |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 636 | # *-config is the only tool treating host identically (by tuple). |
David James | 27ac4ae | 2012-12-03 23:16:15 -0800 | [diff] [blame] | 637 | target = toolchain.GetHostTuple() |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 638 | |
| 639 | # And finally, attach target to it. |
| 640 | desired = '%s-%s' % (target, desired) |
| 641 | |
David James | 7ec5efc | 2012-11-06 09:39:49 -0800 | [diff] [blame] | 642 | extra_env = {'CHOST': target} |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 643 | if root != '/': |
| 644 | extra_env['ROOT'] = root |
David James | 7ec5efc | 2012-11-06 09:39:49 -0800 | [diff] [blame] | 645 | cmd = ['%s-config' % package, '-c', target] |
Mike Frysinger | 45602c7 | 2019-09-22 02:15:11 -0400 | [diff] [blame] | 646 | result = cros_build_lib.run( |
Mike Frysinger | 0282d22 | 2019-12-17 17:15:48 -0500 | [diff] [blame] | 647 | cmd, print_cmd=False, stdout=True, encoding='utf-8', |
Mike Frysinger | b3202be | 2019-11-15 22:25:59 -0500 | [diff] [blame] | 648 | extra_env=extra_env) |
Mike Frysinger | d6e2df0 | 2014-11-26 02:55:04 -0500 | [diff] [blame] | 649 | current = result.output.splitlines()[0] |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 650 | |
| 651 | # Do not reconfig when the current is live or nothing needs to be done. |
| 652 | extra_env = {'ROOT': root} if root != '/' else None |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 653 | if current != desired and current != '9999': |
Mike Frysinger | d6e2df0 | 2014-11-26 02:55:04 -0500 | [diff] [blame] | 654 | cmd = [package + '-config', desired] |
Mike Frysinger | 45602c7 | 2019-09-22 02:15:11 -0400 | [diff] [blame] | 655 | cros_build_lib.run(cmd, print_cmd=False, extra_env=extra_env) |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 656 | |
| 657 | |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 658 | def ExpandTargets(targets_wanted): |
| 659 | """Expand any possible toolchain aliases into full targets |
| 660 | |
| 661 | This will expand 'all' and 'sdk' into the respective toolchain tuples. |
| 662 | |
| 663 | Args: |
| 664 | targets_wanted: The targets specified by the user. |
Mike Frysinger | 1a736a8 | 2013-12-12 01:50:59 -0500 | [diff] [blame] | 665 | |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 666 | Returns: |
Gilad Arnold | 8195b53 | 2015-04-07 10:56:30 +0300 | [diff] [blame] | 667 | Dictionary of concrete targets and their toolchain tuples. |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 668 | """ |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 669 | targets_wanted = set(targets_wanted) |
Don Garrett | c0c7400 | 2015-10-09 12:58:19 -0700 | [diff] [blame] | 670 | if targets_wanted == set(['boards']): |
| 671 | # Only pull targets from the included boards. |
Gilad Arnold | 8195b53 | 2015-04-07 10:56:30 +0300 | [diff] [blame] | 672 | return {} |
| 673 | |
| 674 | all_targets = toolchain.GetAllTargets() |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 675 | if targets_wanted == set(['all']): |
Gilad Arnold | 8195b53 | 2015-04-07 10:56:30 +0300 | [diff] [blame] | 676 | return all_targets |
| 677 | if targets_wanted == set(['sdk']): |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 678 | # Filter out all the non-sdk toolchains as we don't want to mess |
| 679 | # with those in all of our builds. |
Gilad Arnold | 8195b53 | 2015-04-07 10:56:30 +0300 | [diff] [blame] | 680 | return toolchain.FilterToolchains(all_targets, 'sdk', True) |
| 681 | |
| 682 | # Verify user input. |
| 683 | nonexistent = targets_wanted.difference(all_targets) |
| 684 | if nonexistent: |
Mike Frysinger | cd79066 | 2019-10-17 00:23:13 -0400 | [diff] [blame] | 685 | raise ValueError('Invalid targets: %s' % (','.join(nonexistent),)) |
Gilad Arnold | 8195b53 | 2015-04-07 10:56:30 +0300 | [diff] [blame] | 686 | return {t: all_targets[t] for t in targets_wanted} |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 687 | |
| 688 | |
David James | f8c672f | 2012-11-06 13:38:11 -0800 | [diff] [blame] | 689 | def UpdateToolchains(usepkg, deleteold, hostonly, reconfig, |
Don Garrett | c0c7400 | 2015-10-09 12:58:19 -0700 | [diff] [blame] | 690 | targets_wanted, boards_wanted, root='/'): |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 691 | """Performs all steps to create a synchronized toolchain enviroment. |
| 692 | |
Mike Frysinger | dc649ae | 2014-11-26 19:29:24 -0500 | [diff] [blame] | 693 | Args: |
| 694 | usepkg: Use prebuilt packages |
| 695 | deleteold: Unmerge deprecated packages |
| 696 | hostonly: Only setup the host toolchain |
| 697 | reconfig: Reload crossdev config and reselect toolchains |
| 698 | targets_wanted: All the targets to update |
| 699 | boards_wanted: Load targets from these boards |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 700 | root: The root in which to install the toolchains. |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 701 | """ |
David James | f8c672f | 2012-11-06 13:38:11 -0800 | [diff] [blame] | 702 | targets, crossdev_targets, reconfig_targets = {}, {}, {} |
Zdenek Behan | 7e33b4e | 2012-03-12 17:00:56 +0100 | [diff] [blame] | 703 | if not hostonly: |
| 704 | # For hostonly, we can skip most of the below logic, much of which won't |
| 705 | # work on bare systems where this is useful. |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 706 | targets = ExpandTargets(targets_wanted) |
Mike Frysinger | 7ccee99 | 2012-06-01 21:27:59 -0400 | [diff] [blame] | 707 | |
Don Garrett | c0c7400 | 2015-10-09 12:58:19 -0700 | [diff] [blame] | 708 | # Now re-add any targets that might be from this board. This is to |
Gilad Arnold | 8195b53 | 2015-04-07 10:56:30 +0300 | [diff] [blame] | 709 | # allow unofficial boards to declare their own toolchains. |
Mike Frysinger | 7ccee99 | 2012-06-01 21:27:59 -0400 | [diff] [blame] | 710 | for board in boards_wanted: |
David James | 27ac4ae | 2012-12-03 23:16:15 -0800 | [diff] [blame] | 711 | targets.update(toolchain.GetToolchainsForBoard(board)) |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 712 | |
Zdenek Behan | 7e33b4e | 2012-03-12 17:00:56 +0100 | [diff] [blame] | 713 | # First check and initialize all cross targets that need to be. |
Mike Frysinger | 7ccee99 | 2012-06-01 21:27:59 -0400 | [diff] [blame] | 714 | for target in targets: |
| 715 | if TargetIsInitialized(target): |
| 716 | reconfig_targets[target] = targets[target] |
| 717 | else: |
| 718 | crossdev_targets[target] = targets[target] |
Zdenek Behan | 7e33b4e | 2012-03-12 17:00:56 +0100 | [diff] [blame] | 719 | if crossdev_targets: |
Mike Frysinger | 3bba503 | 2016-09-20 14:15:04 -0400 | [diff] [blame] | 720 | logging.info('The following targets need to be re-initialized:') |
| 721 | logging.info('%s', crossdev_targets) |
David James | 66a09c4 | 2012-11-05 13:31:38 -0800 | [diff] [blame] | 722 | Crossdev.UpdateTargets(crossdev_targets, usepkg) |
Zdenek Behan | 8be29ba | 2012-05-29 23:10:34 +0200 | [diff] [blame] | 723 | # Those that were not initialized may need a config update. |
David James | 66a09c4 | 2012-11-05 13:31:38 -0800 | [diff] [blame] | 724 | Crossdev.UpdateTargets(reconfig_targets, usepkg, config_only=True) |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 725 | |
Mike Frysinger | 66814c3 | 2017-10-09 18:11:46 -0400 | [diff] [blame] | 726 | # If we're building a subset of toolchains for a board, we might not have |
| 727 | # all the tuples that the packages expect. We don't define the "full" set |
| 728 | # of tuples currently other than "whatever the full sdk has normally". |
| 729 | if usepkg or set(('all', 'sdk')) & targets_wanted: |
| 730 | # Since we have cross-compilers now, we can update these packages. |
| 731 | targets['host-post-cross'] = {} |
Mike Frysinger | 785b0c3 | 2017-09-13 01:35:59 -0400 | [diff] [blame] | 732 | |
Zdenek Behan | 7e33b4e | 2012-03-12 17:00:56 +0100 | [diff] [blame] | 733 | # We want host updated. |
Mike Frysinger | 7ccee99 | 2012-06-01 21:27:59 -0400 | [diff] [blame] | 734 | targets['host'] = {} |
Zdenek Behan | 7e33b4e | 2012-03-12 17:00:56 +0100 | [diff] [blame] | 735 | |
| 736 | # Now update all packages. |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 737 | if UpdateTargets(targets, usepkg, root=root) or crossdev_targets or reconfig: |
Adrian Ratiu | 78e534d | 2021-01-06 19:58:38 +0200 | [diff] [blame] | 738 | SelectActiveToolchains(targets, root=root) |
David James | 7ec5efc | 2012-11-06 09:39:49 -0800 | [diff] [blame] | 739 | |
| 740 | if deleteold: |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 741 | CleanTargets(targets, root=root) |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 742 | |
Mike Frysinger | c880a96 | 2013-11-08 13:59:06 -0500 | [diff] [blame] | 743 | # Now that we've cleared out old versions, see if we need to rebuild |
| 744 | # anything. Can't do this earlier as it might not be broken. |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 745 | RebuildLibtool(root=root) |
Mike Frysinger | c880a96 | 2013-11-08 13:59:06 -0500 | [diff] [blame] | 746 | |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 747 | |
Bertrand SIMONNET | cae9d5f | 2015-03-09 15:58:01 -0700 | [diff] [blame] | 748 | def ShowConfig(name): |
| 749 | """Show the toolchain tuples used by |name| |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 750 | |
| 751 | Args: |
Don Garrett | c0c7400 | 2015-10-09 12:58:19 -0700 | [diff] [blame] | 752 | name: The board name to query. |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 753 | """ |
Don Garrett | c0c7400 | 2015-10-09 12:58:19 -0700 | [diff] [blame] | 754 | toolchains = toolchain.GetToolchainsForBoard(name) |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 755 | # Make sure we display the default toolchain first. |
Mike Frysinger | 3bba503 | 2016-09-20 14:15:04 -0400 | [diff] [blame] | 756 | # Note: Do not use logging here as this is meant to be used by other tools. |
Mike Frysinger | 383367e | 2014-09-16 15:06:17 -0400 | [diff] [blame] | 757 | print(','.join( |
Mike Frysinger | 818d963 | 2019-08-24 14:43:05 -0400 | [diff] [blame] | 758 | list(toolchain.FilterToolchains(toolchains, 'default', True)) + |
| 759 | list(toolchain.FilterToolchains(toolchains, 'default', False)))) |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 760 | |
| 761 | |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 762 | def GeneratePathWrapper(root, wrappath, path): |
| 763 | """Generate a shell script to execute another shell script |
| 764 | |
| 765 | Since we can't symlink a wrapped ELF (see GenerateLdsoWrapper) because the |
| 766 | argv[0] won't be pointing to the correct path, generate a shell script that |
| 767 | just executes another program with its full path. |
| 768 | |
| 769 | Args: |
| 770 | root: The root tree to generate scripts inside of |
| 771 | wrappath: The full path (inside |root|) to create the wrapper |
| 772 | path: The target program which this wrapper will execute |
| 773 | """ |
| 774 | replacements = { |
| 775 | 'path': path, |
| 776 | 'relroot': os.path.relpath('/', os.path.dirname(wrappath)), |
| 777 | } |
Takuto Ikuta | 5840397 | 2018-08-16 18:52:51 +0900 | [diff] [blame] | 778 | |
| 779 | # Do not use exec here, because exec invokes script with absolute path in |
| 780 | # argv0. Keeping relativeness allows us to remove abs path from compile result |
| 781 | # and leads directory independent build cache sharing in some distributed |
| 782 | # build system. |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 783 | wrapper = """#!/bin/sh |
Takuto Ikuta | 5840397 | 2018-08-16 18:52:51 +0900 | [diff] [blame] | 784 | basedir=$(dirname "$0") |
| 785 | "${basedir}/%(relroot)s%(path)s" "$@" |
| 786 | exit "$?" |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 787 | """ % replacements |
| 788 | root_wrapper = root + wrappath |
| 789 | if os.path.islink(root_wrapper): |
| 790 | os.unlink(root_wrapper) |
| 791 | else: |
| 792 | osutils.SafeMakedirs(os.path.dirname(root_wrapper)) |
| 793 | osutils.WriteFile(root_wrapper, wrapper) |
Mike Frysinger | 60ec101 | 2013-10-21 00:11:10 -0400 | [diff] [blame] | 794 | os.chmod(root_wrapper, 0o755) |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 795 | |
| 796 | |
Rahul Chaudhry | a8127bb | 2016-07-22 15:53:17 -0700 | [diff] [blame] | 797 | def FixClangXXWrapper(root, path): |
| 798 | """Fix wrapper shell scripts and symlinks for invoking clang++ |
| 799 | |
| 800 | In a typical installation, clang++ symlinks to clang, which symlinks to the |
| 801 | elf executable. The executable distinguishes between clang and clang++ based |
| 802 | on argv[0]. |
| 803 | |
| 804 | When invoked through the LdsoWrapper, argv[0] always contains the path to the |
| 805 | executable elf file, making clang/clang++ invocations indistinguishable. |
| 806 | |
| 807 | This function detects if the elf executable being wrapped is clang-X.Y, and |
| 808 | fixes wrappers/symlinks as necessary so that clang++ will work correctly. |
| 809 | |
| 810 | The calling sequence now becomes: |
| 811 | -) clang++ invocation turns into clang++-3.9 (which is a copy of clang-3.9, |
| 812 | the Ldsowrapper). |
| 813 | -) clang++-3.9 uses the Ldso to invoke clang++-3.9.elf, which is a symlink |
| 814 | to the original clang-3.9 elf. |
| 815 | -) The difference this time is that inside the elf file execution, $0 is |
| 816 | set as .../usr/bin/clang++-3.9.elf, which contains 'clang++' in the name. |
| 817 | |
Manoj Gupta | ae26814 | 2018-04-27 23:28:36 -0700 | [diff] [blame] | 818 | Update: Starting since clang 7, the clang and clang++ are symlinks to |
| 819 | clang-7 binary, not clang-7.0. The pattern match is extended to handle |
| 820 | both clang-7 and clang-7.0 cases for now. (https://crbug.com/837889) |
| 821 | |
Rahul Chaudhry | a8127bb | 2016-07-22 15:53:17 -0700 | [diff] [blame] | 822 | Args: |
| 823 | root: The root tree to generate scripts / symlinks inside of |
| 824 | path: The target elf for which LdsoWrapper was created |
| 825 | """ |
Manoj Gupta | ae26814 | 2018-04-27 23:28:36 -0700 | [diff] [blame] | 826 | if re.match(r'/usr/bin/clang-\d+(\.\d+)*$', path): |
Rahul Chaudhry | a8127bb | 2016-07-22 15:53:17 -0700 | [diff] [blame] | 827 | logging.info('fixing clang++ invocation for %s', path) |
| 828 | clangdir = os.path.dirname(root + path) |
| 829 | clang = os.path.basename(path) |
| 830 | clangxx = clang.replace('clang', 'clang++') |
| 831 | |
| 832 | # Create a symlink clang++-X.Y.elf to point to clang-X.Y.elf |
| 833 | os.symlink(clang + '.elf', os.path.join(clangdir, clangxx + '.elf')) |
| 834 | |
| 835 | # Create a hardlink clang++-X.Y pointing to clang-X.Y |
| 836 | os.link(os.path.join(clangdir, clang), os.path.join(clangdir, clangxx)) |
| 837 | |
| 838 | # Adjust the clang++ symlink to point to clang++-X.Y |
| 839 | os.unlink(os.path.join(clangdir, 'clang++')) |
| 840 | os.symlink(clangxx, os.path.join(clangdir, 'clang++')) |
| 841 | |
| 842 | |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 843 | def FileIsCrosSdkElf(elf): |
| 844 | """Determine if |elf| is an ELF that we execute in the cros_sdk |
| 845 | |
| 846 | We don't need this to be perfect, just quick. It makes sure the ELF |
| 847 | is a 64bit LSB x86_64 ELF. That is the native type of cros_sdk. |
| 848 | |
| 849 | Args: |
| 850 | elf: The file to check |
Mike Frysinger | 1a736a8 | 2013-12-12 01:50:59 -0500 | [diff] [blame] | 851 | |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 852 | Returns: |
| 853 | True if we think |elf| is a native ELF |
| 854 | """ |
Mike Frysinger | 4a12bf1 | 2019-12-04 04:20:10 -0500 | [diff] [blame] | 855 | with open(elf, 'rb') as f: |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 856 | data = f.read(20) |
| 857 | # Check the magic number, EI_CLASS, EI_DATA, and e_machine. |
Mike Frysinger | 4a12bf1 | 2019-12-04 04:20:10 -0500 | [diff] [blame] | 858 | return (data[0:4] == b'\x7fELF' and |
Mike Frysinger | dd34dfe | 2019-12-10 16:25:47 -0500 | [diff] [blame] | 859 | data[4:5] == b'\x02' and |
| 860 | data[5:6] == b'\x01' and |
| 861 | data[18:19] == b'\x3e') |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 862 | |
| 863 | |
| 864 | def IsPathPackagable(ptype, path): |
| 865 | """Should the specified file be included in a toolchain package? |
| 866 | |
| 867 | We only need to handle files as we'll create dirs as we need them. |
| 868 | |
| 869 | Further, trim files that won't be useful: |
| 870 | - non-english translations (.mo) since it'd require env vars |
| 871 | - debug files since these are for the host compiler itself |
| 872 | - info/man pages as they're big, and docs are online, and the |
| 873 | native docs should work fine for the most part (`man gcc`) |
| 874 | |
| 875 | Args: |
| 876 | ptype: A string describing the path type (i.e. 'file' or 'dir' or 'sym') |
| 877 | path: The full path to inspect |
Mike Frysinger | 1a736a8 | 2013-12-12 01:50:59 -0500 | [diff] [blame] | 878 | |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 879 | Returns: |
| 880 | True if we want to include this path in the package |
| 881 | """ |
| 882 | return not (ptype in ('dir',) or |
| 883 | path.startswith('/usr/lib/debug/') or |
| 884 | os.path.splitext(path)[1] == '.mo' or |
| 885 | ('/man/' in path or '/info/' in path)) |
| 886 | |
| 887 | |
| 888 | def ReadlinkRoot(path, root): |
| 889 | """Like os.readlink(), but relative to a |root| |
| 890 | |
| 891 | Args: |
| 892 | path: The symlink to read |
| 893 | root: The path to use for resolving absolute symlinks |
Mike Frysinger | 1a736a8 | 2013-12-12 01:50:59 -0500 | [diff] [blame] | 894 | |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 895 | Returns: |
| 896 | A fully resolved symlink path |
| 897 | """ |
| 898 | while os.path.islink(root + path): |
| 899 | path = os.path.join(os.path.dirname(path), os.readlink(root + path)) |
| 900 | return path |
| 901 | |
| 902 | |
| 903 | def _GetFilesForTarget(target, root='/'): |
| 904 | """Locate all the files to package for |target| |
| 905 | |
| 906 | This does not cover ELF dependencies. |
| 907 | |
| 908 | Args: |
| 909 | target: The toolchain target name |
| 910 | root: The root path to pull all packages from |
Mike Frysinger | 1a736a8 | 2013-12-12 01:50:59 -0500 | [diff] [blame] | 911 | |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 912 | Returns: |
| 913 | A tuple of a set of all packable paths, and a set of all paths which |
| 914 | are also native ELFs |
| 915 | """ |
| 916 | paths = set() |
| 917 | elfs = set() |
| 918 | |
| 919 | # Find all the files owned by the packages for this target. |
| 920 | for pkg in GetTargetPackages(target): |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 921 | |
Rahul Chaudhry | 4b80305 | 2015-05-13 15:25:56 -0700 | [diff] [blame] | 922 | # Skip Go compiler from redistributable packages. |
| 923 | # The "go" executable has GOROOT=/usr/lib/go/${CTARGET} hardcoded |
| 924 | # into it. Due to this, the toolchain cannot be unpacked anywhere |
| 925 | # else and be readily useful. To enable packaging Go, we need to: |
| 926 | # -) Tweak the wrappers/environment to override GOROOT |
| 927 | # automatically based on the unpack location. |
| 928 | # -) Make sure the ELF dependency checking and wrapping logic |
| 929 | # below skips the Go toolchain executables and libraries. |
| 930 | # -) Make sure the packaging process maintains the relative |
| 931 | # timestamps of precompiled standard library packages. |
| 932 | # (see dev-lang/go ebuild for details). |
| 933 | if pkg == 'ex_go': |
| 934 | continue |
| 935 | |
Yunlian Jiang | 36f3524 | 2018-04-27 10:18:40 -0700 | [diff] [blame] | 936 | # Use armv7a-cros-linux-gnueabi/compiler-rt for |
| 937 | # armv7a-cros-linux-gnueabihf/compiler-rt. |
| 938 | # Currently the armv7a-cros-linux-gnueabi is actually |
| 939 | # the same as armv7a-cros-linux-gnueabihf with different names. |
| 940 | # Because of that, for compiler-rt, it generates the same binary in |
| 941 | # the same location. To avoid the installation conflict, we do not |
| 942 | # install anything for 'armv7a-cros-linux-gnueabihf'. This would cause |
| 943 | # problem if other people try to use standalone armv7a-cros-linux-gnueabihf |
| 944 | # toolchain. |
| 945 | if 'compiler-rt' in pkg and 'armv7a-cros-linux-gnueabi' in target: |
| 946 | atom = GetPortagePackage(target, pkg) |
| 947 | cat, pn = atom.split('/') |
| 948 | ver = GetInstalledPackageVersions(atom, root=root)[0] |
Yunlian Jiang | 36f3524 | 2018-04-27 10:18:40 -0700 | [diff] [blame] | 949 | dblink = portage.dblink(cat, pn + '-' + ver, myroot=root, |
| 950 | settings=portage.settings) |
| 951 | contents = dblink.getcontents() |
| 952 | if not contents: |
| 953 | if 'hf' in target: |
| 954 | new_target = 'armv7a-cros-linux-gnueabi' |
| 955 | else: |
| 956 | new_target = 'armv7a-cros-linux-gnueabihf' |
| 957 | atom = GetPortagePackage(new_target, pkg) |
| 958 | else: |
| 959 | atom = GetPortagePackage(target, pkg) |
| 960 | |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 961 | cat, pn = atom.split('/') |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 962 | ver = GetInstalledPackageVersions(atom, root=root)[0] |
Ralph Nathan | 0304728 | 2015-03-23 11:09:32 -0700 | [diff] [blame] | 963 | logging.info('packaging %s-%s', atom, ver) |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 964 | |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 965 | dblink = portage.dblink(cat, pn + '-' + ver, myroot=root, |
| 966 | settings=portage.settings) |
| 967 | contents = dblink.getcontents() |
| 968 | for obj in contents: |
| 969 | ptype = contents[obj][0] |
| 970 | if not IsPathPackagable(ptype, obj): |
| 971 | continue |
| 972 | |
| 973 | if ptype == 'obj': |
| 974 | # For native ELFs, we need to pull in their dependencies too. |
| 975 | if FileIsCrosSdkElf(obj): |
Mike Frysinger | 60a2f6c | 2019-12-04 04:18:58 -0500 | [diff] [blame] | 976 | logging.debug('Adding ELF %s', obj) |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 977 | elfs.add(obj) |
Mike Frysinger | 60a2f6c | 2019-12-04 04:18:58 -0500 | [diff] [blame] | 978 | logging.debug('Adding path %s', obj) |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 979 | paths.add(obj) |
| 980 | |
| 981 | return paths, elfs |
| 982 | |
| 983 | |
| 984 | def _BuildInitialPackageRoot(output_dir, paths, elfs, ldpaths, |
Mike Frysinger | d6e2df0 | 2014-11-26 02:55:04 -0500 | [diff] [blame] | 985 | path_rewrite_func=lambda x: x, root='/'): |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 986 | """Link in all packable files and their runtime dependencies |
| 987 | |
| 988 | This also wraps up executable ELFs with helper scripts. |
| 989 | |
| 990 | Args: |
| 991 | output_dir: The output directory to store files |
| 992 | paths: All the files to include |
| 993 | elfs: All the files which are ELFs (a subset of |paths|) |
| 994 | ldpaths: A dict of static ldpath information |
| 995 | path_rewrite_func: User callback to rewrite paths in output_dir |
| 996 | root: The root path to pull all packages/files from |
| 997 | """ |
| 998 | # Link in all the files. |
Mike Frysinger | 221bd82 | 2017-09-29 02:51:47 -0400 | [diff] [blame] | 999 | sym_paths = {} |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1000 | for path in paths: |
| 1001 | new_path = path_rewrite_func(path) |
Mike Frysinger | 60a2f6c | 2019-12-04 04:18:58 -0500 | [diff] [blame] | 1002 | logging.debug('Transformed %s to %s', path, new_path) |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1003 | dst = output_dir + new_path |
| 1004 | osutils.SafeMakedirs(os.path.dirname(dst)) |
| 1005 | |
| 1006 | # Is this a symlink which we have to rewrite or wrap? |
| 1007 | # Delay wrap check until after we have created all paths. |
| 1008 | src = root + path |
| 1009 | if os.path.islink(src): |
| 1010 | tgt = os.readlink(src) |
| 1011 | if os.path.sep in tgt: |
Mike Frysinger | 221bd82 | 2017-09-29 02:51:47 -0400 | [diff] [blame] | 1012 | sym_paths[lddtree.normpath(ReadlinkRoot(src, root))] = new_path |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1013 | |
| 1014 | # Rewrite absolute links to relative and then generate the symlink |
| 1015 | # ourselves. All other symlinks can be hardlinked below. |
| 1016 | if tgt[0] == '/': |
| 1017 | tgt = os.path.relpath(tgt, os.path.dirname(new_path)) |
| 1018 | os.symlink(tgt, dst) |
| 1019 | continue |
| 1020 | |
Mike Frysinger | 60a2f6c | 2019-12-04 04:18:58 -0500 | [diff] [blame] | 1021 | logging.debug('Linking path %s -> %s', src, dst) |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1022 | os.link(src, dst) |
| 1023 | |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1024 | # Locate all the dependencies for all the ELFs. Stick them all in the |
| 1025 | # top level "lib" dir to make the wrapper simpler. This exact path does |
| 1026 | # not matter since we execute ldso directly, and we tell the ldso the |
| 1027 | # exact path to search for its libraries. |
| 1028 | libdir = os.path.join(output_dir, 'lib') |
| 1029 | osutils.SafeMakedirs(libdir) |
| 1030 | donelibs = set() |
Mike Frysinger | 9fe0234 | 2019-12-12 17:52:53 -0500 | [diff] [blame] | 1031 | basenamelibs = set() |
Mike Frysinger | 4331fc6 | 2017-09-28 14:03:25 -0400 | [diff] [blame] | 1032 | glibc_re = re.compile(r'/lib(c|pthread)-[0-9.]+\.so$') |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1033 | for elf in elfs: |
Mike Frysinger | ea7688e | 2014-07-31 22:40:33 -0400 | [diff] [blame] | 1034 | e = lddtree.ParseELF(elf, root=root, ldpaths=ldpaths) |
Mike Frysinger | 60a2f6c | 2019-12-04 04:18:58 -0500 | [diff] [blame] | 1035 | logging.debug('Parsed elf %s data: %s', elf, e) |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1036 | interp = e['interp'] |
Mike Frysinger | 221bd82 | 2017-09-29 02:51:47 -0400 | [diff] [blame] | 1037 | |
Mike Frysinger | 9fe0234 | 2019-12-12 17:52:53 -0500 | [diff] [blame] | 1038 | # TODO(crbug.com/917193): Drop this hack once libopcodes linkage is fixed. |
| 1039 | if os.path.basename(elf).startswith('libopcodes-'): |
| 1040 | continue |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1041 | |
Mike Frysinger | 00b129f | 2021-04-21 18:11:48 -0400 | [diff] [blame] | 1042 | # Copy all the dependencies before we copy the program & generate wrappers. |
Mike Frysinger | 9fe0234 | 2019-12-12 17:52:53 -0500 | [diff] [blame] | 1043 | for lib, lib_data in e['libs'].items(): |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1044 | src = path = lib_data['path'] |
| 1045 | if path is None: |
Ralph Nathan | 446aee9 | 2015-03-23 14:44:56 -0700 | [diff] [blame] | 1046 | logging.warning('%s: could not locate %s', elf, lib) |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1047 | continue |
Mike Frysinger | 9fe0234 | 2019-12-12 17:52:53 -0500 | [diff] [blame] | 1048 | |
| 1049 | # No need to try and copy the same source lib multiple times. |
| 1050 | if path in donelibs: |
| 1051 | continue |
| 1052 | donelibs.add(path) |
| 1053 | |
| 1054 | # Die if we try to normalize different source libs with the same basename. |
| 1055 | if lib in basenamelibs: |
| 1056 | logging.error('Multiple sources detected for %s:\n new: %s\n old: %s', |
| 1057 | os.path.join('/lib', lib), path, |
| 1058 | ' '.join(x for x in donelibs |
| 1059 | if x != path and os.path.basename(x) == lib)) |
| 1060 | # TODO(crbug.com/917193): Make this fatal. |
| 1061 | # cros_build_lib.Die('Unable to resolve lib conflicts') |
| 1062 | continue |
| 1063 | basenamelibs.add(lib) |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1064 | |
| 1065 | # Needed libs are the SONAME, but that is usually a symlink, not a |
| 1066 | # real file. So link in the target rather than the symlink itself. |
| 1067 | # We have to walk all the possible symlinks (SONAME could point to a |
| 1068 | # symlink which points to a symlink), and we have to handle absolute |
| 1069 | # ourselves (since we have a "root" argument). |
| 1070 | dst = os.path.join(libdir, os.path.basename(path)) |
| 1071 | src = ReadlinkRoot(src, root) |
| 1072 | |
Mike Frysinger | 60a2f6c | 2019-12-04 04:18:58 -0500 | [diff] [blame] | 1073 | logging.debug('Linking lib %s -> %s', root + src, dst) |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1074 | os.link(root + src, dst) |
| 1075 | |
Mike Frysinger | 00b129f | 2021-04-21 18:11:48 -0400 | [diff] [blame] | 1076 | # Do not create wrapper for libc. crbug.com/766827 |
| 1077 | if interp and not glibc_re.search(elf): |
| 1078 | # Generate a wrapper if it is executable. |
| 1079 | interp = os.path.join('/lib', os.path.basename(interp)) |
| 1080 | lddtree.GenerateLdsoWrapper(output_dir, path_rewrite_func(elf), interp, |
| 1081 | libpaths=e['rpath'] + e['runpath']) |
| 1082 | FixClangXXWrapper(output_dir, path_rewrite_func(elf)) |
| 1083 | |
| 1084 | # Wrap any symlinks to the wrapper. |
| 1085 | if elf in sym_paths: |
| 1086 | link = sym_paths[elf] |
| 1087 | GeneratePathWrapper(output_dir, link, elf) |
| 1088 | |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1089 | |
| 1090 | def _EnvdGetVar(envd, var): |
| 1091 | """Given a Gentoo env.d file, extract a var from it |
| 1092 | |
| 1093 | Args: |
| 1094 | envd: The env.d file to load (may be a glob path) |
| 1095 | var: The var to extract |
Mike Frysinger | 1a736a8 | 2013-12-12 01:50:59 -0500 | [diff] [blame] | 1096 | |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1097 | Returns: |
| 1098 | The value of |var| |
| 1099 | """ |
| 1100 | envds = glob.glob(envd) |
| 1101 | assert len(envds) == 1, '%s: should have exactly 1 env.d file' % envd |
| 1102 | envd = envds[0] |
Mike Frysinger | e652ba1 | 2019-09-08 00:57:43 -0400 | [diff] [blame] | 1103 | return key_value_store.LoadFile(envd)[var] |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1104 | |
| 1105 | |
| 1106 | def _ProcessBinutilsConfig(target, output_dir): |
| 1107 | """Do what binutils-config would have done""" |
| 1108 | binpath = os.path.join('/bin', target + '-') |
Mike Frysinger | d4d40fd | 2014-11-06 17:30:57 -0500 | [diff] [blame] | 1109 | |
Adrian Ratiu | 78e534d | 2021-01-06 19:58:38 +0200 | [diff] [blame] | 1110 | # Locate the bin dir holding the linker and perform some sanity checks |
Mike Frysinger | d4d40fd | 2014-11-06 17:30:57 -0500 | [diff] [blame] | 1111 | binutils_bin_path = os.path.join(output_dir, 'usr', toolchain.GetHostTuple(), |
| 1112 | target, 'binutils-bin') |
Adrian Ratiu | 78e534d | 2021-01-06 19:58:38 +0200 | [diff] [blame] | 1113 | globpath = os.path.join(binutils_bin_path, '*') |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1114 | srcpath = glob.glob(globpath) |
Adrian Ratiu | 78e534d | 2021-01-06 19:58:38 +0200 | [diff] [blame] | 1115 | assert len(srcpath) == 1, ('%s: matched more than one path. Is Gold enabled?' |
| 1116 | % globpath) |
| 1117 | srcpath = srcpath[0] |
| 1118 | ld_path = os.path.join(srcpath, 'ld') |
| 1119 | assert os.path.exists(ld_path), '%s: linker is missing!' % ld_path |
| 1120 | ld_path = os.path.join(srcpath, 'ld.bfd') |
| 1121 | assert os.path.exists(ld_path), '%s: linker is missing!' % ld_path |
Rahul Chaudhry | 4891b4d | 2017-03-08 10:31:27 -0800 | [diff] [blame] | 1122 | |
Mike Frysinger | 78b7a81 | 2014-11-26 19:45:23 -0500 | [diff] [blame] | 1123 | srcpath = srcpath[len(output_dir):] |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1124 | gccpath = os.path.join('/usr', 'libexec', 'gcc') |
| 1125 | for prog in os.listdir(output_dir + srcpath): |
| 1126 | # Skip binaries already wrapped. |
| 1127 | if not prog.endswith('.real'): |
| 1128 | GeneratePathWrapper(output_dir, binpath + prog, |
| 1129 | os.path.join(srcpath, prog)) |
| 1130 | GeneratePathWrapper(output_dir, os.path.join(gccpath, prog), |
| 1131 | os.path.join(srcpath, prog)) |
| 1132 | |
David James | 27ac4ae | 2012-12-03 23:16:15 -0800 | [diff] [blame] | 1133 | libpath = os.path.join('/usr', toolchain.GetHostTuple(), target, 'lib') |
Mike Frysinger | d4d40fd | 2014-11-06 17:30:57 -0500 | [diff] [blame] | 1134 | envd = os.path.join(output_dir, 'etc', 'env.d', 'binutils', '*') |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1135 | srcpath = _EnvdGetVar(envd, 'LIBPATH') |
| 1136 | os.symlink(os.path.relpath(srcpath, os.path.dirname(libpath)), |
| 1137 | output_dir + libpath) |
| 1138 | |
| 1139 | |
| 1140 | def _ProcessGccConfig(target, output_dir): |
| 1141 | """Do what gcc-config would have done""" |
| 1142 | binpath = '/bin' |
| 1143 | envd = os.path.join(output_dir, 'etc', 'env.d', 'gcc', '*') |
| 1144 | srcpath = _EnvdGetVar(envd, 'GCC_PATH') |
| 1145 | for prog in os.listdir(output_dir + srcpath): |
| 1146 | # Skip binaries already wrapped. |
| 1147 | if (not prog.endswith('.real') and |
| 1148 | not prog.endswith('.elf') and |
| 1149 | prog.startswith(target)): |
| 1150 | GeneratePathWrapper(output_dir, os.path.join(binpath, prog), |
| 1151 | os.path.join(srcpath, prog)) |
| 1152 | return srcpath |
| 1153 | |
| 1154 | |
Frank Henigman | 179ec7c | 2015-02-06 03:01:09 -0500 | [diff] [blame] | 1155 | def _ProcessSysrootWrappers(_target, output_dir, srcpath): |
| 1156 | """Remove chroot-specific things from our sysroot wrappers""" |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1157 | # Disable ccache since we know it won't work outside of chroot. |
Tobias Bosch | ddd1649 | 2019-08-14 09:29:54 -0700 | [diff] [blame] | 1158 | |
Tobias Bosch | ddd1649 | 2019-08-14 09:29:54 -0700 | [diff] [blame] | 1159 | # Use the version of the wrapper that does not use ccache. |
Frank Henigman | 179ec7c | 2015-02-06 03:01:09 -0500 | [diff] [blame] | 1160 | for sysroot_wrapper in glob.glob(os.path.join( |
Tobias Bosch | ddd1649 | 2019-08-14 09:29:54 -0700 | [diff] [blame] | 1161 | output_dir + srcpath, 'sysroot_wrapper*.ccache')): |
| 1162 | # Can't update the wrapper in place to not affect the chroot, |
| 1163 | # but only the extracted toolchain. |
| 1164 | os.unlink(sysroot_wrapper) |
| 1165 | shutil.copy(sysroot_wrapper[:-6] + 'noccache', sysroot_wrapper) |
Tobias Bosch | 7bc907a | 2019-10-09 11:52:40 -0700 | [diff] [blame] | 1166 | shutil.copy(sysroot_wrapper[:-6] + 'noccache.elf', sysroot_wrapper + '.elf') |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1167 | |
| 1168 | |
Yunlian Jiang | 5ad6b51 | 2017-09-20 09:27:45 -0700 | [diff] [blame] | 1169 | def _CreateMainLibDir(target, output_dir): |
| 1170 | """Create some lib dirs so that compiler can get the right Gcc paths""" |
| 1171 | osutils.SafeMakedirs(os.path.join(output_dir, 'usr', target, 'lib')) |
| 1172 | osutils.SafeMakedirs(os.path.join(output_dir, 'usr', target, 'usr/lib')) |
| 1173 | |
| 1174 | |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1175 | def _ProcessDistroCleanups(target, output_dir): |
| 1176 | """Clean up the tree and remove all distro-specific requirements |
| 1177 | |
| 1178 | Args: |
| 1179 | target: The toolchain target name |
| 1180 | output_dir: The output directory to clean up |
| 1181 | """ |
| 1182 | _ProcessBinutilsConfig(target, output_dir) |
| 1183 | gcc_path = _ProcessGccConfig(target, output_dir) |
Frank Henigman | 179ec7c | 2015-02-06 03:01:09 -0500 | [diff] [blame] | 1184 | _ProcessSysrootWrappers(target, output_dir, gcc_path) |
Yunlian Jiang | 5ad6b51 | 2017-09-20 09:27:45 -0700 | [diff] [blame] | 1185 | _CreateMainLibDir(target, output_dir) |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1186 | |
| 1187 | osutils.RmDir(os.path.join(output_dir, 'etc')) |
| 1188 | |
| 1189 | |
| 1190 | def CreatePackagableRoot(target, output_dir, ldpaths, root='/'): |
| 1191 | """Setup a tree from the packages for the specified target |
| 1192 | |
| 1193 | This populates a path with all the files from toolchain packages so that |
| 1194 | a tarball can easily be generated from the result. |
| 1195 | |
| 1196 | Args: |
| 1197 | target: The target to create a packagable root from |
| 1198 | output_dir: The output directory to place all the files |
| 1199 | ldpaths: A dict of static ldpath information |
| 1200 | root: The root path to pull all packages/files from |
| 1201 | """ |
| 1202 | # Find all the files owned by the packages for this target. |
| 1203 | paths, elfs = _GetFilesForTarget(target, root=root) |
| 1204 | |
| 1205 | # Link in all the package's files, any ELF dependencies, and wrap any |
| 1206 | # executable ELFs with helper scripts. |
| 1207 | def MoveUsrBinToBin(path): |
Han Shen | 699ea19 | 2016-03-02 10:42:47 -0800 | [diff] [blame] | 1208 | """Move /usr/bin to /bin so people can just use that toplevel dir |
| 1209 | |
George Burgess IV | ca1d761 | 2020-10-01 00:38:32 -0700 | [diff] [blame] | 1210 | Note we do not apply this to clang or rust - there is correlation between |
| 1211 | clang's search path for libraries / inclusion and its installation path. |
Han Shen | 699ea19 | 2016-03-02 10:42:47 -0800 | [diff] [blame] | 1212 | """ |
George Burgess IV | ca1d761 | 2020-10-01 00:38:32 -0700 | [diff] [blame] | 1213 | if (path.startswith('/usr/bin/') and |
| 1214 | not any(x in path for x in ('clang', 'rust', 'cargo'))): |
Han Shen | 699ea19 | 2016-03-02 10:42:47 -0800 | [diff] [blame] | 1215 | return path[4:] |
| 1216 | return path |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1217 | _BuildInitialPackageRoot(output_dir, paths, elfs, ldpaths, |
| 1218 | path_rewrite_func=MoveUsrBinToBin, root=root) |
| 1219 | |
| 1220 | # The packages, when part of the normal distro, have helper scripts |
| 1221 | # that setup paths and such. Since we are making this standalone, we |
| 1222 | # need to preprocess all that ourselves. |
| 1223 | _ProcessDistroCleanups(target, output_dir) |
| 1224 | |
| 1225 | |
| 1226 | def CreatePackages(targets_wanted, output_dir, root='/'): |
| 1227 | """Create redistributable cross-compiler packages for the specified targets |
| 1228 | |
| 1229 | This creates toolchain packages that should be usable in conjunction with |
| 1230 | a downloaded sysroot (created elsewhere). |
| 1231 | |
| 1232 | Tarballs (one per target) will be created in $PWD. |
| 1233 | |
| 1234 | Args: |
Don Garrett | 25f309a | 2014-03-19 14:02:12 -0700 | [diff] [blame] | 1235 | targets_wanted: The targets to package up. |
| 1236 | output_dir: The directory to put the packages in. |
| 1237 | root: The root path to pull all packages/files from. |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1238 | """ |
Ralph Nathan | 0304728 | 2015-03-23 11:09:32 -0700 | [diff] [blame] | 1239 | logging.info('Writing tarballs to %s', output_dir) |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1240 | osutils.SafeMakedirs(output_dir) |
| 1241 | ldpaths = lddtree.LoadLdpaths(root) |
| 1242 | targets = ExpandTargets(targets_wanted) |
| 1243 | |
Mike Frysinger | 221bd82 | 2017-09-29 02:51:47 -0400 | [diff] [blame] | 1244 | with osutils.TempDir(prefix='create-packages') as tempdir: |
| 1245 | logging.debug('Using tempdir: %s', tempdir) |
| 1246 | |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1247 | # We have to split the root generation from the compression stages. This is |
| 1248 | # because we hardlink in all the files (to avoid overhead of reading/writing |
| 1249 | # the copies multiple times). But tar gets angry if a file's hardlink count |
| 1250 | # changes from when it starts reading a file to when it finishes. |
| 1251 | with parallel.BackgroundTaskRunner(CreatePackagableRoot) as queue: |
| 1252 | for target in targets: |
| 1253 | output_target_dir = os.path.join(tempdir, target) |
| 1254 | queue.put([target, output_target_dir, ldpaths, root]) |
| 1255 | |
| 1256 | # Build the tarball. |
| 1257 | with parallel.BackgroundTaskRunner(cros_build_lib.CreateTarball) as queue: |
| 1258 | for target in targets: |
| 1259 | tar_file = os.path.join(output_dir, target + '.tar.xz') |
| 1260 | queue.put([tar_file, os.path.join(tempdir, target)]) |
| 1261 | |
| 1262 | |
Mike Frysinger | 07534cf | 2017-09-12 17:40:21 -0400 | [diff] [blame] | 1263 | def GetParser(): |
| 1264 | """Return a command line parser.""" |
Mike Frysinger | 0c80845 | 2014-11-06 17:30:23 -0500 | [diff] [blame] | 1265 | parser = commandline.ArgumentParser(description=__doc__) |
| 1266 | parser.add_argument('-u', '--nousepkg', |
| 1267 | action='store_false', dest='usepkg', default=True, |
| 1268 | help='Use prebuilt packages if possible') |
| 1269 | parser.add_argument('-d', '--deleteold', |
| 1270 | action='store_true', dest='deleteold', default=False, |
| 1271 | help='Unmerge deprecated packages') |
| 1272 | parser.add_argument('-t', '--targets', |
| 1273 | dest='targets', default='sdk', |
Mike Frysinger | 80de501 | 2019-08-01 14:10:53 -0400 | [diff] [blame] | 1274 | help='Comma separated list of tuples. Special keywords ' |
Don Garrett | c0c7400 | 2015-10-09 12:58:19 -0700 | [diff] [blame] | 1275 | "'host', 'sdk', 'boards', and 'all' are " |
Gilad Arnold | 8195b53 | 2015-04-07 10:56:30 +0300 | [diff] [blame] | 1276 | "allowed. Defaults to 'sdk'.") |
| 1277 | parser.add_argument('--include-boards', default='', metavar='BOARDS', |
| 1278 | help='Comma separated list of boards whose toolchains we ' |
| 1279 | 'will always include. Default: none') |
Mike Frysinger | 0c80845 | 2014-11-06 17:30:23 -0500 | [diff] [blame] | 1280 | parser.add_argument('--hostonly', |
| 1281 | dest='hostonly', default=False, action='store_true', |
| 1282 | help='Only setup the host toolchain. ' |
| 1283 | 'Useful for bootstrapping chroot') |
Bertrand SIMONNET | cae9d5f | 2015-03-09 15:58:01 -0700 | [diff] [blame] | 1284 | parser.add_argument('--show-board-cfg', '--show-cfg', |
| 1285 | dest='cfg_name', default=None, |
Don Garrett | c0c7400 | 2015-10-09 12:58:19 -0700 | [diff] [blame] | 1286 | help='Board to list toolchains tuples for') |
Mike Frysinger | 91f5288 | 2017-09-13 00:16:58 -0400 | [diff] [blame] | 1287 | parser.add_argument('--show-packages', default=None, |
| 1288 | help='List all packages the specified target uses') |
Mike Frysinger | 0c80845 | 2014-11-06 17:30:23 -0500 | [diff] [blame] | 1289 | parser.add_argument('--create-packages', |
| 1290 | action='store_true', default=False, |
| 1291 | help='Build redistributable packages') |
| 1292 | parser.add_argument('--output-dir', default=os.getcwd(), type='path', |
| 1293 | help='Output directory') |
| 1294 | parser.add_argument('--reconfig', default=False, action='store_true', |
| 1295 | help='Reload crossdev config and reselect toolchains') |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 1296 | parser.add_argument('--sysroot', type='path', |
| 1297 | help='The sysroot in which to install the toolchains') |
Mike Frysinger | 07534cf | 2017-09-12 17:40:21 -0400 | [diff] [blame] | 1298 | return parser |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 1299 | |
Mike Frysinger | 07534cf | 2017-09-12 17:40:21 -0400 | [diff] [blame] | 1300 | |
| 1301 | def main(argv): |
| 1302 | parser = GetParser() |
Mike Frysinger | 0c80845 | 2014-11-06 17:30:23 -0500 | [diff] [blame] | 1303 | options = parser.parse_args(argv) |
| 1304 | options.Freeze() |
Zdenek Behan | 508dcce | 2011-12-05 15:39:32 +0100 | [diff] [blame] | 1305 | |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1306 | # Figure out what we're supposed to do and reject conflicting options. |
Mike Frysinger | 91f5288 | 2017-09-13 00:16:58 -0400 | [diff] [blame] | 1307 | conflicting_options = ( |
| 1308 | options.cfg_name, |
| 1309 | options.show_packages, |
| 1310 | options.create_packages, |
| 1311 | ) |
| 1312 | if sum(bool(x) for x in conflicting_options) > 1: |
| 1313 | parser.error('conflicting options: create-packages & show-packages & ' |
| 1314 | 'show-board-cfg') |
Mike Frysinger | 984d062 | 2012-06-01 16:08:44 -0400 | [diff] [blame] | 1315 | |
Gilad Arnold | 8195b53 | 2015-04-07 10:56:30 +0300 | [diff] [blame] | 1316 | targets_wanted = set(options.targets.split(',')) |
| 1317 | boards_wanted = (set(options.include_boards.split(',')) |
| 1318 | if options.include_boards else set()) |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1319 | |
Bertrand SIMONNET | cae9d5f | 2015-03-09 15:58:01 -0700 | [diff] [blame] | 1320 | if options.cfg_name: |
| 1321 | ShowConfig(options.cfg_name) |
Mike Frysinger | 91f5288 | 2017-09-13 00:16:58 -0400 | [diff] [blame] | 1322 | elif options.show_packages is not None: |
| 1323 | cros_build_lib.AssertInsideChroot() |
| 1324 | target = options.show_packages |
| 1325 | Crossdev.Load(False) |
| 1326 | for package in GetTargetPackages(target): |
| 1327 | print(GetPortagePackage(target, package)) |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1328 | elif options.create_packages: |
| 1329 | cros_build_lib.AssertInsideChroot() |
| 1330 | Crossdev.Load(False) |
Gilad Arnold | 8195b53 | 2015-04-07 10:56:30 +0300 | [diff] [blame] | 1331 | CreatePackages(targets_wanted, options.output_dir) |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1332 | else: |
| 1333 | cros_build_lib.AssertInsideChroot() |
| 1334 | # This has to be always run as root. |
| 1335 | if os.geteuid() != 0: |
| 1336 | cros_build_lib.Die('this script must be run as root') |
| 1337 | |
| 1338 | Crossdev.Load(options.reconfig) |
Gilad Arnold | 2dab78c | 2015-05-21 14:43:33 -0700 | [diff] [blame] | 1339 | root = options.sysroot or '/' |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1340 | UpdateToolchains(options.usepkg, options.deleteold, options.hostonly, |
Gilad Arnold | 8195b53 | 2015-04-07 10:56:30 +0300 | [diff] [blame] | 1341 | options.reconfig, targets_wanted, boards_wanted, |
Don Garrett | c0c7400 | 2015-10-09 12:58:19 -0700 | [diff] [blame] | 1342 | root=root) |
Mike Frysinger | 35247af | 2012-11-16 18:58:06 -0500 | [diff] [blame] | 1343 | Crossdev.Save() |
| 1344 | |
| 1345 | return 0 |