blob: 93dce28ce77ae23c8ca1d121f60d4d712061d647 [file] [log] [blame]
Mike Frysingere58c0e22017-10-04 15:43:30 -04001# -*- coding: utf-8 -*-
Zdenek Behan508dcce2011-12-05 15:39:32 +01002# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
Mike Frysinger750c5f52014-09-16 16:16:57 -04006"""This script manages the installed toolchains in the chroot."""
Zdenek Behan508dcce2011-12-05 15:39:32 +01007
Mike Frysinger383367e2014-09-16 15:06:17 -04008from __future__ import print_function
9
Mike Frysinger3ed47722017-08-08 14:59:08 -040010import errno
Mike Frysinger35247af2012-11-16 18:58:06 -050011import glob
Mike Frysinger3ed47722017-08-08 14:59:08 -040012import hashlib
Mike Frysinger7ccee992012-06-01 21:27:59 -040013import json
Zdenek Behan508dcce2011-12-05 15:39:32 +010014import os
Rahul Chaudhrya8127bb2016-07-22 15:53:17 -070015import re
Tobias Boschddd16492019-08-14 09:29:54 -070016import shutil
Mike Frysinger499ca392020-04-28 07:35:54 -040017import sys
Zdenek Behan508dcce2011-12-05 15:39:32 +010018
Aviv Keshetb7519e12016-10-04 00:50:00 -070019from chromite.lib import constants
Mike Frysinger506e75f2012-12-17 14:21:13 -050020from chromite.lib import commandline
Brian Harring503f3ab2012-03-09 21:39:41 -080021from chromite.lib import cros_build_lib
Ralph Nathan03047282015-03-23 11:09:32 -070022from chromite.lib import cros_logging as logging
Brian Harringaf019fb2012-05-10 15:06:13 -070023from chromite.lib import osutils
Mike Frysinger35247af2012-11-16 18:58:06 -050024from chromite.lib import parallel
David James27ac4ae2012-12-03 23:16:15 -080025from chromite.lib import toolchain
Mike Frysingere652ba12019-09-08 00:57:43 -040026from chromite.utils import key_value_store
Mike Frysinger35247af2012-11-16 18:58:06 -050027
28# Needs to be after chromite imports.
Mike Frysingerabb7d812020-05-15 00:13:10 -040029import lddtree # pylint: disable=wrong-import-order
Zdenek Behan508dcce2011-12-05 15:39:32 +010030
Mike Frysinger31596002012-12-03 23:54:24 -050031if cros_build_lib.IsInsideChroot():
32 # Only import portage after we've checked that we're inside the chroot.
33 # Outside may not have portage, in which case the above may not happen.
34 # We'll check in main() if the operation needs portage.
Mike Frysinger27e21b72018-07-12 14:20:21 -040035 # pylint: disable=import-error
Mike Frysinger31596002012-12-03 23:54:24 -050036 import portage
Zdenek Behan508dcce2011-12-05 15:39:32 +010037
38
Mike Frysinger499ca392020-04-28 07:35:54 -040039assert sys.version_info >= (3, 6), 'This module requires Python 3.6+'
40
41
Matt Tennantf1e30972012-03-02 16:30:07 -080042EMERGE_CMD = os.path.join(constants.CHROMITE_BIN_DIR, 'parallel_emerge')
Zdenek Behan508dcce2011-12-05 15:39:32 +010043PACKAGE_STABLE = '[stable]'
Zdenek Behan4eb6fd22012-03-12 17:00:56 +010044
45CHROMIUMOS_OVERLAY = '/usr/local/portage/chromiumos'
Christopher Wileyb22c0712015-06-02 10:37:03 -070046ECLASS_OVERLAY = '/usr/local/portage/eclass-overlay'
Zdenek Behan4eb6fd22012-03-12 17:00:56 +010047STABLE_OVERLAY = '/usr/local/portage/stable'
48CROSSDEV_OVERLAY = '/usr/local/portage/crossdev'
Zdenek Behan508dcce2011-12-05 15:39:32 +010049
50
Mike Frysinger66bfde52017-09-12 16:42:57 -040051# The exact list of host toolchain packages we care about. These are the
52# packages that bots/devs install only from binpkgs and rely on the SDK bot
53# (chromiumos-sdk) to validate+uprev.
54#
Mike Frysinger66bfde52017-09-12 16:42:57 -040055# We don't use crossdev to manage the host toolchain for us, especially since
56# we diverge significantly now (with llvm/clang/etc...), and we don't need or
57# want crossdev managing /etc/portage config files for the sdk
58HOST_PACKAGES = (
59 'dev-lang/go',
Yunlian Jiang2cb91dc2018-03-08 10:56:27 -080060 'dev-libs/elfutils',
Mike Frysinger66bfde52017-09-12 16:42:57 -040061 'sys-devel/binutils',
Mike Frysinger66bfde52017-09-12 16:42:57 -040062 'sys-devel/gcc',
63 'sys-devel/llvm',
64 'sys-kernel/linux-headers',
65 'sys-libs/glibc',
66 'sys-libs/libcxx',
67 'sys-libs/libcxxabi',
Manoj Guptade64cb22019-03-31 18:48:58 -070068 'sys-libs/llvm-libunwind',
Mike Frysinger66bfde52017-09-12 16:42:57 -040069)
70
Mike Frysinger785b0c32017-09-13 01:35:59 -040071# These packages are also installed into the host SDK. However, they require
72# the cross-compilers to be installed first (because they need them to actually
73# build), so we have to delay their installation.
74HOST_POST_CROSS_PACKAGES = (
Manoj Gupta65f88442018-04-12 22:42:19 -070075 'dev-lang/rust',
George Burgess IV9d1e2322021-04-30 09:54:45 -070076 'dev-lang/rust-bootstrap',
Mike Frysinger61a24392017-10-17 17:14:27 -040077 'virtual/target-sdk-post-cross',
Patrick Georgi043ce6e2019-02-20 22:27:09 +010078 'dev-embedded/coreboot-sdk',
George Burgess IV288ecc82021-04-21 07:52:45 -070079 'dev-embedded/ti50-sdk',
Mike Frysinger785b0c32017-09-13 01:35:59 -040080)
81
82# New packages that we're in the process of adding to the SDK. Since the SDK
83# bot hasn't had a chance to run yet, there are no binary packages available,
84# so we have to list them here and wait. Once it completes, entries here can
85# be removed so they'll end up on bots & dev's systems.
George Burgess IV66e199c2021-05-05 15:38:40 -070086NEW_PACKAGES = ()
Mike Frysinger785b0c32017-09-13 01:35:59 -040087
Rahul Chaudhry4b803052015-05-13 15:25:56 -070088# Enable the Go compiler for these targets.
89TARGET_GO_ENABLED = (
90 'x86_64-cros-linux-gnu',
Rahul Chaudhry4b803052015-05-13 15:25:56 -070091 'armv7a-cros-linux-gnueabi',
Yunlian Jiang16c1a422017-10-04 10:33:22 -070092 'armv7a-cros-linux-gnueabihf',
Rahul Chaudhry4d416582017-10-25 12:31:58 -070093 'aarch64-cros-linux-gnu',
Rahul Chaudhry4b803052015-05-13 15:25:56 -070094)
95CROSSDEV_GO_ARGS = ['--ex-pkg', 'dev-lang/go']
96
Manoj Gupta1b5642e2017-03-08 16:44:12 -080097# Enable llvm's compiler-rt for these targets.
98TARGET_COMPILER_RT_ENABLED = (
99 'armv7a-cros-linux-gnueabi',
Yunlian Jiang1b77ee42017-10-06 13:44:29 -0700100 'armv7a-cros-linux-gnueabihf',
Manoj Gupta946abb42017-04-12 14:27:19 -0700101 'aarch64-cros-linux-gnu',
Manoj Gupta21f3a082018-03-06 21:25:39 -0800102 'armv7m-cros-eabi',
Manoj Gupta1b5642e2017-03-08 16:44:12 -0800103)
104CROSSDEV_COMPILER_RT_ARGS = ['--ex-pkg', 'sys-libs/compiler-rt']
105
Manoj Gupta946abb42017-04-12 14:27:19 -0700106TARGET_LLVM_PKGS_ENABLED = (
107 'armv7a-cros-linux-gnueabi',
Yunlian Jiang16c1a422017-10-04 10:33:22 -0700108 'armv7a-cros-linux-gnueabihf',
Manoj Gupta946abb42017-04-12 14:27:19 -0700109 'aarch64-cros-linux-gnu',
Manoj Gupta93713122020-10-29 17:52:16 -0700110 'i686-pc-linux-gnu',
Manoj Gupta946abb42017-04-12 14:27:19 -0700111 'x86_64-cros-linux-gnu',
112)
113
114LLVM_PKGS_TABLE = {
Yunlian Jiangbb2a3d32017-04-26 14:44:09 -0700115 'ex_libcxxabi' : ['--ex-pkg', 'sys-libs/libcxxabi'],
116 'ex_libcxx' : ['--ex-pkg', 'sys-libs/libcxx'],
Chirantan Ekbotee694cad2018-07-12 15:07:24 -0700117 'ex_llvm-libunwind' : ['--ex-pkg', 'sys-libs/llvm-libunwind'],
Manoj Gupta946abb42017-04-12 14:27:19 -0700118}
119
David James66a09c42012-11-05 13:31:38 -0800120class Crossdev(object):
121 """Class for interacting with crossdev and caching its output."""
122
123 _CACHE_FILE = os.path.join(CROSSDEV_OVERLAY, '.configured.json')
124 _CACHE = {}
Han Shene23782f2016-02-18 12:20:00 -0800125 # Packages that needs separate handling, in addition to what we have from
126 # crossdev.
127 MANUAL_PKGS = {
George Burgess IVca1d7612020-10-01 00:38:32 -0700128 'rust': 'dev-lang',
Han Shene23782f2016-02-18 12:20:00 -0800129 'llvm': 'sys-devel',
Manoj Gupta20cfc6c2017-07-19 15:49:55 -0700130 'libcxxabi': 'sys-libs',
131 'libcxx': 'sys-libs',
Yunlian Jiangda3ce5f2018-04-25 14:10:01 -0700132 'elfutils': 'dev-libs',
Chirantan Ekbotee694cad2018-07-12 15:07:24 -0700133 'llvm-libunwind': 'sys-libs',
Han Shene23782f2016-02-18 12:20:00 -0800134 }
David James66a09c42012-11-05 13:31:38 -0800135
136 @classmethod
137 def Load(cls, reconfig):
Mike Frysinger3ed47722017-08-08 14:59:08 -0400138 """Load crossdev cache from disk.
139
140 We invalidate the cache when crossdev updates or this script changes.
141 """
David James90239b92012-11-05 15:31:34 -0800142 crossdev_version = GetStablePackageVersion('sys-devel/crossdev', True)
Mike Frysinger3ed47722017-08-08 14:59:08 -0400143 # If we run the compiled/cached .pyc file, we'll read/hash that when we
144 # really always want to track the source .py file.
145 script = os.path.abspath(__file__)
146 if script.endswith('.pyc'):
147 script = script[:-1]
Mike Frysingerb3202be2019-11-15 22:25:59 -0500148 setup_toolchains_hash = hashlib.md5(
149 osutils.ReadFile(script, mode='rb')).hexdigest()
Mike Frysinger3ed47722017-08-08 14:59:08 -0400150
151 cls._CACHE = {
152 'crossdev_version': crossdev_version,
153 'setup_toolchains_hash': setup_toolchains_hash,
154 }
155
156 logging.debug('cache: checking file: %s', cls._CACHE_FILE)
157 if reconfig:
158 logging.debug('cache: forcing regen due to reconfig')
159 return
160
161 try:
162 file_data = osutils.ReadFile(cls._CACHE_FILE)
163 except IOError as e:
164 if e.errno != errno.ENOENT:
165 logging.warning('cache: reading failed: %s', e)
166 osutils.SafeUnlink(cls._CACHE_FILE)
167 return
168
169 try:
170 data = json.loads(file_data)
171 except ValueError as e:
172 logging.warning('cache: ignoring invalid content: %s', e)
173 return
174
175 if crossdev_version != data.get('crossdev_version'):
176 logging.debug('cache: rebuilding after crossdev upgrade')
177 elif setup_toolchains_hash != data.get('setup_toolchains_hash'):
178 logging.debug('cache: rebuilding after cros_setup_toolchains change')
179 else:
180 logging.debug('cache: content is up-to-date!')
181 cls._CACHE = data
David James66a09c42012-11-05 13:31:38 -0800182
183 @classmethod
184 def Save(cls):
185 """Store crossdev cache on disk."""
186 # Save the cache from the successful run.
187 with open(cls._CACHE_FILE, 'w') as f:
188 json.dump(cls._CACHE, f)
189
190 @classmethod
191 def GetConfig(cls, target):
192 """Returns a map of crossdev provided variables about a tuple."""
193 CACHE_ATTR = '_target_tuple_map'
194
195 val = cls._CACHE.setdefault(CACHE_ATTR, {})
196 if not target in val:
Mike Frysinger785b0c32017-09-13 01:35:59 -0400197 if target.startswith('host'):
Mike Frysinger66bfde52017-09-12 16:42:57 -0400198 conf = {
199 'crosspkgs': [],
200 'target': toolchain.GetHostTuple(),
201 }
Mike Frysinger785b0c32017-09-13 01:35:59 -0400202 if target == 'host':
203 packages_list = HOST_PACKAGES
204 else:
205 packages_list = HOST_POST_CROSS_PACKAGES
Mike Frysinger66bfde52017-09-12 16:42:57 -0400206 manual_pkgs = dict((pkg, cat) for cat, pkg in
Mike Frysinger785b0c32017-09-13 01:35:59 -0400207 [x.split('/') for x in packages_list])
Mike Frysinger66bfde52017-09-12 16:42:57 -0400208 else:
209 # Build the crossdev command.
210 cmd = ['crossdev', '--show-target-cfg', '--ex-gdb']
211 if target in TARGET_COMPILER_RT_ENABLED:
212 cmd.extend(CROSSDEV_COMPILER_RT_ARGS)
213 if target in TARGET_GO_ENABLED:
214 cmd.extend(CROSSDEV_GO_ARGS)
215 if target in TARGET_LLVM_PKGS_ENABLED:
216 for pkg in LLVM_PKGS_TABLE:
217 cmd.extend(LLVM_PKGS_TABLE[pkg])
218 cmd.extend(['-t', target])
219 # Catch output of crossdev.
Mike Frysinger45602c72019-09-22 02:15:11 -0400220 out = cros_build_lib.run(
Mike Frysinger0282d222019-12-17 17:15:48 -0500221 cmd, print_cmd=False, stdout=True,
Mike Frysingerb3202be2019-11-15 22:25:59 -0500222 encoding='utf-8').stdout.splitlines()
Mike Frysinger66bfde52017-09-12 16:42:57 -0400223 # List of tuples split at the first '=', converted into dict.
224 conf = dict((k, cros_build_lib.ShellUnquote(v))
225 for k, v in (x.split('=', 1) for x in out))
226 conf['crosspkgs'] = conf['crosspkgs'].split()
Han Shene23782f2016-02-18 12:20:00 -0800227
Mike Frysinger66bfde52017-09-12 16:42:57 -0400228 manual_pkgs = cls.MANUAL_PKGS
229
230 for pkg, cat in manual_pkgs.items():
Mike Frysinger3bba5032016-09-20 14:15:04 -0400231 conf[pkg + '_pn'] = pkg
232 conf[pkg + '_category'] = cat
233 if pkg not in conf['crosspkgs']:
234 conf['crosspkgs'].append(pkg)
Han Shene23782f2016-02-18 12:20:00 -0800235
236 val[target] = conf
237
David James66a09c42012-11-05 13:31:38 -0800238 return val[target]
239
240 @classmethod
241 def UpdateTargets(cls, targets, usepkg, config_only=False):
242 """Calls crossdev to initialize a cross target.
243
244 Args:
Nicolas Boichat3fecf392019-11-25 02:58:28 +0000245 targets: The list of targets to initialize using crossdev.
Don Garrett25f309a2014-03-19 14:02:12 -0700246 usepkg: Copies the commandline opts.
247 config_only: Just update.
David James66a09c42012-11-05 13:31:38 -0800248 """
249 configured_targets = cls._CACHE.setdefault('configured_targets', [])
250
251 cmdbase = ['crossdev', '--show-fail-log']
252 cmdbase.extend(['--env', 'FEATURES=splitdebug'])
253 # Pick stable by default, and override as necessary.
254 cmdbase.extend(['-P', '--oneshot'])
255 if usepkg:
256 cmdbase.extend(['-P', '--getbinpkg',
257 '-P', '--usepkgonly',
258 '--without-headers'])
259
Christopher Wileyb22c0712015-06-02 10:37:03 -0700260 overlays = ' '.join((CHROMIUMOS_OVERLAY, ECLASS_OVERLAY, STABLE_OVERLAY))
David James66a09c42012-11-05 13:31:38 -0800261 cmdbase.extend(['--overlays', overlays])
262 cmdbase.extend(['--ov-output', CROSSDEV_OVERLAY])
263
Nicolas Boichat3fecf392019-11-25 02:58:28 +0000264 # Build target by the reversed alphabetical order to make sure
265 # armv7a-cros-linux-gnueabihf builds before armv7a-cros-linux-gnueabi
266 # because some dependency issue. This can be reverted once we
267 # migrated to armv7a-cros-linux-gnueabihf. crbug.com/711369
268 for target in sorted(targets, reverse=True):
269 if config_only and target in configured_targets:
270 continue
David James66a09c42012-11-05 13:31:38 -0800271
Nicolas Boichat3fecf392019-11-25 02:58:28 +0000272 cmd = cmdbase + ['-t', target]
273
274 for pkg in GetTargetPackages(target):
275 if pkg == 'gdb':
276 # Gdb does not have selectable versions.
277 cmd.append('--ex-gdb')
278 elif pkg == 'ex_compiler-rt':
279 cmd.extend(CROSSDEV_COMPILER_RT_ARGS)
280 elif pkg == 'ex_go':
281 # Go does not have selectable versions.
282 cmd.extend(CROSSDEV_GO_ARGS)
283 elif pkg in LLVM_PKGS_TABLE:
284 cmd.extend(LLVM_PKGS_TABLE[pkg])
285 elif pkg in cls.MANUAL_PKGS:
286 pass
287 else:
288 # The first of the desired versions is the "primary" one.
289 version = GetDesiredPackageVersions(target, pkg)[0]
290 cmd.extend(['--%s' % pkg, version])
291
292 cmd.extend(targets[target]['crossdev'].split())
293 if config_only:
294 # In this case we want to just quietly reinit
295 cmd.append('--init-target')
Mike Frysinger0282d222019-12-17 17:15:48 -0500296 cros_build_lib.run(cmd, print_cmd=False, stdout=True)
David James66a09c42012-11-05 13:31:38 -0800297 else:
Nicolas Boichat3fecf392019-11-25 02:58:28 +0000298 cros_build_lib.run(cmd)
David James66a09c42012-11-05 13:31:38 -0800299
Nicolas Boichat3fecf392019-11-25 02:58:28 +0000300 configured_targets.append(target)
David James66a09c42012-11-05 13:31:38 -0800301
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100302
Zdenek Behanf4d18a02012-03-22 15:45:05 +0100303def GetTargetPackages(target):
304 """Returns a list of packages for a given target."""
David James66a09c42012-11-05 13:31:38 -0800305 conf = Crossdev.GetConfig(target)
Zdenek Behanf4d18a02012-03-22 15:45:05 +0100306 # Undesired packages are denoted by empty ${pkg}_pn variable.
Han Shene23782f2016-02-18 12:20:00 -0800307 return [x for x in conf['crosspkgs'] if conf.get(x+'_pn')]
Zdenek Behanf4d18a02012-03-22 15:45:05 +0100308
309
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100310# Portage helper functions:
311def GetPortagePackage(target, package):
312 """Returns a package name for the given target."""
David James66a09c42012-11-05 13:31:38 -0800313 conf = Crossdev.GetConfig(target)
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100314 # Portage category:
Mike Frysinger785b0c32017-09-13 01:35:59 -0400315 if target.startswith('host') or package in Crossdev.MANUAL_PKGS:
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100316 category = conf[package + '_category']
Zdenek Behan508dcce2011-12-05 15:39:32 +0100317 else:
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100318 category = conf['category']
319 # Portage package:
320 pn = conf[package + '_pn']
321 # Final package name:
Mike Frysinger422faf42014-11-12 23:16:48 -0500322 assert category
323 assert pn
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100324 return '%s/%s' % (category, pn)
325
326
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700327def PortageTrees(root):
328 """Return the portage trees for a given root."""
329 if root == '/':
330 return portage.db['/']
331 # The portage logic requires the path always end in a slash.
332 root = root.rstrip('/') + '/'
333 return portage.create_trees(target_root=root, config_root=root)[root]
334
335
336def GetInstalledPackageVersions(atom, root='/'):
Zdenek Behan508dcce2011-12-05 15:39:32 +0100337 """Extracts the list of current versions of a target, package pair.
338
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500339 Args:
340 atom: The atom to operate on (e.g. sys-devel/gcc)
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700341 root: The root to check for installed packages.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100342
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500343 Returns:
344 The list of versions of the package currently installed.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100345 """
Zdenek Behan508dcce2011-12-05 15:39:32 +0100346 versions = []
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700347 for pkg in PortageTrees(root)['vartree'].dbapi.match(atom, use_cache=0):
Zdenek Behan508dcce2011-12-05 15:39:32 +0100348 version = portage.versions.cpv_getversion(pkg)
349 versions.append(version)
350 return versions
351
352
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700353def GetStablePackageVersion(atom, installed, root='/'):
Zdenek Behan508dcce2011-12-05 15:39:32 +0100354 """Extracts the current stable version for a given package.
355
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500356 Args:
357 atom: The target/package to operate on eg. i686-pc-linux-gnu,gcc
358 installed: Whether we want installed packages or ebuilds
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700359 root: The root to use when querying packages.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100360
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500361 Returns:
362 A string containing the latest version.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100363 """
David James90239b92012-11-05 15:31:34 -0800364 pkgtype = 'vartree' if installed else 'porttree'
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700365 cpv = portage.best(PortageTrees(root)[pkgtype].dbapi.match(atom, use_cache=0))
David James90239b92012-11-05 15:31:34 -0800366 return portage.versions.cpv_getversion(cpv) if cpv else None
Zdenek Behan508dcce2011-12-05 15:39:32 +0100367
368
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700369def VersionListToNumeric(target, package, versions, installed, root='/'):
Zdenek Behan508dcce2011-12-05 15:39:32 +0100370 """Resolves keywords in a given version list for a particular package.
371
372 Resolving means replacing PACKAGE_STABLE with the actual number.
373
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500374 Args:
375 target: The target to operate on (e.g. i686-pc-linux-gnu)
376 package: The target/package to operate on (e.g. gcc)
377 versions: List of versions to resolve
378 installed: Query installed packages
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700379 root: The install root to use; ignored if |installed| is False.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100380
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500381 Returns:
382 List of purely numeric versions equivalent to argument
Zdenek Behan508dcce2011-12-05 15:39:32 +0100383 """
384 resolved = []
David James90239b92012-11-05 15:31:34 -0800385 atom = GetPortagePackage(target, package)
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700386 if not installed:
387 root = '/'
Zdenek Behan508dcce2011-12-05 15:39:32 +0100388 for version in versions:
389 if version == PACKAGE_STABLE:
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700390 resolved.append(GetStablePackageVersion(atom, installed, root=root))
Mike Frysinger7f0e1982017-09-12 17:08:50 -0400391 else:
Zdenek Behan508dcce2011-12-05 15:39:32 +0100392 resolved.append(version)
393 return resolved
394
395
396def GetDesiredPackageVersions(target, package):
397 """Produces the list of desired versions for each target, package pair.
398
399 The first version in the list is implicitly treated as primary, ie.
400 the version that will be initialized by crossdev and selected.
401
402 If the version is PACKAGE_STABLE, it really means the current version which
403 is emerged by using the package atom with no particular version key.
404 Since crossdev unmasks all packages by default, this will actually
405 mean 'unstable' in most cases.
406
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500407 Args:
408 target: The target to operate on (e.g. i686-pc-linux-gnu)
409 package: The target/package to operate on (e.g. gcc)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100410
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500411 Returns:
412 A list composed of either a version string, PACKAGE_STABLE
Zdenek Behan508dcce2011-12-05 15:39:32 +0100413 """
Mike Frysingerd23be272017-09-12 17:18:44 -0400414 if package in GetTargetPackages(target):
415 return [PACKAGE_STABLE]
416 else:
417 return []
Zdenek Behan508dcce2011-12-05 15:39:32 +0100418
419
420def TargetIsInitialized(target):
421 """Verifies if the given list of targets has been correctly initialized.
422
423 This determines whether we have to call crossdev while emerging
424 toolchain packages or can do it using emerge. Emerge is naturally
425 preferred, because all packages can be updated in a single pass.
426
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500427 Args:
428 target: The target to operate on (e.g. i686-pc-linux-gnu)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100429
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500430 Returns:
431 True if |target| is completely initialized, else False
Zdenek Behan508dcce2011-12-05 15:39:32 +0100432 """
433 # Check if packages for the given target all have a proper version.
434 try:
Zdenek Behanf4d18a02012-03-22 15:45:05 +0100435 for package in GetTargetPackages(target):
David James66a09c42012-11-05 13:31:38 -0800436 atom = GetPortagePackage(target, package)
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100437 # Do we even want this package && is it initialized?
Mike Frysinger7f0e1982017-09-12 17:08:50 -0400438 if not (GetStablePackageVersion(atom, True) and
439 GetStablePackageVersion(atom, False)):
Zdenek Behan508dcce2011-12-05 15:39:32 +0100440 return False
441 return True
442 except cros_build_lib.RunCommandError:
443 # Fails - The target has likely never been initialized before.
444 return False
445
446
447def RemovePackageMask(target):
448 """Removes a package.mask file for the given platform.
449
450 The pre-existing package.mask files can mess with the keywords.
451
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500452 Args:
453 target: The target to operate on (e.g. i686-pc-linux-gnu)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100454 """
455 maskfile = os.path.join('/etc/portage/package.mask', 'cross-' + target)
Brian Harringaf019fb2012-05-10 15:06:13 -0700456 osutils.SafeUnlink(maskfile)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100457
458
Zdenek Behan508dcce2011-12-05 15:39:32 +0100459# Main functions performing the actual update steps.
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700460def RebuildLibtool(root='/'):
Mike Frysingerc880a962013-11-08 13:59:06 -0500461 """Rebuild libtool as needed
462
463 Libtool hardcodes full paths to internal gcc files, so whenever we upgrade
464 gcc, libtool will break. We can't use binary packages either as those will
465 most likely be compiled against the previous version of gcc.
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700466
467 Args:
468 root: The install root where we want libtool rebuilt.
Mike Frysingerc880a962013-11-08 13:59:06 -0500469 """
470 needs_update = False
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700471 with open(os.path.join(root, 'usr/bin/libtool')) as f:
Mike Frysingerc880a962013-11-08 13:59:06 -0500472 for line in f:
473 # Look for a line like:
474 # sys_lib_search_path_spec="..."
475 # It'll be a list of paths and gcc will be one of them.
476 if line.startswith('sys_lib_search_path_spec='):
477 line = line.rstrip()
478 for path in line.split('=', 1)[1].strip('"').split():
Mike Frysinger3bba5032016-09-20 14:15:04 -0400479 root_path = os.path.join(root, path.lstrip(os.path.sep))
480 logging.debug('Libtool: checking %s', root_path)
481 if not os.path.exists(root_path):
482 logging.info('Rebuilding libtool after gcc upgrade')
483 logging.info(' %s', line)
484 logging.info(' missing path: %s', path)
Mike Frysingerc880a962013-11-08 13:59:06 -0500485 needs_update = True
486 break
487
488 if needs_update:
489 break
490
491 if needs_update:
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700492 cmd = [EMERGE_CMD, '--oneshot']
493 if root != '/':
494 cmd.extend(['--sysroot=%s' % root, '--root=%s' % root])
495 cmd.append('sys-devel/libtool')
Mike Frysinger45602c72019-09-22 02:15:11 -0400496 cros_build_lib.run(cmd)
Mike Frysinger3bba5032016-09-20 14:15:04 -0400497 else:
498 logging.debug('Libtool is up-to-date; no need to rebuild')
Mike Frysingerc880a962013-11-08 13:59:06 -0500499
500
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700501def UpdateTargets(targets, usepkg, root='/'):
Zdenek Behan508dcce2011-12-05 15:39:32 +0100502 """Determines which packages need update/unmerge and defers to portage.
503
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500504 Args:
505 targets: The list of targets to update
506 usepkg: Copies the commandline option
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700507 root: The install root in which we want packages updated.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100508 """
Zdenek Behan508dcce2011-12-05 15:39:32 +0100509 # For each target, we do two things. Figure out the list of updates,
510 # and figure out the appropriate keywords/masks. Crossdev will initialize
511 # these, but they need to be regenerated on every update.
Mike Frysinger3bba5032016-09-20 14:15:04 -0400512 logging.info('Determining required toolchain updates...')
David James90239b92012-11-05 15:31:34 -0800513 mergemap = {}
ChromeOS Developereb8a5c42021-01-12 00:05:02 +0000514 # Used to keep track of post-cross packages. These are allowed to have
515 # implicit dependencies on toolchain packages, and therefore need to
516 # be built last.
517 post_cross_pkgs = set()
Zdenek Behan508dcce2011-12-05 15:39:32 +0100518 for target in targets:
ChromeOS Developereb8a5c42021-01-12 00:05:02 +0000519 is_post_cross_target = target.endswith('-post-cross')
Mike Frysinger3bba5032016-09-20 14:15:04 -0400520 logging.debug('Updating target %s', target)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100521 # Record the highest needed version for each target, for masking purposes.
522 RemovePackageMask(target)
Zdenek Behanf4d18a02012-03-22 15:45:05 +0100523 for package in GetTargetPackages(target):
Zdenek Behan508dcce2011-12-05 15:39:32 +0100524 # Portage name for the package
Mike Frysinger7f0e1982017-09-12 17:08:50 -0400525 logging.debug(' Checking package %s', package)
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100526 pkg = GetPortagePackage(target, package)
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700527 current = GetInstalledPackageVersions(pkg, root=root)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100528 desired = GetDesiredPackageVersions(target, package)
Zdenek Behan699ddd32012-04-13 07:14:08 +0200529 desired_num = VersionListToNumeric(target, package, desired, False)
Mike Frysinger785b0c32017-09-13 01:35:59 -0400530 if pkg in NEW_PACKAGES and usepkg:
531 # Skip this binary package (for now).
532 continue
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100533 mergemap[pkg] = set(desired_num).difference(current)
Mike Frysinger7f0e1982017-09-12 17:08:50 -0400534 logging.debug(' %s -> %s', current, desired_num)
ChromeOS Developereb8a5c42021-01-12 00:05:02 +0000535 if is_post_cross_target:
536 post_cross_pkgs.add(pkg)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100537
Mike Frysinger7f0e1982017-09-12 17:08:50 -0400538 packages = [pkg for pkg, vers in mergemap.items() if vers]
Zdenek Behan508dcce2011-12-05 15:39:32 +0100539 if not packages:
Mike Frysinger3bba5032016-09-20 14:15:04 -0400540 logging.info('Nothing to update!')
David Jamesf8c672f2012-11-06 13:38:11 -0800541 return False
Zdenek Behan508dcce2011-12-05 15:39:32 +0100542
Mike Frysinger3bba5032016-09-20 14:15:04 -0400543 logging.info('Updating packages:')
544 logging.info('%s', packages)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100545
Zdenek Behan7e33b4e2012-03-12 17:00:56 +0100546 cmd = [EMERGE_CMD, '--oneshot', '--update']
Zdenek Behan508dcce2011-12-05 15:39:32 +0100547 if usepkg:
548 cmd.extend(['--getbinpkg', '--usepkgonly'])
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700549 if root != '/':
550 cmd.extend(['--sysroot=%s' % root, '--root=%s' % root])
Zdenek Behan508dcce2011-12-05 15:39:32 +0100551
ChromeOS Developereb8a5c42021-01-12 00:05:02 +0000552 if usepkg:
553 # Since we are not building from source, we can handle
554 # all packages in one go.
555 cmd.extend(packages)
556 cros_build_lib.run(cmd)
557 else:
George Burgess IVd7762ab2021-04-29 10:54:55 -0700558 pre_cross_items = [pkg for pkg in packages if pkg not in post_cross_pkgs]
559 if pre_cross_items:
560 cros_build_lib.run(cmd + pre_cross_items)
ChromeOS Developereb8a5c42021-01-12 00:05:02 +0000561 post_cross_items = [pkg for pkg in packages if pkg in post_cross_pkgs]
George Burgess IVd7762ab2021-04-29 10:54:55 -0700562 if post_cross_items:
563 cros_build_lib.run(cmd + post_cross_items)
David Jamesf8c672f2012-11-06 13:38:11 -0800564 return True
Zdenek Behan508dcce2011-12-05 15:39:32 +0100565
566
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700567def CleanTargets(targets, root='/'):
568 """Unmerges old packages that are assumed unnecessary.
569
570 Args:
571 targets: The list of targets to clean up.
572 root: The install root in which we want packages cleaned up.
573 """
Zdenek Behan508dcce2011-12-05 15:39:32 +0100574 unmergemap = {}
575 for target in targets:
Mike Frysinger3bba5032016-09-20 14:15:04 -0400576 logging.debug('Cleaning target %s', target)
Zdenek Behanf4d18a02012-03-22 15:45:05 +0100577 for package in GetTargetPackages(target):
Mike Frysinger3bba5032016-09-20 14:15:04 -0400578 logging.debug(' Cleaning package %s', package)
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100579 pkg = GetPortagePackage(target, package)
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700580 current = GetInstalledPackageVersions(pkg, root=root)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100581 desired = GetDesiredPackageVersions(target, package)
Gilad Arnold2fcb0aa2015-05-21 14:51:41 -0700582 # NOTE: This refers to installed packages (vartree) rather than the
583 # Portage version (porttree and/or bintree) when determining the current
584 # version. While this isn't the most accurate thing to do, it is probably
585 # a good simple compromise, which should have the desired result of
586 # uninstalling everything but the latest installed version. In
587 # particular, using the bintree (--usebinpkg) requires a non-trivial
588 # binhost sync and is probably more complex than useful.
Zdenek Behan699ddd32012-04-13 07:14:08 +0200589 desired_num = VersionListToNumeric(target, package, desired, True)
590 if not set(desired_num).issubset(current):
Mike Frysinger3bba5032016-09-20 14:15:04 -0400591 logging.warning('Error detecting stable version for %s, '
592 'skipping clean!', pkg)
Zdenek Behan699ddd32012-04-13 07:14:08 +0200593 return
Zdenek Behan508dcce2011-12-05 15:39:32 +0100594 unmergemap[pkg] = set(current).difference(desired_num)
595
596 # Cleaning doesn't care about consistency and rebuilding package.* files.
597 packages = []
Mike Frysinger0bdbc102019-06-13 15:27:29 -0400598 for pkg, vers in unmergemap.items():
Zdenek Behan508dcce2011-12-05 15:39:32 +0100599 packages.extend('=%s-%s' % (pkg, ver) for ver in vers if ver != '9999')
600
601 if packages:
Mike Frysinger3bba5032016-09-20 14:15:04 -0400602 logging.info('Cleaning packages:')
603 logging.info('%s', packages)
Zdenek Behan7e33b4e2012-03-12 17:00:56 +0100604 cmd = [EMERGE_CMD, '--unmerge']
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700605 if root != '/':
606 cmd.extend(['--sysroot=%s' % root, '--root=%s' % root])
Zdenek Behan508dcce2011-12-05 15:39:32 +0100607 cmd.extend(packages)
Mike Frysinger45602c72019-09-22 02:15:11 -0400608 cros_build_lib.run(cmd)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100609 else:
Mike Frysinger3bba5032016-09-20 14:15:04 -0400610 logging.info('Nothing to clean!')
Zdenek Behan508dcce2011-12-05 15:39:32 +0100611
612
Adrian Ratiu78e534d2021-01-06 19:58:38 +0200613def SelectActiveToolchains(targets, root='/'):
Zdenek Behan508dcce2011-12-05 15:39:32 +0100614 """Runs gcc-config and binutils-config to select the desired.
615
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500616 Args:
617 targets: The targets to select
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700618 root: The root where we want to select toolchain versions.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100619 """
620 for package in ['gcc', 'binutils']:
621 for target in targets:
Mike Frysinger785b0c32017-09-13 01:35:59 -0400622 # See if this package is part of this target.
623 if package not in GetTargetPackages(target):
624 logging.debug('%s: %s is not used', target, package)
625 continue
626
Zdenek Behan508dcce2011-12-05 15:39:32 +0100627 # Pick the first version in the numbered list as the selected one.
628 desired = GetDesiredPackageVersions(target, package)
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700629 desired_num = VersionListToNumeric(target, package, desired, True,
630 root=root)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100631 desired = desired_num[0]
632 # *-config does not play revisions, strip them, keep just PV.
633 desired = portage.versions.pkgsplit('%s-%s' % (package, desired))[1]
634
Mike Frysinger785b0c32017-09-13 01:35:59 -0400635 if target.startswith('host'):
Zdenek Behan508dcce2011-12-05 15:39:32 +0100636 # *-config is the only tool treating host identically (by tuple).
David James27ac4ae2012-12-03 23:16:15 -0800637 target = toolchain.GetHostTuple()
Zdenek Behan508dcce2011-12-05 15:39:32 +0100638
639 # And finally, attach target to it.
640 desired = '%s-%s' % (target, desired)
641
David James7ec5efc2012-11-06 09:39:49 -0800642 extra_env = {'CHOST': target}
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700643 if root != '/':
644 extra_env['ROOT'] = root
David James7ec5efc2012-11-06 09:39:49 -0800645 cmd = ['%s-config' % package, '-c', target]
Mike Frysinger45602c72019-09-22 02:15:11 -0400646 result = cros_build_lib.run(
Mike Frysinger0282d222019-12-17 17:15:48 -0500647 cmd, print_cmd=False, stdout=True, encoding='utf-8',
Mike Frysingerb3202be2019-11-15 22:25:59 -0500648 extra_env=extra_env)
Mike Frysingerd6e2df02014-11-26 02:55:04 -0500649 current = result.output.splitlines()[0]
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700650
651 # Do not reconfig when the current is live or nothing needs to be done.
652 extra_env = {'ROOT': root} if root != '/' else None
Zdenek Behan508dcce2011-12-05 15:39:32 +0100653 if current != desired and current != '9999':
Mike Frysingerd6e2df02014-11-26 02:55:04 -0500654 cmd = [package + '-config', desired]
Mike Frysinger45602c72019-09-22 02:15:11 -0400655 cros_build_lib.run(cmd, print_cmd=False, extra_env=extra_env)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100656
657
Mike Frysinger35247af2012-11-16 18:58:06 -0500658def ExpandTargets(targets_wanted):
659 """Expand any possible toolchain aliases into full targets
660
661 This will expand 'all' and 'sdk' into the respective toolchain tuples.
662
663 Args:
664 targets_wanted: The targets specified by the user.
Mike Frysinger1a736a82013-12-12 01:50:59 -0500665
Mike Frysinger35247af2012-11-16 18:58:06 -0500666 Returns:
Gilad Arnold8195b532015-04-07 10:56:30 +0300667 Dictionary of concrete targets and their toolchain tuples.
Mike Frysinger35247af2012-11-16 18:58:06 -0500668 """
Mike Frysinger35247af2012-11-16 18:58:06 -0500669 targets_wanted = set(targets_wanted)
Don Garrettc0c74002015-10-09 12:58:19 -0700670 if targets_wanted == set(['boards']):
671 # Only pull targets from the included boards.
Gilad Arnold8195b532015-04-07 10:56:30 +0300672 return {}
673
674 all_targets = toolchain.GetAllTargets()
Mike Frysinger35247af2012-11-16 18:58:06 -0500675 if targets_wanted == set(['all']):
Gilad Arnold8195b532015-04-07 10:56:30 +0300676 return all_targets
677 if targets_wanted == set(['sdk']):
Mike Frysinger35247af2012-11-16 18:58:06 -0500678 # Filter out all the non-sdk toolchains as we don't want to mess
679 # with those in all of our builds.
Gilad Arnold8195b532015-04-07 10:56:30 +0300680 return toolchain.FilterToolchains(all_targets, 'sdk', True)
681
682 # Verify user input.
683 nonexistent = targets_wanted.difference(all_targets)
684 if nonexistent:
Mike Frysingercd790662019-10-17 00:23:13 -0400685 raise ValueError('Invalid targets: %s' % (','.join(nonexistent),))
Gilad Arnold8195b532015-04-07 10:56:30 +0300686 return {t: all_targets[t] for t in targets_wanted}
Mike Frysinger35247af2012-11-16 18:58:06 -0500687
688
David Jamesf8c672f2012-11-06 13:38:11 -0800689def UpdateToolchains(usepkg, deleteold, hostonly, reconfig,
Don Garrettc0c74002015-10-09 12:58:19 -0700690 targets_wanted, boards_wanted, root='/'):
Zdenek Behan508dcce2011-12-05 15:39:32 +0100691 """Performs all steps to create a synchronized toolchain enviroment.
692
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500693 Args:
694 usepkg: Use prebuilt packages
695 deleteold: Unmerge deprecated packages
696 hostonly: Only setup the host toolchain
697 reconfig: Reload crossdev config and reselect toolchains
698 targets_wanted: All the targets to update
699 boards_wanted: Load targets from these boards
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700700 root: The root in which to install the toolchains.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100701 """
David Jamesf8c672f2012-11-06 13:38:11 -0800702 targets, crossdev_targets, reconfig_targets = {}, {}, {}
Zdenek Behan7e33b4e2012-03-12 17:00:56 +0100703 if not hostonly:
704 # For hostonly, we can skip most of the below logic, much of which won't
705 # work on bare systems where this is useful.
Mike Frysinger35247af2012-11-16 18:58:06 -0500706 targets = ExpandTargets(targets_wanted)
Mike Frysinger7ccee992012-06-01 21:27:59 -0400707
Don Garrettc0c74002015-10-09 12:58:19 -0700708 # Now re-add any targets that might be from this board. This is to
Gilad Arnold8195b532015-04-07 10:56:30 +0300709 # allow unofficial boards to declare their own toolchains.
Mike Frysinger7ccee992012-06-01 21:27:59 -0400710 for board in boards_wanted:
David James27ac4ae2012-12-03 23:16:15 -0800711 targets.update(toolchain.GetToolchainsForBoard(board))
Zdenek Behan508dcce2011-12-05 15:39:32 +0100712
Zdenek Behan7e33b4e2012-03-12 17:00:56 +0100713 # First check and initialize all cross targets that need to be.
Mike Frysinger7ccee992012-06-01 21:27:59 -0400714 for target in targets:
715 if TargetIsInitialized(target):
716 reconfig_targets[target] = targets[target]
717 else:
718 crossdev_targets[target] = targets[target]
Zdenek Behan7e33b4e2012-03-12 17:00:56 +0100719 if crossdev_targets:
Mike Frysinger3bba5032016-09-20 14:15:04 -0400720 logging.info('The following targets need to be re-initialized:')
721 logging.info('%s', crossdev_targets)
David James66a09c42012-11-05 13:31:38 -0800722 Crossdev.UpdateTargets(crossdev_targets, usepkg)
Zdenek Behan8be29ba2012-05-29 23:10:34 +0200723 # Those that were not initialized may need a config update.
David James66a09c42012-11-05 13:31:38 -0800724 Crossdev.UpdateTargets(reconfig_targets, usepkg, config_only=True)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100725
Mike Frysinger66814c32017-10-09 18:11:46 -0400726 # If we're building a subset of toolchains for a board, we might not have
727 # all the tuples that the packages expect. We don't define the "full" set
728 # of tuples currently other than "whatever the full sdk has normally".
729 if usepkg or set(('all', 'sdk')) & targets_wanted:
730 # Since we have cross-compilers now, we can update these packages.
731 targets['host-post-cross'] = {}
Mike Frysinger785b0c32017-09-13 01:35:59 -0400732
Zdenek Behan7e33b4e2012-03-12 17:00:56 +0100733 # We want host updated.
Mike Frysinger7ccee992012-06-01 21:27:59 -0400734 targets['host'] = {}
Zdenek Behan7e33b4e2012-03-12 17:00:56 +0100735
736 # Now update all packages.
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700737 if UpdateTargets(targets, usepkg, root=root) or crossdev_targets or reconfig:
Adrian Ratiu78e534d2021-01-06 19:58:38 +0200738 SelectActiveToolchains(targets, root=root)
David James7ec5efc2012-11-06 09:39:49 -0800739
740 if deleteold:
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700741 CleanTargets(targets, root=root)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100742
Mike Frysingerc880a962013-11-08 13:59:06 -0500743 # Now that we've cleared out old versions, see if we need to rebuild
744 # anything. Can't do this earlier as it might not be broken.
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700745 RebuildLibtool(root=root)
Mike Frysingerc880a962013-11-08 13:59:06 -0500746
Zdenek Behan508dcce2011-12-05 15:39:32 +0100747
Bertrand SIMONNETcae9d5f2015-03-09 15:58:01 -0700748def ShowConfig(name):
749 """Show the toolchain tuples used by |name|
Mike Frysinger35247af2012-11-16 18:58:06 -0500750
751 Args:
Don Garrettc0c74002015-10-09 12:58:19 -0700752 name: The board name to query.
Mike Frysinger35247af2012-11-16 18:58:06 -0500753 """
Don Garrettc0c74002015-10-09 12:58:19 -0700754 toolchains = toolchain.GetToolchainsForBoard(name)
Mike Frysinger35247af2012-11-16 18:58:06 -0500755 # Make sure we display the default toolchain first.
Mike Frysinger3bba5032016-09-20 14:15:04 -0400756 # Note: Do not use logging here as this is meant to be used by other tools.
Mike Frysinger383367e2014-09-16 15:06:17 -0400757 print(','.join(
Mike Frysinger818d9632019-08-24 14:43:05 -0400758 list(toolchain.FilterToolchains(toolchains, 'default', True)) +
759 list(toolchain.FilterToolchains(toolchains, 'default', False))))
Mike Frysinger35247af2012-11-16 18:58:06 -0500760
761
Mike Frysinger35247af2012-11-16 18:58:06 -0500762def GeneratePathWrapper(root, wrappath, path):
763 """Generate a shell script to execute another shell script
764
765 Since we can't symlink a wrapped ELF (see GenerateLdsoWrapper) because the
766 argv[0] won't be pointing to the correct path, generate a shell script that
767 just executes another program with its full path.
768
769 Args:
770 root: The root tree to generate scripts inside of
771 wrappath: The full path (inside |root|) to create the wrapper
772 path: The target program which this wrapper will execute
773 """
774 replacements = {
775 'path': path,
776 'relroot': os.path.relpath('/', os.path.dirname(wrappath)),
777 }
Takuto Ikuta58403972018-08-16 18:52:51 +0900778
779 # Do not use exec here, because exec invokes script with absolute path in
780 # argv0. Keeping relativeness allows us to remove abs path from compile result
781 # and leads directory independent build cache sharing in some distributed
782 # build system.
Mike Frysinger35247af2012-11-16 18:58:06 -0500783 wrapper = """#!/bin/sh
Takuto Ikuta58403972018-08-16 18:52:51 +0900784basedir=$(dirname "$0")
785"${basedir}/%(relroot)s%(path)s" "$@"
786exit "$?"
Mike Frysinger35247af2012-11-16 18:58:06 -0500787""" % replacements
788 root_wrapper = root + wrappath
789 if os.path.islink(root_wrapper):
790 os.unlink(root_wrapper)
791 else:
792 osutils.SafeMakedirs(os.path.dirname(root_wrapper))
793 osutils.WriteFile(root_wrapper, wrapper)
Mike Frysinger60ec1012013-10-21 00:11:10 -0400794 os.chmod(root_wrapper, 0o755)
Mike Frysinger35247af2012-11-16 18:58:06 -0500795
796
Rahul Chaudhrya8127bb2016-07-22 15:53:17 -0700797def FixClangXXWrapper(root, path):
798 """Fix wrapper shell scripts and symlinks for invoking clang++
799
800 In a typical installation, clang++ symlinks to clang, which symlinks to the
801 elf executable. The executable distinguishes between clang and clang++ based
802 on argv[0].
803
804 When invoked through the LdsoWrapper, argv[0] always contains the path to the
805 executable elf file, making clang/clang++ invocations indistinguishable.
806
807 This function detects if the elf executable being wrapped is clang-X.Y, and
808 fixes wrappers/symlinks as necessary so that clang++ will work correctly.
809
810 The calling sequence now becomes:
811 -) clang++ invocation turns into clang++-3.9 (which is a copy of clang-3.9,
812 the Ldsowrapper).
813 -) clang++-3.9 uses the Ldso to invoke clang++-3.9.elf, which is a symlink
814 to the original clang-3.9 elf.
815 -) The difference this time is that inside the elf file execution, $0 is
816 set as .../usr/bin/clang++-3.9.elf, which contains 'clang++' in the name.
817
Manoj Guptaae268142018-04-27 23:28:36 -0700818 Update: Starting since clang 7, the clang and clang++ are symlinks to
819 clang-7 binary, not clang-7.0. The pattern match is extended to handle
820 both clang-7 and clang-7.0 cases for now. (https://crbug.com/837889)
821
Rahul Chaudhrya8127bb2016-07-22 15:53:17 -0700822 Args:
823 root: The root tree to generate scripts / symlinks inside of
824 path: The target elf for which LdsoWrapper was created
825 """
Manoj Guptaae268142018-04-27 23:28:36 -0700826 if re.match(r'/usr/bin/clang-\d+(\.\d+)*$', path):
Rahul Chaudhrya8127bb2016-07-22 15:53:17 -0700827 logging.info('fixing clang++ invocation for %s', path)
828 clangdir = os.path.dirname(root + path)
829 clang = os.path.basename(path)
830 clangxx = clang.replace('clang', 'clang++')
831
832 # Create a symlink clang++-X.Y.elf to point to clang-X.Y.elf
833 os.symlink(clang + '.elf', os.path.join(clangdir, clangxx + '.elf'))
834
835 # Create a hardlink clang++-X.Y pointing to clang-X.Y
836 os.link(os.path.join(clangdir, clang), os.path.join(clangdir, clangxx))
837
838 # Adjust the clang++ symlink to point to clang++-X.Y
839 os.unlink(os.path.join(clangdir, 'clang++'))
840 os.symlink(clangxx, os.path.join(clangdir, 'clang++'))
841
842
Mike Frysinger35247af2012-11-16 18:58:06 -0500843def FileIsCrosSdkElf(elf):
844 """Determine if |elf| is an ELF that we execute in the cros_sdk
845
846 We don't need this to be perfect, just quick. It makes sure the ELF
847 is a 64bit LSB x86_64 ELF. That is the native type of cros_sdk.
848
849 Args:
850 elf: The file to check
Mike Frysinger1a736a82013-12-12 01:50:59 -0500851
Mike Frysinger35247af2012-11-16 18:58:06 -0500852 Returns:
853 True if we think |elf| is a native ELF
854 """
Mike Frysinger4a12bf12019-12-04 04:20:10 -0500855 with open(elf, 'rb') as f:
Mike Frysinger35247af2012-11-16 18:58:06 -0500856 data = f.read(20)
857 # Check the magic number, EI_CLASS, EI_DATA, and e_machine.
Mike Frysinger4a12bf12019-12-04 04:20:10 -0500858 return (data[0:4] == b'\x7fELF' and
Mike Frysingerdd34dfe2019-12-10 16:25:47 -0500859 data[4:5] == b'\x02' and
860 data[5:6] == b'\x01' and
861 data[18:19] == b'\x3e')
Mike Frysinger35247af2012-11-16 18:58:06 -0500862
863
864def IsPathPackagable(ptype, path):
865 """Should the specified file be included in a toolchain package?
866
867 We only need to handle files as we'll create dirs as we need them.
868
869 Further, trim files that won't be useful:
870 - non-english translations (.mo) since it'd require env vars
871 - debug files since these are for the host compiler itself
872 - info/man pages as they're big, and docs are online, and the
873 native docs should work fine for the most part (`man gcc`)
874
875 Args:
876 ptype: A string describing the path type (i.e. 'file' or 'dir' or 'sym')
877 path: The full path to inspect
Mike Frysinger1a736a82013-12-12 01:50:59 -0500878
Mike Frysinger35247af2012-11-16 18:58:06 -0500879 Returns:
880 True if we want to include this path in the package
881 """
882 return not (ptype in ('dir',) or
883 path.startswith('/usr/lib/debug/') or
884 os.path.splitext(path)[1] == '.mo' or
885 ('/man/' in path or '/info/' in path))
886
887
888def ReadlinkRoot(path, root):
889 """Like os.readlink(), but relative to a |root|
890
891 Args:
892 path: The symlink to read
893 root: The path to use for resolving absolute symlinks
Mike Frysinger1a736a82013-12-12 01:50:59 -0500894
Mike Frysinger35247af2012-11-16 18:58:06 -0500895 Returns:
896 A fully resolved symlink path
897 """
898 while os.path.islink(root + path):
899 path = os.path.join(os.path.dirname(path), os.readlink(root + path))
900 return path
901
902
903def _GetFilesForTarget(target, root='/'):
904 """Locate all the files to package for |target|
905
906 This does not cover ELF dependencies.
907
908 Args:
909 target: The toolchain target name
910 root: The root path to pull all packages from
Mike Frysinger1a736a82013-12-12 01:50:59 -0500911
Mike Frysinger35247af2012-11-16 18:58:06 -0500912 Returns:
913 A tuple of a set of all packable paths, and a set of all paths which
914 are also native ELFs
915 """
916 paths = set()
917 elfs = set()
918
919 # Find all the files owned by the packages for this target.
920 for pkg in GetTargetPackages(target):
Mike Frysinger35247af2012-11-16 18:58:06 -0500921
Rahul Chaudhry4b803052015-05-13 15:25:56 -0700922 # Skip Go compiler from redistributable packages.
923 # The "go" executable has GOROOT=/usr/lib/go/${CTARGET} hardcoded
924 # into it. Due to this, the toolchain cannot be unpacked anywhere
925 # else and be readily useful. To enable packaging Go, we need to:
926 # -) Tweak the wrappers/environment to override GOROOT
927 # automatically based on the unpack location.
928 # -) Make sure the ELF dependency checking and wrapping logic
929 # below skips the Go toolchain executables and libraries.
930 # -) Make sure the packaging process maintains the relative
931 # timestamps of precompiled standard library packages.
932 # (see dev-lang/go ebuild for details).
933 if pkg == 'ex_go':
934 continue
935
Yunlian Jiang36f35242018-04-27 10:18:40 -0700936 # Use armv7a-cros-linux-gnueabi/compiler-rt for
937 # armv7a-cros-linux-gnueabihf/compiler-rt.
938 # Currently the armv7a-cros-linux-gnueabi is actually
939 # the same as armv7a-cros-linux-gnueabihf with different names.
940 # Because of that, for compiler-rt, it generates the same binary in
941 # the same location. To avoid the installation conflict, we do not
942 # install anything for 'armv7a-cros-linux-gnueabihf'. This would cause
943 # problem if other people try to use standalone armv7a-cros-linux-gnueabihf
944 # toolchain.
945 if 'compiler-rt' in pkg and 'armv7a-cros-linux-gnueabi' in target:
946 atom = GetPortagePackage(target, pkg)
947 cat, pn = atom.split('/')
948 ver = GetInstalledPackageVersions(atom, root=root)[0]
Yunlian Jiang36f35242018-04-27 10:18:40 -0700949 dblink = portage.dblink(cat, pn + '-' + ver, myroot=root,
950 settings=portage.settings)
951 contents = dblink.getcontents()
952 if not contents:
953 if 'hf' in target:
954 new_target = 'armv7a-cros-linux-gnueabi'
955 else:
956 new_target = 'armv7a-cros-linux-gnueabihf'
957 atom = GetPortagePackage(new_target, pkg)
958 else:
959 atom = GetPortagePackage(target, pkg)
960
Mike Frysinger35247af2012-11-16 18:58:06 -0500961 cat, pn = atom.split('/')
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700962 ver = GetInstalledPackageVersions(atom, root=root)[0]
Ralph Nathan03047282015-03-23 11:09:32 -0700963 logging.info('packaging %s-%s', atom, ver)
Mike Frysinger35247af2012-11-16 18:58:06 -0500964
Mike Frysinger35247af2012-11-16 18:58:06 -0500965 dblink = portage.dblink(cat, pn + '-' + ver, myroot=root,
966 settings=portage.settings)
967 contents = dblink.getcontents()
968 for obj in contents:
969 ptype = contents[obj][0]
970 if not IsPathPackagable(ptype, obj):
971 continue
972
973 if ptype == 'obj':
974 # For native ELFs, we need to pull in their dependencies too.
975 if FileIsCrosSdkElf(obj):
Mike Frysinger60a2f6c2019-12-04 04:18:58 -0500976 logging.debug('Adding ELF %s', obj)
Mike Frysinger35247af2012-11-16 18:58:06 -0500977 elfs.add(obj)
Mike Frysinger60a2f6c2019-12-04 04:18:58 -0500978 logging.debug('Adding path %s', obj)
Mike Frysinger35247af2012-11-16 18:58:06 -0500979 paths.add(obj)
980
981 return paths, elfs
982
983
984def _BuildInitialPackageRoot(output_dir, paths, elfs, ldpaths,
Mike Frysingerd6e2df02014-11-26 02:55:04 -0500985 path_rewrite_func=lambda x: x, root='/'):
Mike Frysinger35247af2012-11-16 18:58:06 -0500986 """Link in all packable files and their runtime dependencies
987
988 This also wraps up executable ELFs with helper scripts.
989
990 Args:
991 output_dir: The output directory to store files
992 paths: All the files to include
993 elfs: All the files which are ELFs (a subset of |paths|)
994 ldpaths: A dict of static ldpath information
995 path_rewrite_func: User callback to rewrite paths in output_dir
996 root: The root path to pull all packages/files from
997 """
998 # Link in all the files.
Mike Frysinger221bd822017-09-29 02:51:47 -0400999 sym_paths = {}
Mike Frysinger35247af2012-11-16 18:58:06 -05001000 for path in paths:
1001 new_path = path_rewrite_func(path)
Mike Frysinger60a2f6c2019-12-04 04:18:58 -05001002 logging.debug('Transformed %s to %s', path, new_path)
Mike Frysinger35247af2012-11-16 18:58:06 -05001003 dst = output_dir + new_path
1004 osutils.SafeMakedirs(os.path.dirname(dst))
1005
1006 # Is this a symlink which we have to rewrite or wrap?
1007 # Delay wrap check until after we have created all paths.
1008 src = root + path
1009 if os.path.islink(src):
1010 tgt = os.readlink(src)
1011 if os.path.sep in tgt:
Mike Frysinger221bd822017-09-29 02:51:47 -04001012 sym_paths[lddtree.normpath(ReadlinkRoot(src, root))] = new_path
Mike Frysinger35247af2012-11-16 18:58:06 -05001013
1014 # Rewrite absolute links to relative and then generate the symlink
1015 # ourselves. All other symlinks can be hardlinked below.
1016 if tgt[0] == '/':
1017 tgt = os.path.relpath(tgt, os.path.dirname(new_path))
1018 os.symlink(tgt, dst)
1019 continue
1020
Mike Frysinger60a2f6c2019-12-04 04:18:58 -05001021 logging.debug('Linking path %s -> %s', src, dst)
Mike Frysinger35247af2012-11-16 18:58:06 -05001022 os.link(src, dst)
1023
Mike Frysinger35247af2012-11-16 18:58:06 -05001024 # Locate all the dependencies for all the ELFs. Stick them all in the
1025 # top level "lib" dir to make the wrapper simpler. This exact path does
1026 # not matter since we execute ldso directly, and we tell the ldso the
1027 # exact path to search for its libraries.
1028 libdir = os.path.join(output_dir, 'lib')
1029 osutils.SafeMakedirs(libdir)
1030 donelibs = set()
Mike Frysinger9fe02342019-12-12 17:52:53 -05001031 basenamelibs = set()
Mike Frysinger4331fc62017-09-28 14:03:25 -04001032 glibc_re = re.compile(r'/lib(c|pthread)-[0-9.]+\.so$')
Mike Frysinger35247af2012-11-16 18:58:06 -05001033 for elf in elfs:
Mike Frysingerea7688e2014-07-31 22:40:33 -04001034 e = lddtree.ParseELF(elf, root=root, ldpaths=ldpaths)
Mike Frysinger60a2f6c2019-12-04 04:18:58 -05001035 logging.debug('Parsed elf %s data: %s', elf, e)
Mike Frysinger35247af2012-11-16 18:58:06 -05001036 interp = e['interp']
Mike Frysinger221bd822017-09-29 02:51:47 -04001037
Mike Frysinger9fe02342019-12-12 17:52:53 -05001038 # TODO(crbug.com/917193): Drop this hack once libopcodes linkage is fixed.
1039 if os.path.basename(elf).startswith('libopcodes-'):
1040 continue
Mike Frysinger35247af2012-11-16 18:58:06 -05001041
Mike Frysinger00b129f2021-04-21 18:11:48 -04001042 # Copy all the dependencies before we copy the program & generate wrappers.
Mike Frysinger9fe02342019-12-12 17:52:53 -05001043 for lib, lib_data in e['libs'].items():
Mike Frysinger35247af2012-11-16 18:58:06 -05001044 src = path = lib_data['path']
1045 if path is None:
Ralph Nathan446aee92015-03-23 14:44:56 -07001046 logging.warning('%s: could not locate %s', elf, lib)
Mike Frysinger35247af2012-11-16 18:58:06 -05001047 continue
Mike Frysinger9fe02342019-12-12 17:52:53 -05001048
1049 # No need to try and copy the same source lib multiple times.
1050 if path in donelibs:
1051 continue
1052 donelibs.add(path)
1053
1054 # Die if we try to normalize different source libs with the same basename.
1055 if lib in basenamelibs:
1056 logging.error('Multiple sources detected for %s:\n new: %s\n old: %s',
1057 os.path.join('/lib', lib), path,
1058 ' '.join(x for x in donelibs
1059 if x != path and os.path.basename(x) == lib))
1060 # TODO(crbug.com/917193): Make this fatal.
1061 # cros_build_lib.Die('Unable to resolve lib conflicts')
1062 continue
1063 basenamelibs.add(lib)
Mike Frysinger35247af2012-11-16 18:58:06 -05001064
1065 # Needed libs are the SONAME, but that is usually a symlink, not a
1066 # real file. So link in the target rather than the symlink itself.
1067 # We have to walk all the possible symlinks (SONAME could point to a
1068 # symlink which points to a symlink), and we have to handle absolute
1069 # ourselves (since we have a "root" argument).
1070 dst = os.path.join(libdir, os.path.basename(path))
1071 src = ReadlinkRoot(src, root)
1072
Mike Frysinger60a2f6c2019-12-04 04:18:58 -05001073 logging.debug('Linking lib %s -> %s', root + src, dst)
Mike Frysinger35247af2012-11-16 18:58:06 -05001074 os.link(root + src, dst)
1075
Mike Frysinger00b129f2021-04-21 18:11:48 -04001076 # Do not create wrapper for libc. crbug.com/766827
1077 if interp and not glibc_re.search(elf):
1078 # Generate a wrapper if it is executable.
1079 interp = os.path.join('/lib', os.path.basename(interp))
1080 lddtree.GenerateLdsoWrapper(output_dir, path_rewrite_func(elf), interp,
1081 libpaths=e['rpath'] + e['runpath'])
1082 FixClangXXWrapper(output_dir, path_rewrite_func(elf))
1083
1084 # Wrap any symlinks to the wrapper.
1085 if elf in sym_paths:
1086 link = sym_paths[elf]
1087 GeneratePathWrapper(output_dir, link, elf)
1088
Mike Frysinger35247af2012-11-16 18:58:06 -05001089
1090def _EnvdGetVar(envd, var):
1091 """Given a Gentoo env.d file, extract a var from it
1092
1093 Args:
1094 envd: The env.d file to load (may be a glob path)
1095 var: The var to extract
Mike Frysinger1a736a82013-12-12 01:50:59 -05001096
Mike Frysinger35247af2012-11-16 18:58:06 -05001097 Returns:
1098 The value of |var|
1099 """
1100 envds = glob.glob(envd)
1101 assert len(envds) == 1, '%s: should have exactly 1 env.d file' % envd
1102 envd = envds[0]
Mike Frysingere652ba12019-09-08 00:57:43 -04001103 return key_value_store.LoadFile(envd)[var]
Mike Frysinger35247af2012-11-16 18:58:06 -05001104
1105
1106def _ProcessBinutilsConfig(target, output_dir):
1107 """Do what binutils-config would have done"""
1108 binpath = os.path.join('/bin', target + '-')
Mike Frysingerd4d40fd2014-11-06 17:30:57 -05001109
Adrian Ratiu78e534d2021-01-06 19:58:38 +02001110 # Locate the bin dir holding the linker and perform some sanity checks
Mike Frysingerd4d40fd2014-11-06 17:30:57 -05001111 binutils_bin_path = os.path.join(output_dir, 'usr', toolchain.GetHostTuple(),
1112 target, 'binutils-bin')
Adrian Ratiu78e534d2021-01-06 19:58:38 +02001113 globpath = os.path.join(binutils_bin_path, '*')
Mike Frysinger35247af2012-11-16 18:58:06 -05001114 srcpath = glob.glob(globpath)
Adrian Ratiu78e534d2021-01-06 19:58:38 +02001115 assert len(srcpath) == 1, ('%s: matched more than one path. Is Gold enabled?'
1116 % globpath)
1117 srcpath = srcpath[0]
1118 ld_path = os.path.join(srcpath, 'ld')
1119 assert os.path.exists(ld_path), '%s: linker is missing!' % ld_path
1120 ld_path = os.path.join(srcpath, 'ld.bfd')
1121 assert os.path.exists(ld_path), '%s: linker is missing!' % ld_path
Rahul Chaudhry4891b4d2017-03-08 10:31:27 -08001122
Mike Frysinger78b7a812014-11-26 19:45:23 -05001123 srcpath = srcpath[len(output_dir):]
Mike Frysinger35247af2012-11-16 18:58:06 -05001124 gccpath = os.path.join('/usr', 'libexec', 'gcc')
1125 for prog in os.listdir(output_dir + srcpath):
1126 # Skip binaries already wrapped.
1127 if not prog.endswith('.real'):
1128 GeneratePathWrapper(output_dir, binpath + prog,
1129 os.path.join(srcpath, prog))
1130 GeneratePathWrapper(output_dir, os.path.join(gccpath, prog),
1131 os.path.join(srcpath, prog))
1132
David James27ac4ae2012-12-03 23:16:15 -08001133 libpath = os.path.join('/usr', toolchain.GetHostTuple(), target, 'lib')
Mike Frysingerd4d40fd2014-11-06 17:30:57 -05001134 envd = os.path.join(output_dir, 'etc', 'env.d', 'binutils', '*')
Mike Frysinger35247af2012-11-16 18:58:06 -05001135 srcpath = _EnvdGetVar(envd, 'LIBPATH')
1136 os.symlink(os.path.relpath(srcpath, os.path.dirname(libpath)),
1137 output_dir + libpath)
1138
1139
1140def _ProcessGccConfig(target, output_dir):
1141 """Do what gcc-config would have done"""
1142 binpath = '/bin'
1143 envd = os.path.join(output_dir, 'etc', 'env.d', 'gcc', '*')
1144 srcpath = _EnvdGetVar(envd, 'GCC_PATH')
1145 for prog in os.listdir(output_dir + srcpath):
1146 # Skip binaries already wrapped.
1147 if (not prog.endswith('.real') and
1148 not prog.endswith('.elf') and
1149 prog.startswith(target)):
1150 GeneratePathWrapper(output_dir, os.path.join(binpath, prog),
1151 os.path.join(srcpath, prog))
1152 return srcpath
1153
1154
Frank Henigman179ec7c2015-02-06 03:01:09 -05001155def _ProcessSysrootWrappers(_target, output_dir, srcpath):
1156 """Remove chroot-specific things from our sysroot wrappers"""
Mike Frysinger35247af2012-11-16 18:58:06 -05001157 # Disable ccache since we know it won't work outside of chroot.
Tobias Boschddd16492019-08-14 09:29:54 -07001158
Tobias Boschddd16492019-08-14 09:29:54 -07001159 # Use the version of the wrapper that does not use ccache.
Frank Henigman179ec7c2015-02-06 03:01:09 -05001160 for sysroot_wrapper in glob.glob(os.path.join(
Tobias Boschddd16492019-08-14 09:29:54 -07001161 output_dir + srcpath, 'sysroot_wrapper*.ccache')):
1162 # Can't update the wrapper in place to not affect the chroot,
1163 # but only the extracted toolchain.
1164 os.unlink(sysroot_wrapper)
1165 shutil.copy(sysroot_wrapper[:-6] + 'noccache', sysroot_wrapper)
Tobias Bosch7bc907a2019-10-09 11:52:40 -07001166 shutil.copy(sysroot_wrapper[:-6] + 'noccache.elf', sysroot_wrapper + '.elf')
Mike Frysinger35247af2012-11-16 18:58:06 -05001167
1168
Yunlian Jiang5ad6b512017-09-20 09:27:45 -07001169def _CreateMainLibDir(target, output_dir):
1170 """Create some lib dirs so that compiler can get the right Gcc paths"""
1171 osutils.SafeMakedirs(os.path.join(output_dir, 'usr', target, 'lib'))
1172 osutils.SafeMakedirs(os.path.join(output_dir, 'usr', target, 'usr/lib'))
1173
1174
Mike Frysinger35247af2012-11-16 18:58:06 -05001175def _ProcessDistroCleanups(target, output_dir):
1176 """Clean up the tree and remove all distro-specific requirements
1177
1178 Args:
1179 target: The toolchain target name
1180 output_dir: The output directory to clean up
1181 """
1182 _ProcessBinutilsConfig(target, output_dir)
1183 gcc_path = _ProcessGccConfig(target, output_dir)
Frank Henigman179ec7c2015-02-06 03:01:09 -05001184 _ProcessSysrootWrappers(target, output_dir, gcc_path)
Yunlian Jiang5ad6b512017-09-20 09:27:45 -07001185 _CreateMainLibDir(target, output_dir)
Mike Frysinger35247af2012-11-16 18:58:06 -05001186
1187 osutils.RmDir(os.path.join(output_dir, 'etc'))
1188
1189
1190def CreatePackagableRoot(target, output_dir, ldpaths, root='/'):
1191 """Setup a tree from the packages for the specified target
1192
1193 This populates a path with all the files from toolchain packages so that
1194 a tarball can easily be generated from the result.
1195
1196 Args:
1197 target: The target to create a packagable root from
1198 output_dir: The output directory to place all the files
1199 ldpaths: A dict of static ldpath information
1200 root: The root path to pull all packages/files from
1201 """
1202 # Find all the files owned by the packages for this target.
1203 paths, elfs = _GetFilesForTarget(target, root=root)
1204
1205 # Link in all the package's files, any ELF dependencies, and wrap any
1206 # executable ELFs with helper scripts.
1207 def MoveUsrBinToBin(path):
Han Shen699ea192016-03-02 10:42:47 -08001208 """Move /usr/bin to /bin so people can just use that toplevel dir
1209
George Burgess IVca1d7612020-10-01 00:38:32 -07001210 Note we do not apply this to clang or rust - there is correlation between
1211 clang's search path for libraries / inclusion and its installation path.
Han Shen699ea192016-03-02 10:42:47 -08001212 """
George Burgess IVca1d7612020-10-01 00:38:32 -07001213 if (path.startswith('/usr/bin/') and
1214 not any(x in path for x in ('clang', 'rust', 'cargo'))):
Han Shen699ea192016-03-02 10:42:47 -08001215 return path[4:]
1216 return path
Mike Frysinger35247af2012-11-16 18:58:06 -05001217 _BuildInitialPackageRoot(output_dir, paths, elfs, ldpaths,
1218 path_rewrite_func=MoveUsrBinToBin, root=root)
1219
1220 # The packages, when part of the normal distro, have helper scripts
1221 # that setup paths and such. Since we are making this standalone, we
1222 # need to preprocess all that ourselves.
1223 _ProcessDistroCleanups(target, output_dir)
1224
1225
1226def CreatePackages(targets_wanted, output_dir, root='/'):
1227 """Create redistributable cross-compiler packages for the specified targets
1228
1229 This creates toolchain packages that should be usable in conjunction with
1230 a downloaded sysroot (created elsewhere).
1231
1232 Tarballs (one per target) will be created in $PWD.
1233
1234 Args:
Don Garrett25f309a2014-03-19 14:02:12 -07001235 targets_wanted: The targets to package up.
1236 output_dir: The directory to put the packages in.
1237 root: The root path to pull all packages/files from.
Mike Frysinger35247af2012-11-16 18:58:06 -05001238 """
Ralph Nathan03047282015-03-23 11:09:32 -07001239 logging.info('Writing tarballs to %s', output_dir)
Mike Frysinger35247af2012-11-16 18:58:06 -05001240 osutils.SafeMakedirs(output_dir)
1241 ldpaths = lddtree.LoadLdpaths(root)
1242 targets = ExpandTargets(targets_wanted)
1243
Mike Frysinger221bd822017-09-29 02:51:47 -04001244 with osutils.TempDir(prefix='create-packages') as tempdir:
1245 logging.debug('Using tempdir: %s', tempdir)
1246
Mike Frysinger35247af2012-11-16 18:58:06 -05001247 # We have to split the root generation from the compression stages. This is
1248 # because we hardlink in all the files (to avoid overhead of reading/writing
1249 # the copies multiple times). But tar gets angry if a file's hardlink count
1250 # changes from when it starts reading a file to when it finishes.
1251 with parallel.BackgroundTaskRunner(CreatePackagableRoot) as queue:
1252 for target in targets:
1253 output_target_dir = os.path.join(tempdir, target)
1254 queue.put([target, output_target_dir, ldpaths, root])
1255
1256 # Build the tarball.
1257 with parallel.BackgroundTaskRunner(cros_build_lib.CreateTarball) as queue:
1258 for target in targets:
1259 tar_file = os.path.join(output_dir, target + '.tar.xz')
1260 queue.put([tar_file, os.path.join(tempdir, target)])
1261
1262
Mike Frysinger07534cf2017-09-12 17:40:21 -04001263def GetParser():
1264 """Return a command line parser."""
Mike Frysinger0c808452014-11-06 17:30:23 -05001265 parser = commandline.ArgumentParser(description=__doc__)
1266 parser.add_argument('-u', '--nousepkg',
1267 action='store_false', dest='usepkg', default=True,
1268 help='Use prebuilt packages if possible')
1269 parser.add_argument('-d', '--deleteold',
1270 action='store_true', dest='deleteold', default=False,
1271 help='Unmerge deprecated packages')
1272 parser.add_argument('-t', '--targets',
1273 dest='targets', default='sdk',
Mike Frysinger80de5012019-08-01 14:10:53 -04001274 help='Comma separated list of tuples. Special keywords '
Don Garrettc0c74002015-10-09 12:58:19 -07001275 "'host', 'sdk', 'boards', and 'all' are "
Gilad Arnold8195b532015-04-07 10:56:30 +03001276 "allowed. Defaults to 'sdk'.")
1277 parser.add_argument('--include-boards', default='', metavar='BOARDS',
1278 help='Comma separated list of boards whose toolchains we '
1279 'will always include. Default: none')
Mike Frysinger0c808452014-11-06 17:30:23 -05001280 parser.add_argument('--hostonly',
1281 dest='hostonly', default=False, action='store_true',
1282 help='Only setup the host toolchain. '
1283 'Useful for bootstrapping chroot')
Bertrand SIMONNETcae9d5f2015-03-09 15:58:01 -07001284 parser.add_argument('--show-board-cfg', '--show-cfg',
1285 dest='cfg_name', default=None,
Don Garrettc0c74002015-10-09 12:58:19 -07001286 help='Board to list toolchains tuples for')
Mike Frysinger91f52882017-09-13 00:16:58 -04001287 parser.add_argument('--show-packages', default=None,
1288 help='List all packages the specified target uses')
Mike Frysinger0c808452014-11-06 17:30:23 -05001289 parser.add_argument('--create-packages',
1290 action='store_true', default=False,
1291 help='Build redistributable packages')
1292 parser.add_argument('--output-dir', default=os.getcwd(), type='path',
1293 help='Output directory')
1294 parser.add_argument('--reconfig', default=False, action='store_true',
1295 help='Reload crossdev config and reselect toolchains')
Gilad Arnold2dab78c2015-05-21 14:43:33 -07001296 parser.add_argument('--sysroot', type='path',
1297 help='The sysroot in which to install the toolchains')
Mike Frysinger07534cf2017-09-12 17:40:21 -04001298 return parser
Zdenek Behan508dcce2011-12-05 15:39:32 +01001299
Mike Frysinger07534cf2017-09-12 17:40:21 -04001300
1301def main(argv):
1302 parser = GetParser()
Mike Frysinger0c808452014-11-06 17:30:23 -05001303 options = parser.parse_args(argv)
1304 options.Freeze()
Zdenek Behan508dcce2011-12-05 15:39:32 +01001305
Mike Frysinger35247af2012-11-16 18:58:06 -05001306 # Figure out what we're supposed to do and reject conflicting options.
Mike Frysinger91f52882017-09-13 00:16:58 -04001307 conflicting_options = (
1308 options.cfg_name,
1309 options.show_packages,
1310 options.create_packages,
1311 )
1312 if sum(bool(x) for x in conflicting_options) > 1:
1313 parser.error('conflicting options: create-packages & show-packages & '
1314 'show-board-cfg')
Mike Frysinger984d0622012-06-01 16:08:44 -04001315
Gilad Arnold8195b532015-04-07 10:56:30 +03001316 targets_wanted = set(options.targets.split(','))
1317 boards_wanted = (set(options.include_boards.split(','))
1318 if options.include_boards else set())
Mike Frysinger35247af2012-11-16 18:58:06 -05001319
Bertrand SIMONNETcae9d5f2015-03-09 15:58:01 -07001320 if options.cfg_name:
1321 ShowConfig(options.cfg_name)
Mike Frysinger91f52882017-09-13 00:16:58 -04001322 elif options.show_packages is not None:
1323 cros_build_lib.AssertInsideChroot()
1324 target = options.show_packages
1325 Crossdev.Load(False)
1326 for package in GetTargetPackages(target):
1327 print(GetPortagePackage(target, package))
Mike Frysinger35247af2012-11-16 18:58:06 -05001328 elif options.create_packages:
1329 cros_build_lib.AssertInsideChroot()
1330 Crossdev.Load(False)
Gilad Arnold8195b532015-04-07 10:56:30 +03001331 CreatePackages(targets_wanted, options.output_dir)
Mike Frysinger35247af2012-11-16 18:58:06 -05001332 else:
1333 cros_build_lib.AssertInsideChroot()
1334 # This has to be always run as root.
1335 if os.geteuid() != 0:
1336 cros_build_lib.Die('this script must be run as root')
1337
1338 Crossdev.Load(options.reconfig)
Gilad Arnold2dab78c2015-05-21 14:43:33 -07001339 root = options.sysroot or '/'
Mike Frysinger35247af2012-11-16 18:58:06 -05001340 UpdateToolchains(options.usepkg, options.deleteold, options.hostonly,
Gilad Arnold8195b532015-04-07 10:56:30 +03001341 options.reconfig, targets_wanted, boards_wanted,
Don Garrettc0c74002015-10-09 12:58:19 -07001342 root=root)
Mike Frysinger35247af2012-11-16 18:58:06 -05001343 Crossdev.Save()
1344
1345 return 0