blob: 207f2270626bc83285a3916258c69401e33299b0 [file] [log] [blame]
Mike Frysingere58c0e22017-10-04 15:43:30 -04001# -*- coding: utf-8 -*-
Zdenek Behan508dcce2011-12-05 15:39:32 +01002# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
Mike Frysinger750c5f52014-09-16 16:16:57 -04006"""This script manages the installed toolchains in the chroot."""
Zdenek Behan508dcce2011-12-05 15:39:32 +01007
Mike Frysinger383367e2014-09-16 15:06:17 -04008from __future__ import print_function
9
Mike Frysinger3ed47722017-08-08 14:59:08 -040010import errno
Mike Frysinger35247af2012-11-16 18:58:06 -050011import glob
Mike Frysinger3ed47722017-08-08 14:59:08 -040012import hashlib
Mike Frysinger7ccee992012-06-01 21:27:59 -040013import json
Zdenek Behan508dcce2011-12-05 15:39:32 +010014import os
Rahul Chaudhrya8127bb2016-07-22 15:53:17 -070015import re
Tobias Boschddd16492019-08-14 09:29:54 -070016import shutil
Mike Frysinger499ca392020-04-28 07:35:54 -040017import sys
Zdenek Behan508dcce2011-12-05 15:39:32 +010018
Aviv Keshetb7519e12016-10-04 00:50:00 -070019from chromite.lib import constants
Mike Frysinger506e75f2012-12-17 14:21:13 -050020from chromite.lib import commandline
Brian Harring503f3ab2012-03-09 21:39:41 -080021from chromite.lib import cros_build_lib
Ralph Nathan03047282015-03-23 11:09:32 -070022from chromite.lib import cros_logging as logging
Brian Harringaf019fb2012-05-10 15:06:13 -070023from chromite.lib import osutils
Mike Frysinger35247af2012-11-16 18:58:06 -050024from chromite.lib import parallel
David James27ac4ae2012-12-03 23:16:15 -080025from chromite.lib import toolchain
Mike Frysingere652ba12019-09-08 00:57:43 -040026from chromite.utils import key_value_store
Mike Frysinger35247af2012-11-16 18:58:06 -050027
28# Needs to be after chromite imports.
Mike Frysingerabb7d812020-05-15 00:13:10 -040029import lddtree # pylint: disable=wrong-import-order
Zdenek Behan508dcce2011-12-05 15:39:32 +010030
Mike Frysinger31596002012-12-03 23:54:24 -050031if cros_build_lib.IsInsideChroot():
32 # Only import portage after we've checked that we're inside the chroot.
33 # Outside may not have portage, in which case the above may not happen.
34 # We'll check in main() if the operation needs portage.
Mike Frysinger27e21b72018-07-12 14:20:21 -040035 # pylint: disable=import-error
Mike Frysinger31596002012-12-03 23:54:24 -050036 import portage
Zdenek Behan508dcce2011-12-05 15:39:32 +010037
38
Mike Frysinger499ca392020-04-28 07:35:54 -040039assert sys.version_info >= (3, 6), 'This module requires Python 3.6+'
40
41
Matt Tennantf1e30972012-03-02 16:30:07 -080042EMERGE_CMD = os.path.join(constants.CHROMITE_BIN_DIR, 'parallel_emerge')
Zdenek Behan508dcce2011-12-05 15:39:32 +010043PACKAGE_STABLE = '[stable]'
Zdenek Behan4eb6fd22012-03-12 17:00:56 +010044
45CHROMIUMOS_OVERLAY = '/usr/local/portage/chromiumos'
Christopher Wileyb22c0712015-06-02 10:37:03 -070046ECLASS_OVERLAY = '/usr/local/portage/eclass-overlay'
Zdenek Behan4eb6fd22012-03-12 17:00:56 +010047STABLE_OVERLAY = '/usr/local/portage/stable'
48CROSSDEV_OVERLAY = '/usr/local/portage/crossdev'
Zdenek Behan508dcce2011-12-05 15:39:32 +010049
50
Mike Frysinger66bfde52017-09-12 16:42:57 -040051# The exact list of host toolchain packages we care about. These are the
52# packages that bots/devs install only from binpkgs and rely on the SDK bot
53# (chromiumos-sdk) to validate+uprev.
54#
Mike Frysinger66bfde52017-09-12 16:42:57 -040055# We don't use crossdev to manage the host toolchain for us, especially since
56# we diverge significantly now (with llvm/clang/etc...), and we don't need or
57# want crossdev managing /etc/portage config files for the sdk
58HOST_PACKAGES = (
59 'dev-lang/go',
Yunlian Jiang2cb91dc2018-03-08 10:56:27 -080060 'dev-libs/elfutils',
Mike Frysinger66bfde52017-09-12 16:42:57 -040061 'sys-devel/binutils',
Mike Frysinger66bfde52017-09-12 16:42:57 -040062 'sys-devel/gcc',
63 'sys-devel/llvm',
64 'sys-kernel/linux-headers',
65 'sys-libs/glibc',
66 'sys-libs/libcxx',
67 'sys-libs/libcxxabi',
Manoj Guptade64cb22019-03-31 18:48:58 -070068 'sys-libs/llvm-libunwind',
Mike Frysinger66bfde52017-09-12 16:42:57 -040069)
70
Mike Frysinger785b0c32017-09-13 01:35:59 -040071# These packages are also installed into the host SDK. However, they require
72# the cross-compilers to be installed first (because they need them to actually
73# build), so we have to delay their installation.
74HOST_POST_CROSS_PACKAGES = (
Manoj Gupta65f88442018-04-12 22:42:19 -070075 'dev-lang/rust',
Bob Haarmane75b16b2020-09-29 20:48:53 +000076 'dev-lang/rust-bootstrap',
Mike Frysinger61a24392017-10-17 17:14:27 -040077 'virtual/target-sdk-post-cross',
Patrick Georgi043ce6e2019-02-20 22:27:09 +010078 'dev-embedded/coreboot-sdk',
Mike Frysinger785b0c32017-09-13 01:35:59 -040079)
80
81# New packages that we're in the process of adding to the SDK. Since the SDK
82# bot hasn't had a chance to run yet, there are no binary packages available,
83# so we have to list them here and wait. Once it completes, entries here can
84# be removed so they'll end up on bots & dev's systems.
85NEW_PACKAGES = (
Bob Haarmane75b16b2020-09-29 20:48:53 +000086 'dev-lang/rust-bootstrap',
Mike Frysinger785b0c32017-09-13 01:35:59 -040087)
88
Rahul Chaudhry4b803052015-05-13 15:25:56 -070089# Enable the Go compiler for these targets.
90TARGET_GO_ENABLED = (
91 'x86_64-cros-linux-gnu',
Rahul Chaudhry4b803052015-05-13 15:25:56 -070092 'armv7a-cros-linux-gnueabi',
Yunlian Jiang16c1a422017-10-04 10:33:22 -070093 'armv7a-cros-linux-gnueabihf',
Rahul Chaudhry4d416582017-10-25 12:31:58 -070094 'aarch64-cros-linux-gnu',
Rahul Chaudhry4b803052015-05-13 15:25:56 -070095)
96CROSSDEV_GO_ARGS = ['--ex-pkg', 'dev-lang/go']
97
Manoj Gupta1b5642e2017-03-08 16:44:12 -080098# Enable llvm's compiler-rt for these targets.
99TARGET_COMPILER_RT_ENABLED = (
100 'armv7a-cros-linux-gnueabi',
Yunlian Jiang1b77ee42017-10-06 13:44:29 -0700101 'armv7a-cros-linux-gnueabihf',
Manoj Gupta946abb42017-04-12 14:27:19 -0700102 'aarch64-cros-linux-gnu',
Manoj Gupta21f3a082018-03-06 21:25:39 -0800103 'armv7m-cros-eabi',
Manoj Gupta1b5642e2017-03-08 16:44:12 -0800104)
105CROSSDEV_COMPILER_RT_ARGS = ['--ex-pkg', 'sys-libs/compiler-rt']
106
Manoj Gupta946abb42017-04-12 14:27:19 -0700107TARGET_LLVM_PKGS_ENABLED = (
108 'armv7a-cros-linux-gnueabi',
Yunlian Jiang16c1a422017-10-04 10:33:22 -0700109 'armv7a-cros-linux-gnueabihf',
Manoj Gupta946abb42017-04-12 14:27:19 -0700110 'aarch64-cros-linux-gnu',
111 'x86_64-cros-linux-gnu',
112)
113
114LLVM_PKGS_TABLE = {
Yunlian Jiangbb2a3d32017-04-26 14:44:09 -0700115 'ex_libcxxabi' : ['--ex-pkg', 'sys-libs/libcxxabi'],
116 'ex_libcxx' : ['--ex-pkg', 'sys-libs/libcxx'],
Chirantan Ekbotee694cad2018-07-12 15:07:24 -0700117 'ex_llvm-libunwind' : ['--ex-pkg', 'sys-libs/llvm-libunwind'],
Manoj Gupta946abb42017-04-12 14:27:19 -0700118}
119
Zdenek Behan508dcce2011-12-05 15:39:32 +0100120# Overrides for {gcc,binutils}-config, pick a package with particular suffix.
121CONFIG_TARGET_SUFFIXES = {
Mike Frysingerd6e2df02014-11-26 02:55:04 -0500122 'binutils' : {
Manoj Guptaa91e38e2018-11-15 11:07:48 -0800123 'aarch64-cros-linux-gnu' : '-gold',
Mike Frysinger8a83c622015-05-28 00:35:05 -0400124 'armv6j-cros-linux-gnueabi': '-gold',
Han Shen43b84422015-02-19 11:38:13 -0800125 'armv7a-cros-linux-gnueabi': '-gold',
Yunlian Jiang16c1a422017-10-04 10:33:22 -0700126 'armv7a-cros-linux-gnueabihf': '-gold',
Mike Frysingerd6e2df02014-11-26 02:55:04 -0500127 'i686-pc-linux-gnu' : '-gold',
128 'x86_64-cros-linux-gnu' : '-gold',
129 },
Zdenek Behan508dcce2011-12-05 15:39:32 +0100130}
Zdenek Behan508dcce2011-12-05 15:39:32 +0100131
132
David James66a09c42012-11-05 13:31:38 -0800133class Crossdev(object):
134 """Class for interacting with crossdev and caching its output."""
135
136 _CACHE_FILE = os.path.join(CROSSDEV_OVERLAY, '.configured.json')
137 _CACHE = {}
Han Shene23782f2016-02-18 12:20:00 -0800138 # Packages that needs separate handling, in addition to what we have from
139 # crossdev.
140 MANUAL_PKGS = {
George Burgess IVca1d7612020-10-01 00:38:32 -0700141 'rust': 'dev-lang',
Han Shene23782f2016-02-18 12:20:00 -0800142 'llvm': 'sys-devel',
Manoj Gupta20cfc6c2017-07-19 15:49:55 -0700143 'libcxxabi': 'sys-libs',
144 'libcxx': 'sys-libs',
Yunlian Jiangda3ce5f2018-04-25 14:10:01 -0700145 'elfutils': 'dev-libs',
Chirantan Ekbotee694cad2018-07-12 15:07:24 -0700146 'llvm-libunwind': 'sys-libs',
Han Shene23782f2016-02-18 12:20:00 -0800147 }
David James66a09c42012-11-05 13:31:38 -0800148
149 @classmethod
150 def Load(cls, reconfig):
Mike Frysinger3ed47722017-08-08 14:59:08 -0400151 """Load crossdev cache from disk.
152
153 We invalidate the cache when crossdev updates or this script changes.
154 """
David James90239b92012-11-05 15:31:34 -0800155 crossdev_version = GetStablePackageVersion('sys-devel/crossdev', True)
Mike Frysinger3ed47722017-08-08 14:59:08 -0400156 # If we run the compiled/cached .pyc file, we'll read/hash that when we
157 # really always want to track the source .py file.
158 script = os.path.abspath(__file__)
159 if script.endswith('.pyc'):
160 script = script[:-1]
Mike Frysingerb3202be2019-11-15 22:25:59 -0500161 setup_toolchains_hash = hashlib.md5(
162 osutils.ReadFile(script, mode='rb')).hexdigest()
Mike Frysinger3ed47722017-08-08 14:59:08 -0400163
164 cls._CACHE = {
165 'crossdev_version': crossdev_version,
166 'setup_toolchains_hash': setup_toolchains_hash,
167 }
168
169 logging.debug('cache: checking file: %s', cls._CACHE_FILE)
170 if reconfig:
171 logging.debug('cache: forcing regen due to reconfig')
172 return
173
174 try:
175 file_data = osutils.ReadFile(cls._CACHE_FILE)
176 except IOError as e:
177 if e.errno != errno.ENOENT:
178 logging.warning('cache: reading failed: %s', e)
179 osutils.SafeUnlink(cls._CACHE_FILE)
180 return
181
182 try:
183 data = json.loads(file_data)
184 except ValueError as e:
185 logging.warning('cache: ignoring invalid content: %s', e)
186 return
187
188 if crossdev_version != data.get('crossdev_version'):
189 logging.debug('cache: rebuilding after crossdev upgrade')
190 elif setup_toolchains_hash != data.get('setup_toolchains_hash'):
191 logging.debug('cache: rebuilding after cros_setup_toolchains change')
192 else:
193 logging.debug('cache: content is up-to-date!')
194 cls._CACHE = data
David James66a09c42012-11-05 13:31:38 -0800195
196 @classmethod
197 def Save(cls):
198 """Store crossdev cache on disk."""
199 # Save the cache from the successful run.
200 with open(cls._CACHE_FILE, 'w') as f:
201 json.dump(cls._CACHE, f)
202
203 @classmethod
204 def GetConfig(cls, target):
205 """Returns a map of crossdev provided variables about a tuple."""
206 CACHE_ATTR = '_target_tuple_map'
207
208 val = cls._CACHE.setdefault(CACHE_ATTR, {})
209 if not target in val:
Mike Frysinger785b0c32017-09-13 01:35:59 -0400210 if target.startswith('host'):
Mike Frysinger66bfde52017-09-12 16:42:57 -0400211 conf = {
212 'crosspkgs': [],
213 'target': toolchain.GetHostTuple(),
214 }
Mike Frysinger785b0c32017-09-13 01:35:59 -0400215 if target == 'host':
216 packages_list = HOST_PACKAGES
217 else:
218 packages_list = HOST_POST_CROSS_PACKAGES
Mike Frysinger66bfde52017-09-12 16:42:57 -0400219 manual_pkgs = dict((pkg, cat) for cat, pkg in
Mike Frysinger785b0c32017-09-13 01:35:59 -0400220 [x.split('/') for x in packages_list])
Mike Frysinger66bfde52017-09-12 16:42:57 -0400221 else:
222 # Build the crossdev command.
223 cmd = ['crossdev', '--show-target-cfg', '--ex-gdb']
224 if target in TARGET_COMPILER_RT_ENABLED:
225 cmd.extend(CROSSDEV_COMPILER_RT_ARGS)
226 if target in TARGET_GO_ENABLED:
227 cmd.extend(CROSSDEV_GO_ARGS)
228 if target in TARGET_LLVM_PKGS_ENABLED:
229 for pkg in LLVM_PKGS_TABLE:
230 cmd.extend(LLVM_PKGS_TABLE[pkg])
231 cmd.extend(['-t', target])
232 # Catch output of crossdev.
Mike Frysinger45602c72019-09-22 02:15:11 -0400233 out = cros_build_lib.run(
Mike Frysinger0282d222019-12-17 17:15:48 -0500234 cmd, print_cmd=False, stdout=True,
Mike Frysingerb3202be2019-11-15 22:25:59 -0500235 encoding='utf-8').stdout.splitlines()
Mike Frysinger66bfde52017-09-12 16:42:57 -0400236 # List of tuples split at the first '=', converted into dict.
237 conf = dict((k, cros_build_lib.ShellUnquote(v))
238 for k, v in (x.split('=', 1) for x in out))
239 conf['crosspkgs'] = conf['crosspkgs'].split()
Han Shene23782f2016-02-18 12:20:00 -0800240
Mike Frysinger66bfde52017-09-12 16:42:57 -0400241 manual_pkgs = cls.MANUAL_PKGS
242
243 for pkg, cat in manual_pkgs.items():
Mike Frysinger3bba5032016-09-20 14:15:04 -0400244 conf[pkg + '_pn'] = pkg
245 conf[pkg + '_category'] = cat
246 if pkg not in conf['crosspkgs']:
247 conf['crosspkgs'].append(pkg)
Han Shene23782f2016-02-18 12:20:00 -0800248
249 val[target] = conf
250
David James66a09c42012-11-05 13:31:38 -0800251 return val[target]
252
253 @classmethod
254 def UpdateTargets(cls, targets, usepkg, config_only=False):
255 """Calls crossdev to initialize a cross target.
256
257 Args:
Nicolas Boichat3fecf392019-11-25 02:58:28 +0000258 targets: The list of targets to initialize using crossdev.
Don Garrett25f309a2014-03-19 14:02:12 -0700259 usepkg: Copies the commandline opts.
260 config_only: Just update.
David James66a09c42012-11-05 13:31:38 -0800261 """
262 configured_targets = cls._CACHE.setdefault('configured_targets', [])
263
264 cmdbase = ['crossdev', '--show-fail-log']
265 cmdbase.extend(['--env', 'FEATURES=splitdebug'])
266 # Pick stable by default, and override as necessary.
267 cmdbase.extend(['-P', '--oneshot'])
268 if usepkg:
269 cmdbase.extend(['-P', '--getbinpkg',
270 '-P', '--usepkgonly',
271 '--without-headers'])
272
Christopher Wileyb22c0712015-06-02 10:37:03 -0700273 overlays = ' '.join((CHROMIUMOS_OVERLAY, ECLASS_OVERLAY, STABLE_OVERLAY))
David James66a09c42012-11-05 13:31:38 -0800274 cmdbase.extend(['--overlays', overlays])
275 cmdbase.extend(['--ov-output', CROSSDEV_OVERLAY])
276
Nicolas Boichat3fecf392019-11-25 02:58:28 +0000277 # Build target by the reversed alphabetical order to make sure
278 # armv7a-cros-linux-gnueabihf builds before armv7a-cros-linux-gnueabi
279 # because some dependency issue. This can be reverted once we
280 # migrated to armv7a-cros-linux-gnueabihf. crbug.com/711369
281 for target in sorted(targets, reverse=True):
282 if config_only and target in configured_targets:
283 continue
David James66a09c42012-11-05 13:31:38 -0800284
Nicolas Boichat3fecf392019-11-25 02:58:28 +0000285 cmd = cmdbase + ['-t', target]
286
287 for pkg in GetTargetPackages(target):
288 if pkg == 'gdb':
289 # Gdb does not have selectable versions.
290 cmd.append('--ex-gdb')
291 elif pkg == 'ex_compiler-rt':
292 cmd.extend(CROSSDEV_COMPILER_RT_ARGS)
293 elif pkg == 'ex_go':
294 # Go does not have selectable versions.
295 cmd.extend(CROSSDEV_GO_ARGS)
296 elif pkg in LLVM_PKGS_TABLE:
297 cmd.extend(LLVM_PKGS_TABLE[pkg])
298 elif pkg in cls.MANUAL_PKGS:
299 pass
300 else:
301 # The first of the desired versions is the "primary" one.
302 version = GetDesiredPackageVersions(target, pkg)[0]
303 cmd.extend(['--%s' % pkg, version])
304
305 cmd.extend(targets[target]['crossdev'].split())
306 if config_only:
307 # In this case we want to just quietly reinit
308 cmd.append('--init-target')
Mike Frysinger0282d222019-12-17 17:15:48 -0500309 cros_build_lib.run(cmd, print_cmd=False, stdout=True)
David James66a09c42012-11-05 13:31:38 -0800310 else:
Nicolas Boichat3fecf392019-11-25 02:58:28 +0000311 cros_build_lib.run(cmd)
David James66a09c42012-11-05 13:31:38 -0800312
Nicolas Boichat3fecf392019-11-25 02:58:28 +0000313 configured_targets.append(target)
David James66a09c42012-11-05 13:31:38 -0800314
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100315
Zdenek Behanf4d18a02012-03-22 15:45:05 +0100316def GetTargetPackages(target):
317 """Returns a list of packages for a given target."""
David James66a09c42012-11-05 13:31:38 -0800318 conf = Crossdev.GetConfig(target)
Zdenek Behanf4d18a02012-03-22 15:45:05 +0100319 # Undesired packages are denoted by empty ${pkg}_pn variable.
Han Shene23782f2016-02-18 12:20:00 -0800320 return [x for x in conf['crosspkgs'] if conf.get(x+'_pn')]
Zdenek Behanf4d18a02012-03-22 15:45:05 +0100321
322
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100323# Portage helper functions:
324def GetPortagePackage(target, package):
325 """Returns a package name for the given target."""
David James66a09c42012-11-05 13:31:38 -0800326 conf = Crossdev.GetConfig(target)
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100327 # Portage category:
Mike Frysinger785b0c32017-09-13 01:35:59 -0400328 if target.startswith('host') or package in Crossdev.MANUAL_PKGS:
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100329 category = conf[package + '_category']
Zdenek Behan508dcce2011-12-05 15:39:32 +0100330 else:
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100331 category = conf['category']
332 # Portage package:
333 pn = conf[package + '_pn']
334 # Final package name:
Mike Frysinger422faf42014-11-12 23:16:48 -0500335 assert category
336 assert pn
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100337 return '%s/%s' % (category, pn)
338
339
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700340def PortageTrees(root):
341 """Return the portage trees for a given root."""
342 if root == '/':
343 return portage.db['/']
344 # The portage logic requires the path always end in a slash.
345 root = root.rstrip('/') + '/'
346 return portage.create_trees(target_root=root, config_root=root)[root]
347
348
349def GetInstalledPackageVersions(atom, root='/'):
Zdenek Behan508dcce2011-12-05 15:39:32 +0100350 """Extracts the list of current versions of a target, package pair.
351
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500352 Args:
353 atom: The atom to operate on (e.g. sys-devel/gcc)
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700354 root: The root to check for installed packages.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100355
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500356 Returns:
357 The list of versions of the package currently installed.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100358 """
Zdenek Behan508dcce2011-12-05 15:39:32 +0100359 versions = []
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700360 for pkg in PortageTrees(root)['vartree'].dbapi.match(atom, use_cache=0):
Zdenek Behan508dcce2011-12-05 15:39:32 +0100361 version = portage.versions.cpv_getversion(pkg)
362 versions.append(version)
363 return versions
364
365
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700366def GetStablePackageVersion(atom, installed, root='/'):
Zdenek Behan508dcce2011-12-05 15:39:32 +0100367 """Extracts the current stable version for a given package.
368
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500369 Args:
370 atom: The target/package to operate on eg. i686-pc-linux-gnu,gcc
371 installed: Whether we want installed packages or ebuilds
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700372 root: The root to use when querying packages.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100373
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500374 Returns:
375 A string containing the latest version.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100376 """
David James90239b92012-11-05 15:31:34 -0800377 pkgtype = 'vartree' if installed else 'porttree'
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700378 cpv = portage.best(PortageTrees(root)[pkgtype].dbapi.match(atom, use_cache=0))
David James90239b92012-11-05 15:31:34 -0800379 return portage.versions.cpv_getversion(cpv) if cpv else None
Zdenek Behan508dcce2011-12-05 15:39:32 +0100380
381
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700382def VersionListToNumeric(target, package, versions, installed, root='/'):
Zdenek Behan508dcce2011-12-05 15:39:32 +0100383 """Resolves keywords in a given version list for a particular package.
384
385 Resolving means replacing PACKAGE_STABLE with the actual number.
386
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500387 Args:
388 target: The target to operate on (e.g. i686-pc-linux-gnu)
389 package: The target/package to operate on (e.g. gcc)
390 versions: List of versions to resolve
391 installed: Query installed packages
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700392 root: The install root to use; ignored if |installed| is False.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100393
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500394 Returns:
395 List of purely numeric versions equivalent to argument
Zdenek Behan508dcce2011-12-05 15:39:32 +0100396 """
397 resolved = []
David James90239b92012-11-05 15:31:34 -0800398 atom = GetPortagePackage(target, package)
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700399 if not installed:
400 root = '/'
Zdenek Behan508dcce2011-12-05 15:39:32 +0100401 for version in versions:
402 if version == PACKAGE_STABLE:
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700403 resolved.append(GetStablePackageVersion(atom, installed, root=root))
Mike Frysinger7f0e1982017-09-12 17:08:50 -0400404 else:
Zdenek Behan508dcce2011-12-05 15:39:32 +0100405 resolved.append(version)
406 return resolved
407
408
409def GetDesiredPackageVersions(target, package):
410 """Produces the list of desired versions for each target, package pair.
411
412 The first version in the list is implicitly treated as primary, ie.
413 the version that will be initialized by crossdev and selected.
414
415 If the version is PACKAGE_STABLE, it really means the current version which
416 is emerged by using the package atom with no particular version key.
417 Since crossdev unmasks all packages by default, this will actually
418 mean 'unstable' in most cases.
419
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500420 Args:
421 target: The target to operate on (e.g. i686-pc-linux-gnu)
422 package: The target/package to operate on (e.g. gcc)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100423
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500424 Returns:
425 A list composed of either a version string, PACKAGE_STABLE
Zdenek Behan508dcce2011-12-05 15:39:32 +0100426 """
Mike Frysingerd23be272017-09-12 17:18:44 -0400427 if package in GetTargetPackages(target):
428 return [PACKAGE_STABLE]
429 else:
430 return []
Zdenek Behan508dcce2011-12-05 15:39:32 +0100431
432
433def TargetIsInitialized(target):
434 """Verifies if the given list of targets has been correctly initialized.
435
436 This determines whether we have to call crossdev while emerging
437 toolchain packages or can do it using emerge. Emerge is naturally
438 preferred, because all packages can be updated in a single pass.
439
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500440 Args:
441 target: The target to operate on (e.g. i686-pc-linux-gnu)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100442
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500443 Returns:
444 True if |target| is completely initialized, else False
Zdenek Behan508dcce2011-12-05 15:39:32 +0100445 """
446 # Check if packages for the given target all have a proper version.
447 try:
Zdenek Behanf4d18a02012-03-22 15:45:05 +0100448 for package in GetTargetPackages(target):
David James66a09c42012-11-05 13:31:38 -0800449 atom = GetPortagePackage(target, package)
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100450 # Do we even want this package && is it initialized?
Mike Frysinger7f0e1982017-09-12 17:08:50 -0400451 if not (GetStablePackageVersion(atom, True) and
452 GetStablePackageVersion(atom, False)):
Zdenek Behan508dcce2011-12-05 15:39:32 +0100453 return False
454 return True
455 except cros_build_lib.RunCommandError:
456 # Fails - The target has likely never been initialized before.
457 return False
458
459
460def RemovePackageMask(target):
461 """Removes a package.mask file for the given platform.
462
463 The pre-existing package.mask files can mess with the keywords.
464
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500465 Args:
466 target: The target to operate on (e.g. i686-pc-linux-gnu)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100467 """
468 maskfile = os.path.join('/etc/portage/package.mask', 'cross-' + target)
Brian Harringaf019fb2012-05-10 15:06:13 -0700469 osutils.SafeUnlink(maskfile)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100470
471
Zdenek Behan508dcce2011-12-05 15:39:32 +0100472# Main functions performing the actual update steps.
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700473def RebuildLibtool(root='/'):
Mike Frysingerc880a962013-11-08 13:59:06 -0500474 """Rebuild libtool as needed
475
476 Libtool hardcodes full paths to internal gcc files, so whenever we upgrade
477 gcc, libtool will break. We can't use binary packages either as those will
478 most likely be compiled against the previous version of gcc.
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700479
480 Args:
481 root: The install root where we want libtool rebuilt.
Mike Frysingerc880a962013-11-08 13:59:06 -0500482 """
483 needs_update = False
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700484 with open(os.path.join(root, 'usr/bin/libtool')) as f:
Mike Frysingerc880a962013-11-08 13:59:06 -0500485 for line in f:
486 # Look for a line like:
487 # sys_lib_search_path_spec="..."
488 # It'll be a list of paths and gcc will be one of them.
489 if line.startswith('sys_lib_search_path_spec='):
490 line = line.rstrip()
491 for path in line.split('=', 1)[1].strip('"').split():
Mike Frysinger3bba5032016-09-20 14:15:04 -0400492 root_path = os.path.join(root, path.lstrip(os.path.sep))
493 logging.debug('Libtool: checking %s', root_path)
494 if not os.path.exists(root_path):
495 logging.info('Rebuilding libtool after gcc upgrade')
496 logging.info(' %s', line)
497 logging.info(' missing path: %s', path)
Mike Frysingerc880a962013-11-08 13:59:06 -0500498 needs_update = True
499 break
500
501 if needs_update:
502 break
503
504 if needs_update:
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700505 cmd = [EMERGE_CMD, '--oneshot']
506 if root != '/':
507 cmd.extend(['--sysroot=%s' % root, '--root=%s' % root])
508 cmd.append('sys-devel/libtool')
Mike Frysinger45602c72019-09-22 02:15:11 -0400509 cros_build_lib.run(cmd)
Mike Frysinger3bba5032016-09-20 14:15:04 -0400510 else:
511 logging.debug('Libtool is up-to-date; no need to rebuild')
Mike Frysingerc880a962013-11-08 13:59:06 -0500512
513
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700514def UpdateTargets(targets, usepkg, root='/'):
Zdenek Behan508dcce2011-12-05 15:39:32 +0100515 """Determines which packages need update/unmerge and defers to portage.
516
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500517 Args:
518 targets: The list of targets to update
519 usepkg: Copies the commandline option
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700520 root: The install root in which we want packages updated.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100521 """
Zdenek Behan508dcce2011-12-05 15:39:32 +0100522 # For each target, we do two things. Figure out the list of updates,
523 # and figure out the appropriate keywords/masks. Crossdev will initialize
524 # these, but they need to be regenerated on every update.
Mike Frysinger3bba5032016-09-20 14:15:04 -0400525 logging.info('Determining required toolchain updates...')
David James90239b92012-11-05 15:31:34 -0800526 mergemap = {}
Zdenek Behan508dcce2011-12-05 15:39:32 +0100527 for target in targets:
Mike Frysinger3bba5032016-09-20 14:15:04 -0400528 logging.debug('Updating target %s', target)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100529 # Record the highest needed version for each target, for masking purposes.
530 RemovePackageMask(target)
Zdenek Behanf4d18a02012-03-22 15:45:05 +0100531 for package in GetTargetPackages(target):
Zdenek Behan508dcce2011-12-05 15:39:32 +0100532 # Portage name for the package
Mike Frysinger7f0e1982017-09-12 17:08:50 -0400533 logging.debug(' Checking package %s', package)
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100534 pkg = GetPortagePackage(target, package)
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700535 current = GetInstalledPackageVersions(pkg, root=root)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100536 desired = GetDesiredPackageVersions(target, package)
Zdenek Behan699ddd32012-04-13 07:14:08 +0200537 desired_num = VersionListToNumeric(target, package, desired, False)
Mike Frysinger785b0c32017-09-13 01:35:59 -0400538 if pkg in NEW_PACKAGES and usepkg:
539 # Skip this binary package (for now).
540 continue
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100541 mergemap[pkg] = set(desired_num).difference(current)
Mike Frysinger7f0e1982017-09-12 17:08:50 -0400542 logging.debug(' %s -> %s', current, desired_num)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100543
Mike Frysinger7f0e1982017-09-12 17:08:50 -0400544 packages = [pkg for pkg, vers in mergemap.items() if vers]
Zdenek Behan508dcce2011-12-05 15:39:32 +0100545 if not packages:
Mike Frysinger3bba5032016-09-20 14:15:04 -0400546 logging.info('Nothing to update!')
David Jamesf8c672f2012-11-06 13:38:11 -0800547 return False
Zdenek Behan508dcce2011-12-05 15:39:32 +0100548
Mike Frysinger3bba5032016-09-20 14:15:04 -0400549 logging.info('Updating packages:')
550 logging.info('%s', packages)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100551
Zdenek Behan7e33b4e2012-03-12 17:00:56 +0100552 cmd = [EMERGE_CMD, '--oneshot', '--update']
Zdenek Behan508dcce2011-12-05 15:39:32 +0100553 if usepkg:
554 cmd.extend(['--getbinpkg', '--usepkgonly'])
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700555 if root != '/':
556 cmd.extend(['--sysroot=%s' % root, '--root=%s' % root])
Zdenek Behan508dcce2011-12-05 15:39:32 +0100557
558 cmd.extend(packages)
Mike Frysinger45602c72019-09-22 02:15:11 -0400559 cros_build_lib.run(cmd)
David Jamesf8c672f2012-11-06 13:38:11 -0800560 return True
Zdenek Behan508dcce2011-12-05 15:39:32 +0100561
562
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700563def CleanTargets(targets, root='/'):
564 """Unmerges old packages that are assumed unnecessary.
565
566 Args:
567 targets: The list of targets to clean up.
568 root: The install root in which we want packages cleaned up.
569 """
Zdenek Behan508dcce2011-12-05 15:39:32 +0100570 unmergemap = {}
571 for target in targets:
Mike Frysinger3bba5032016-09-20 14:15:04 -0400572 logging.debug('Cleaning target %s', target)
Zdenek Behanf4d18a02012-03-22 15:45:05 +0100573 for package in GetTargetPackages(target):
Mike Frysinger3bba5032016-09-20 14:15:04 -0400574 logging.debug(' Cleaning package %s', package)
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100575 pkg = GetPortagePackage(target, package)
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700576 current = GetInstalledPackageVersions(pkg, root=root)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100577 desired = GetDesiredPackageVersions(target, package)
Gilad Arnold2fcb0aa2015-05-21 14:51:41 -0700578 # NOTE: This refers to installed packages (vartree) rather than the
579 # Portage version (porttree and/or bintree) when determining the current
580 # version. While this isn't the most accurate thing to do, it is probably
581 # a good simple compromise, which should have the desired result of
582 # uninstalling everything but the latest installed version. In
583 # particular, using the bintree (--usebinpkg) requires a non-trivial
584 # binhost sync and is probably more complex than useful.
Zdenek Behan699ddd32012-04-13 07:14:08 +0200585 desired_num = VersionListToNumeric(target, package, desired, True)
586 if not set(desired_num).issubset(current):
Mike Frysinger3bba5032016-09-20 14:15:04 -0400587 logging.warning('Error detecting stable version for %s, '
588 'skipping clean!', pkg)
Zdenek Behan699ddd32012-04-13 07:14:08 +0200589 return
Zdenek Behan508dcce2011-12-05 15:39:32 +0100590 unmergemap[pkg] = set(current).difference(desired_num)
591
592 # Cleaning doesn't care about consistency and rebuilding package.* files.
593 packages = []
Mike Frysinger0bdbc102019-06-13 15:27:29 -0400594 for pkg, vers in unmergemap.items():
Zdenek Behan508dcce2011-12-05 15:39:32 +0100595 packages.extend('=%s-%s' % (pkg, ver) for ver in vers if ver != '9999')
596
597 if packages:
Mike Frysinger3bba5032016-09-20 14:15:04 -0400598 logging.info('Cleaning packages:')
599 logging.info('%s', packages)
Zdenek Behan7e33b4e2012-03-12 17:00:56 +0100600 cmd = [EMERGE_CMD, '--unmerge']
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700601 if root != '/':
602 cmd.extend(['--sysroot=%s' % root, '--root=%s' % root])
Zdenek Behan508dcce2011-12-05 15:39:32 +0100603 cmd.extend(packages)
Mike Frysinger45602c72019-09-22 02:15:11 -0400604 cros_build_lib.run(cmd)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100605 else:
Mike Frysinger3bba5032016-09-20 14:15:04 -0400606 logging.info('Nothing to clean!')
Zdenek Behan508dcce2011-12-05 15:39:32 +0100607
608
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700609def SelectActiveToolchains(targets, suffixes, root='/'):
Zdenek Behan508dcce2011-12-05 15:39:32 +0100610 """Runs gcc-config and binutils-config to select the desired.
611
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500612 Args:
613 targets: The targets to select
614 suffixes: Optional target-specific hacks
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700615 root: The root where we want to select toolchain versions.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100616 """
617 for package in ['gcc', 'binutils']:
618 for target in targets:
Mike Frysinger785b0c32017-09-13 01:35:59 -0400619 # See if this package is part of this target.
620 if package not in GetTargetPackages(target):
621 logging.debug('%s: %s is not used', target, package)
622 continue
623
Zdenek Behan508dcce2011-12-05 15:39:32 +0100624 # Pick the first version in the numbered list as the selected one.
625 desired = GetDesiredPackageVersions(target, package)
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700626 desired_num = VersionListToNumeric(target, package, desired, True,
627 root=root)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100628 desired = desired_num[0]
629 # *-config does not play revisions, strip them, keep just PV.
630 desired = portage.versions.pkgsplit('%s-%s' % (package, desired))[1]
631
Mike Frysinger785b0c32017-09-13 01:35:59 -0400632 if target.startswith('host'):
Zdenek Behan508dcce2011-12-05 15:39:32 +0100633 # *-config is the only tool treating host identically (by tuple).
David James27ac4ae2012-12-03 23:16:15 -0800634 target = toolchain.GetHostTuple()
Zdenek Behan508dcce2011-12-05 15:39:32 +0100635
636 # And finally, attach target to it.
637 desired = '%s-%s' % (target, desired)
638
639 # Target specific hacks
640 if package in suffixes:
641 if target in suffixes[package]:
642 desired += suffixes[package][target]
643
David James7ec5efc2012-11-06 09:39:49 -0800644 extra_env = {'CHOST': target}
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700645 if root != '/':
646 extra_env['ROOT'] = root
David James7ec5efc2012-11-06 09:39:49 -0800647 cmd = ['%s-config' % package, '-c', target]
Mike Frysinger45602c72019-09-22 02:15:11 -0400648 result = cros_build_lib.run(
Mike Frysinger0282d222019-12-17 17:15:48 -0500649 cmd, print_cmd=False, stdout=True, encoding='utf-8',
Mike Frysingerb3202be2019-11-15 22:25:59 -0500650 extra_env=extra_env)
Mike Frysingerd6e2df02014-11-26 02:55:04 -0500651 current = result.output.splitlines()[0]
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700652
653 # Do not reconfig when the current is live or nothing needs to be done.
654 extra_env = {'ROOT': root} if root != '/' else None
Zdenek Behan508dcce2011-12-05 15:39:32 +0100655 if current != desired and current != '9999':
Mike Frysingerd6e2df02014-11-26 02:55:04 -0500656 cmd = [package + '-config', desired]
Mike Frysinger45602c72019-09-22 02:15:11 -0400657 cros_build_lib.run(cmd, print_cmd=False, extra_env=extra_env)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100658
659
Mike Frysinger35247af2012-11-16 18:58:06 -0500660def ExpandTargets(targets_wanted):
661 """Expand any possible toolchain aliases into full targets
662
663 This will expand 'all' and 'sdk' into the respective toolchain tuples.
664
665 Args:
666 targets_wanted: The targets specified by the user.
Mike Frysinger1a736a82013-12-12 01:50:59 -0500667
Mike Frysinger35247af2012-11-16 18:58:06 -0500668 Returns:
Gilad Arnold8195b532015-04-07 10:56:30 +0300669 Dictionary of concrete targets and their toolchain tuples.
Mike Frysinger35247af2012-11-16 18:58:06 -0500670 """
Mike Frysinger35247af2012-11-16 18:58:06 -0500671 targets_wanted = set(targets_wanted)
Don Garrettc0c74002015-10-09 12:58:19 -0700672 if targets_wanted == set(['boards']):
673 # Only pull targets from the included boards.
Gilad Arnold8195b532015-04-07 10:56:30 +0300674 return {}
675
676 all_targets = toolchain.GetAllTargets()
Mike Frysinger35247af2012-11-16 18:58:06 -0500677 if targets_wanted == set(['all']):
Gilad Arnold8195b532015-04-07 10:56:30 +0300678 return all_targets
679 if targets_wanted == set(['sdk']):
Mike Frysinger35247af2012-11-16 18:58:06 -0500680 # Filter out all the non-sdk toolchains as we don't want to mess
681 # with those in all of our builds.
Gilad Arnold8195b532015-04-07 10:56:30 +0300682 return toolchain.FilterToolchains(all_targets, 'sdk', True)
683
684 # Verify user input.
685 nonexistent = targets_wanted.difference(all_targets)
686 if nonexistent:
Mike Frysingercd790662019-10-17 00:23:13 -0400687 raise ValueError('Invalid targets: %s' % (','.join(nonexistent),))
Gilad Arnold8195b532015-04-07 10:56:30 +0300688 return {t: all_targets[t] for t in targets_wanted}
Mike Frysinger35247af2012-11-16 18:58:06 -0500689
690
David Jamesf8c672f2012-11-06 13:38:11 -0800691def UpdateToolchains(usepkg, deleteold, hostonly, reconfig,
Don Garrettc0c74002015-10-09 12:58:19 -0700692 targets_wanted, boards_wanted, root='/'):
Zdenek Behan508dcce2011-12-05 15:39:32 +0100693 """Performs all steps to create a synchronized toolchain enviroment.
694
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500695 Args:
696 usepkg: Use prebuilt packages
697 deleteold: Unmerge deprecated packages
698 hostonly: Only setup the host toolchain
699 reconfig: Reload crossdev config and reselect toolchains
700 targets_wanted: All the targets to update
701 boards_wanted: Load targets from these boards
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700702 root: The root in which to install the toolchains.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100703 """
David Jamesf8c672f2012-11-06 13:38:11 -0800704 targets, crossdev_targets, reconfig_targets = {}, {}, {}
Zdenek Behan7e33b4e2012-03-12 17:00:56 +0100705 if not hostonly:
706 # For hostonly, we can skip most of the below logic, much of which won't
707 # work on bare systems where this is useful.
Mike Frysinger35247af2012-11-16 18:58:06 -0500708 targets = ExpandTargets(targets_wanted)
Mike Frysinger7ccee992012-06-01 21:27:59 -0400709
Don Garrettc0c74002015-10-09 12:58:19 -0700710 # Now re-add any targets that might be from this board. This is to
Gilad Arnold8195b532015-04-07 10:56:30 +0300711 # allow unofficial boards to declare their own toolchains.
Mike Frysinger7ccee992012-06-01 21:27:59 -0400712 for board in boards_wanted:
David James27ac4ae2012-12-03 23:16:15 -0800713 targets.update(toolchain.GetToolchainsForBoard(board))
Zdenek Behan508dcce2011-12-05 15:39:32 +0100714
Zdenek Behan7e33b4e2012-03-12 17:00:56 +0100715 # First check and initialize all cross targets that need to be.
Mike Frysinger7ccee992012-06-01 21:27:59 -0400716 for target in targets:
717 if TargetIsInitialized(target):
718 reconfig_targets[target] = targets[target]
719 else:
720 crossdev_targets[target] = targets[target]
Zdenek Behan7e33b4e2012-03-12 17:00:56 +0100721 if crossdev_targets:
Mike Frysinger3bba5032016-09-20 14:15:04 -0400722 logging.info('The following targets need to be re-initialized:')
723 logging.info('%s', crossdev_targets)
David James66a09c42012-11-05 13:31:38 -0800724 Crossdev.UpdateTargets(crossdev_targets, usepkg)
Zdenek Behan8be29ba2012-05-29 23:10:34 +0200725 # Those that were not initialized may need a config update.
David James66a09c42012-11-05 13:31:38 -0800726 Crossdev.UpdateTargets(reconfig_targets, usepkg, config_only=True)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100727
Mike Frysinger66814c32017-10-09 18:11:46 -0400728 # If we're building a subset of toolchains for a board, we might not have
729 # all the tuples that the packages expect. We don't define the "full" set
730 # of tuples currently other than "whatever the full sdk has normally".
731 if usepkg or set(('all', 'sdk')) & targets_wanted:
732 # Since we have cross-compilers now, we can update these packages.
733 targets['host-post-cross'] = {}
Mike Frysinger785b0c32017-09-13 01:35:59 -0400734
Zdenek Behan7e33b4e2012-03-12 17:00:56 +0100735 # We want host updated.
Mike Frysinger7ccee992012-06-01 21:27:59 -0400736 targets['host'] = {}
Zdenek Behan7e33b4e2012-03-12 17:00:56 +0100737
738 # Now update all packages.
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700739 if UpdateTargets(targets, usepkg, root=root) or crossdev_targets or reconfig:
740 SelectActiveToolchains(targets, CONFIG_TARGET_SUFFIXES, root=root)
David James7ec5efc2012-11-06 09:39:49 -0800741
742 if deleteold:
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700743 CleanTargets(targets, root=root)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100744
Mike Frysingerc880a962013-11-08 13:59:06 -0500745 # Now that we've cleared out old versions, see if we need to rebuild
746 # anything. Can't do this earlier as it might not be broken.
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700747 RebuildLibtool(root=root)
Mike Frysingerc880a962013-11-08 13:59:06 -0500748
Zdenek Behan508dcce2011-12-05 15:39:32 +0100749
Bertrand SIMONNETcae9d5f2015-03-09 15:58:01 -0700750def ShowConfig(name):
751 """Show the toolchain tuples used by |name|
Mike Frysinger35247af2012-11-16 18:58:06 -0500752
753 Args:
Don Garrettc0c74002015-10-09 12:58:19 -0700754 name: The board name to query.
Mike Frysinger35247af2012-11-16 18:58:06 -0500755 """
Don Garrettc0c74002015-10-09 12:58:19 -0700756 toolchains = toolchain.GetToolchainsForBoard(name)
Mike Frysinger35247af2012-11-16 18:58:06 -0500757 # Make sure we display the default toolchain first.
Mike Frysinger3bba5032016-09-20 14:15:04 -0400758 # Note: Do not use logging here as this is meant to be used by other tools.
Mike Frysinger383367e2014-09-16 15:06:17 -0400759 print(','.join(
Mike Frysinger818d9632019-08-24 14:43:05 -0400760 list(toolchain.FilterToolchains(toolchains, 'default', True)) +
761 list(toolchain.FilterToolchains(toolchains, 'default', False))))
Mike Frysinger35247af2012-11-16 18:58:06 -0500762
763
Mike Frysinger35247af2012-11-16 18:58:06 -0500764def GeneratePathWrapper(root, wrappath, path):
765 """Generate a shell script to execute another shell script
766
767 Since we can't symlink a wrapped ELF (see GenerateLdsoWrapper) because the
768 argv[0] won't be pointing to the correct path, generate a shell script that
769 just executes another program with its full path.
770
771 Args:
772 root: The root tree to generate scripts inside of
773 wrappath: The full path (inside |root|) to create the wrapper
774 path: The target program which this wrapper will execute
775 """
776 replacements = {
777 'path': path,
778 'relroot': os.path.relpath('/', os.path.dirname(wrappath)),
779 }
Takuto Ikuta58403972018-08-16 18:52:51 +0900780
781 # Do not use exec here, because exec invokes script with absolute path in
782 # argv0. Keeping relativeness allows us to remove abs path from compile result
783 # and leads directory independent build cache sharing in some distributed
784 # build system.
Mike Frysinger35247af2012-11-16 18:58:06 -0500785 wrapper = """#!/bin/sh
Takuto Ikuta58403972018-08-16 18:52:51 +0900786basedir=$(dirname "$0")
787"${basedir}/%(relroot)s%(path)s" "$@"
788exit "$?"
Mike Frysinger35247af2012-11-16 18:58:06 -0500789""" % replacements
790 root_wrapper = root + wrappath
791 if os.path.islink(root_wrapper):
792 os.unlink(root_wrapper)
793 else:
794 osutils.SafeMakedirs(os.path.dirname(root_wrapper))
795 osutils.WriteFile(root_wrapper, wrapper)
Mike Frysinger60ec1012013-10-21 00:11:10 -0400796 os.chmod(root_wrapper, 0o755)
Mike Frysinger35247af2012-11-16 18:58:06 -0500797
798
Rahul Chaudhrya8127bb2016-07-22 15:53:17 -0700799def FixClangXXWrapper(root, path):
800 """Fix wrapper shell scripts and symlinks for invoking clang++
801
802 In a typical installation, clang++ symlinks to clang, which symlinks to the
803 elf executable. The executable distinguishes between clang and clang++ based
804 on argv[0].
805
806 When invoked through the LdsoWrapper, argv[0] always contains the path to the
807 executable elf file, making clang/clang++ invocations indistinguishable.
808
809 This function detects if the elf executable being wrapped is clang-X.Y, and
810 fixes wrappers/symlinks as necessary so that clang++ will work correctly.
811
812 The calling sequence now becomes:
813 -) clang++ invocation turns into clang++-3.9 (which is a copy of clang-3.9,
814 the Ldsowrapper).
815 -) clang++-3.9 uses the Ldso to invoke clang++-3.9.elf, which is a symlink
816 to the original clang-3.9 elf.
817 -) The difference this time is that inside the elf file execution, $0 is
818 set as .../usr/bin/clang++-3.9.elf, which contains 'clang++' in the name.
819
Manoj Guptaae268142018-04-27 23:28:36 -0700820 Update: Starting since clang 7, the clang and clang++ are symlinks to
821 clang-7 binary, not clang-7.0. The pattern match is extended to handle
822 both clang-7 and clang-7.0 cases for now. (https://crbug.com/837889)
823
Rahul Chaudhrya8127bb2016-07-22 15:53:17 -0700824 Args:
825 root: The root tree to generate scripts / symlinks inside of
826 path: The target elf for which LdsoWrapper was created
827 """
Manoj Guptaae268142018-04-27 23:28:36 -0700828 if re.match(r'/usr/bin/clang-\d+(\.\d+)*$', path):
Rahul Chaudhrya8127bb2016-07-22 15:53:17 -0700829 logging.info('fixing clang++ invocation for %s', path)
830 clangdir = os.path.dirname(root + path)
831 clang = os.path.basename(path)
832 clangxx = clang.replace('clang', 'clang++')
833
834 # Create a symlink clang++-X.Y.elf to point to clang-X.Y.elf
835 os.symlink(clang + '.elf', os.path.join(clangdir, clangxx + '.elf'))
836
837 # Create a hardlink clang++-X.Y pointing to clang-X.Y
838 os.link(os.path.join(clangdir, clang), os.path.join(clangdir, clangxx))
839
840 # Adjust the clang++ symlink to point to clang++-X.Y
841 os.unlink(os.path.join(clangdir, 'clang++'))
842 os.symlink(clangxx, os.path.join(clangdir, 'clang++'))
843
844
Mike Frysinger35247af2012-11-16 18:58:06 -0500845def FileIsCrosSdkElf(elf):
846 """Determine if |elf| is an ELF that we execute in the cros_sdk
847
848 We don't need this to be perfect, just quick. It makes sure the ELF
849 is a 64bit LSB x86_64 ELF. That is the native type of cros_sdk.
850
851 Args:
852 elf: The file to check
Mike Frysinger1a736a82013-12-12 01:50:59 -0500853
Mike Frysinger35247af2012-11-16 18:58:06 -0500854 Returns:
855 True if we think |elf| is a native ELF
856 """
Mike Frysinger4a12bf12019-12-04 04:20:10 -0500857 with open(elf, 'rb') as f:
Mike Frysinger35247af2012-11-16 18:58:06 -0500858 data = f.read(20)
859 # Check the magic number, EI_CLASS, EI_DATA, and e_machine.
Mike Frysinger4a12bf12019-12-04 04:20:10 -0500860 return (data[0:4] == b'\x7fELF' and
Mike Frysingerdd34dfe2019-12-10 16:25:47 -0500861 data[4:5] == b'\x02' and
862 data[5:6] == b'\x01' and
863 data[18:19] == b'\x3e')
Mike Frysinger35247af2012-11-16 18:58:06 -0500864
865
866def IsPathPackagable(ptype, path):
867 """Should the specified file be included in a toolchain package?
868
869 We only need to handle files as we'll create dirs as we need them.
870
871 Further, trim files that won't be useful:
872 - non-english translations (.mo) since it'd require env vars
873 - debug files since these are for the host compiler itself
874 - info/man pages as they're big, and docs are online, and the
875 native docs should work fine for the most part (`man gcc`)
876
877 Args:
878 ptype: A string describing the path type (i.e. 'file' or 'dir' or 'sym')
879 path: The full path to inspect
Mike Frysinger1a736a82013-12-12 01:50:59 -0500880
Mike Frysinger35247af2012-11-16 18:58:06 -0500881 Returns:
882 True if we want to include this path in the package
883 """
884 return not (ptype in ('dir',) or
885 path.startswith('/usr/lib/debug/') or
886 os.path.splitext(path)[1] == '.mo' or
887 ('/man/' in path or '/info/' in path))
888
889
890def ReadlinkRoot(path, root):
891 """Like os.readlink(), but relative to a |root|
892
893 Args:
894 path: The symlink to read
895 root: The path to use for resolving absolute symlinks
Mike Frysinger1a736a82013-12-12 01:50:59 -0500896
Mike Frysinger35247af2012-11-16 18:58:06 -0500897 Returns:
898 A fully resolved symlink path
899 """
900 while os.path.islink(root + path):
901 path = os.path.join(os.path.dirname(path), os.readlink(root + path))
902 return path
903
904
905def _GetFilesForTarget(target, root='/'):
906 """Locate all the files to package for |target|
907
908 This does not cover ELF dependencies.
909
910 Args:
911 target: The toolchain target name
912 root: The root path to pull all packages from
Mike Frysinger1a736a82013-12-12 01:50:59 -0500913
Mike Frysinger35247af2012-11-16 18:58:06 -0500914 Returns:
915 A tuple of a set of all packable paths, and a set of all paths which
916 are also native ELFs
917 """
918 paths = set()
919 elfs = set()
920
921 # Find all the files owned by the packages for this target.
922 for pkg in GetTargetPackages(target):
Mike Frysinger35247af2012-11-16 18:58:06 -0500923
Rahul Chaudhry4b803052015-05-13 15:25:56 -0700924 # Skip Go compiler from redistributable packages.
925 # The "go" executable has GOROOT=/usr/lib/go/${CTARGET} hardcoded
926 # into it. Due to this, the toolchain cannot be unpacked anywhere
927 # else and be readily useful. To enable packaging Go, we need to:
928 # -) Tweak the wrappers/environment to override GOROOT
929 # automatically based on the unpack location.
930 # -) Make sure the ELF dependency checking and wrapping logic
931 # below skips the Go toolchain executables and libraries.
932 # -) Make sure the packaging process maintains the relative
933 # timestamps of precompiled standard library packages.
934 # (see dev-lang/go ebuild for details).
935 if pkg == 'ex_go':
936 continue
937
Yunlian Jiang36f35242018-04-27 10:18:40 -0700938 # Use armv7a-cros-linux-gnueabi/compiler-rt for
939 # armv7a-cros-linux-gnueabihf/compiler-rt.
940 # Currently the armv7a-cros-linux-gnueabi is actually
941 # the same as armv7a-cros-linux-gnueabihf with different names.
942 # Because of that, for compiler-rt, it generates the same binary in
943 # the same location. To avoid the installation conflict, we do not
944 # install anything for 'armv7a-cros-linux-gnueabihf'. This would cause
945 # problem if other people try to use standalone armv7a-cros-linux-gnueabihf
946 # toolchain.
947 if 'compiler-rt' in pkg and 'armv7a-cros-linux-gnueabi' in target:
948 atom = GetPortagePackage(target, pkg)
949 cat, pn = atom.split('/')
950 ver = GetInstalledPackageVersions(atom, root=root)[0]
Yunlian Jiang36f35242018-04-27 10:18:40 -0700951 dblink = portage.dblink(cat, pn + '-' + ver, myroot=root,
952 settings=portage.settings)
953 contents = dblink.getcontents()
954 if not contents:
955 if 'hf' in target:
956 new_target = 'armv7a-cros-linux-gnueabi'
957 else:
958 new_target = 'armv7a-cros-linux-gnueabihf'
959 atom = GetPortagePackage(new_target, pkg)
960 else:
961 atom = GetPortagePackage(target, pkg)
962
Mike Frysinger35247af2012-11-16 18:58:06 -0500963 cat, pn = atom.split('/')
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700964 ver = GetInstalledPackageVersions(atom, root=root)[0]
Ralph Nathan03047282015-03-23 11:09:32 -0700965 logging.info('packaging %s-%s', atom, ver)
Mike Frysinger35247af2012-11-16 18:58:06 -0500966
Mike Frysinger35247af2012-11-16 18:58:06 -0500967 dblink = portage.dblink(cat, pn + '-' + ver, myroot=root,
968 settings=portage.settings)
969 contents = dblink.getcontents()
970 for obj in contents:
971 ptype = contents[obj][0]
972 if not IsPathPackagable(ptype, obj):
973 continue
974
975 if ptype == 'obj':
976 # For native ELFs, we need to pull in their dependencies too.
977 if FileIsCrosSdkElf(obj):
Mike Frysinger60a2f6c2019-12-04 04:18:58 -0500978 logging.debug('Adding ELF %s', obj)
Mike Frysinger35247af2012-11-16 18:58:06 -0500979 elfs.add(obj)
Mike Frysinger60a2f6c2019-12-04 04:18:58 -0500980 logging.debug('Adding path %s', obj)
Mike Frysinger35247af2012-11-16 18:58:06 -0500981 paths.add(obj)
982
983 return paths, elfs
984
985
986def _BuildInitialPackageRoot(output_dir, paths, elfs, ldpaths,
Mike Frysingerd6e2df02014-11-26 02:55:04 -0500987 path_rewrite_func=lambda x: x, root='/'):
Mike Frysinger35247af2012-11-16 18:58:06 -0500988 """Link in all packable files and their runtime dependencies
989
990 This also wraps up executable ELFs with helper scripts.
991
992 Args:
993 output_dir: The output directory to store files
994 paths: All the files to include
995 elfs: All the files which are ELFs (a subset of |paths|)
996 ldpaths: A dict of static ldpath information
997 path_rewrite_func: User callback to rewrite paths in output_dir
998 root: The root path to pull all packages/files from
999 """
1000 # Link in all the files.
Mike Frysinger221bd822017-09-29 02:51:47 -04001001 sym_paths = {}
Mike Frysinger35247af2012-11-16 18:58:06 -05001002 for path in paths:
1003 new_path = path_rewrite_func(path)
Mike Frysinger60a2f6c2019-12-04 04:18:58 -05001004 logging.debug('Transformed %s to %s', path, new_path)
Mike Frysinger35247af2012-11-16 18:58:06 -05001005 dst = output_dir + new_path
1006 osutils.SafeMakedirs(os.path.dirname(dst))
1007
1008 # Is this a symlink which we have to rewrite or wrap?
1009 # Delay wrap check until after we have created all paths.
1010 src = root + path
1011 if os.path.islink(src):
1012 tgt = os.readlink(src)
1013 if os.path.sep in tgt:
Mike Frysinger221bd822017-09-29 02:51:47 -04001014 sym_paths[lddtree.normpath(ReadlinkRoot(src, root))] = new_path
Mike Frysinger35247af2012-11-16 18:58:06 -05001015
1016 # Rewrite absolute links to relative and then generate the symlink
1017 # ourselves. All other symlinks can be hardlinked below.
1018 if tgt[0] == '/':
1019 tgt = os.path.relpath(tgt, os.path.dirname(new_path))
1020 os.symlink(tgt, dst)
1021 continue
1022
Mike Frysinger60a2f6c2019-12-04 04:18:58 -05001023 logging.debug('Linking path %s -> %s', src, dst)
Mike Frysinger35247af2012-11-16 18:58:06 -05001024 os.link(src, dst)
1025
Mike Frysinger35247af2012-11-16 18:58:06 -05001026 # Locate all the dependencies for all the ELFs. Stick them all in the
1027 # top level "lib" dir to make the wrapper simpler. This exact path does
1028 # not matter since we execute ldso directly, and we tell the ldso the
1029 # exact path to search for its libraries.
1030 libdir = os.path.join(output_dir, 'lib')
1031 osutils.SafeMakedirs(libdir)
1032 donelibs = set()
Mike Frysinger9fe02342019-12-12 17:52:53 -05001033 basenamelibs = set()
Mike Frysinger4331fc62017-09-28 14:03:25 -04001034 glibc_re = re.compile(r'/lib(c|pthread)-[0-9.]+\.so$')
Mike Frysinger35247af2012-11-16 18:58:06 -05001035 for elf in elfs:
Mike Frysingerea7688e2014-07-31 22:40:33 -04001036 e = lddtree.ParseELF(elf, root=root, ldpaths=ldpaths)
Mike Frysinger60a2f6c2019-12-04 04:18:58 -05001037 logging.debug('Parsed elf %s data: %s', elf, e)
Mike Frysinger35247af2012-11-16 18:58:06 -05001038 interp = e['interp']
Yunlian Jiang196e8f42017-09-20 12:29:47 -07001039 # Do not create wrapper for libc. crbug.com/766827
1040 if interp and not glibc_re.search(elf):
Mike Frysinger35247af2012-11-16 18:58:06 -05001041 # Generate a wrapper if it is executable.
Mike Frysingerc2ec0902013-03-26 01:28:45 -04001042 interp = os.path.join('/lib', os.path.basename(interp))
1043 lddtree.GenerateLdsoWrapper(output_dir, path_rewrite_func(elf), interp,
1044 libpaths=e['rpath'] + e['runpath'])
Rahul Chaudhrya8127bb2016-07-22 15:53:17 -07001045 FixClangXXWrapper(output_dir, path_rewrite_func(elf))
Mike Frysinger35247af2012-11-16 18:58:06 -05001046
Mike Frysinger221bd822017-09-29 02:51:47 -04001047 # Wrap any symlinks to the wrapper.
1048 if elf in sym_paths:
1049 link = sym_paths[elf]
1050 GeneratePathWrapper(output_dir, link, elf)
1051
Mike Frysinger9fe02342019-12-12 17:52:53 -05001052 # TODO(crbug.com/917193): Drop this hack once libopcodes linkage is fixed.
1053 if os.path.basename(elf).startswith('libopcodes-'):
1054 continue
Mike Frysinger35247af2012-11-16 18:58:06 -05001055
Mike Frysinger9fe02342019-12-12 17:52:53 -05001056 for lib, lib_data in e['libs'].items():
Mike Frysinger35247af2012-11-16 18:58:06 -05001057 src = path = lib_data['path']
1058 if path is None:
Ralph Nathan446aee92015-03-23 14:44:56 -07001059 logging.warning('%s: could not locate %s', elf, lib)
Mike Frysinger35247af2012-11-16 18:58:06 -05001060 continue
Mike Frysinger9fe02342019-12-12 17:52:53 -05001061
1062 # No need to try and copy the same source lib multiple times.
1063 if path in donelibs:
1064 continue
1065 donelibs.add(path)
1066
1067 # Die if we try to normalize different source libs with the same basename.
1068 if lib in basenamelibs:
1069 logging.error('Multiple sources detected for %s:\n new: %s\n old: %s',
1070 os.path.join('/lib', lib), path,
1071 ' '.join(x for x in donelibs
1072 if x != path and os.path.basename(x) == lib))
1073 # TODO(crbug.com/917193): Make this fatal.
1074 # cros_build_lib.Die('Unable to resolve lib conflicts')
1075 continue
1076 basenamelibs.add(lib)
Mike Frysinger35247af2012-11-16 18:58:06 -05001077
1078 # Needed libs are the SONAME, but that is usually a symlink, not a
1079 # real file. So link in the target rather than the symlink itself.
1080 # We have to walk all the possible symlinks (SONAME could point to a
1081 # symlink which points to a symlink), and we have to handle absolute
1082 # ourselves (since we have a "root" argument).
1083 dst = os.path.join(libdir, os.path.basename(path))
1084 src = ReadlinkRoot(src, root)
1085
Mike Frysinger60a2f6c2019-12-04 04:18:58 -05001086 logging.debug('Linking lib %s -> %s', root + src, dst)
Mike Frysinger35247af2012-11-16 18:58:06 -05001087 os.link(root + src, dst)
1088
1089
1090def _EnvdGetVar(envd, var):
1091 """Given a Gentoo env.d file, extract a var from it
1092
1093 Args:
1094 envd: The env.d file to load (may be a glob path)
1095 var: The var to extract
Mike Frysinger1a736a82013-12-12 01:50:59 -05001096
Mike Frysinger35247af2012-11-16 18:58:06 -05001097 Returns:
1098 The value of |var|
1099 """
1100 envds = glob.glob(envd)
1101 assert len(envds) == 1, '%s: should have exactly 1 env.d file' % envd
1102 envd = envds[0]
Mike Frysingere652ba12019-09-08 00:57:43 -04001103 return key_value_store.LoadFile(envd)[var]
Mike Frysinger35247af2012-11-16 18:58:06 -05001104
1105
1106def _ProcessBinutilsConfig(target, output_dir):
1107 """Do what binutils-config would have done"""
1108 binpath = os.path.join('/bin', target + '-')
Mike Frysingerd4d40fd2014-11-06 17:30:57 -05001109
1110 # Locate the bin dir holding the gold linker.
1111 binutils_bin_path = os.path.join(output_dir, 'usr', toolchain.GetHostTuple(),
1112 target, 'binutils-bin')
1113 globpath = os.path.join(binutils_bin_path, '*-gold')
Mike Frysinger35247af2012-11-16 18:58:06 -05001114 srcpath = glob.glob(globpath)
Mike Frysingerd4d40fd2014-11-06 17:30:57 -05001115 if not srcpath:
1116 # Maybe this target doesn't support gold.
1117 globpath = os.path.join(binutils_bin_path, '*')
1118 srcpath = glob.glob(globpath)
1119 assert len(srcpath) == 1, ('%s: matched more than one path (but not *-gold)'
1120 % globpath)
1121 srcpath = srcpath[0]
1122 ld_path = os.path.join(srcpath, 'ld')
1123 assert os.path.exists(ld_path), '%s: linker is missing!' % ld_path
1124 ld_path = os.path.join(srcpath, 'ld.bfd')
1125 assert os.path.exists(ld_path), '%s: linker is missing!' % ld_path
1126 ld_path = os.path.join(srcpath, 'ld.gold')
1127 assert not os.path.exists(ld_path), ('%s: exists, but gold dir does not!'
1128 % ld_path)
1129
1130 # Nope, no gold support to be found.
1131 gold_supported = False
Ralph Nathan446aee92015-03-23 14:44:56 -07001132 logging.warning('%s: binutils lacks support for the gold linker', target)
Mike Frysingerd4d40fd2014-11-06 17:30:57 -05001133 else:
1134 assert len(srcpath) == 1, '%s: did not match exactly 1 path' % globpath
Mike Frysinger78b7a812014-11-26 19:45:23 -05001135 srcpath = srcpath[0]
Mike Frysingerd4d40fd2014-11-06 17:30:57 -05001136
Rahul Chaudhry4891b4d2017-03-08 10:31:27 -08001137 # Package the binutils-bin directory without the '-gold' suffix
1138 # if gold is not enabled as the default linker for this target.
1139 gold_supported = CONFIG_TARGET_SUFFIXES['binutils'].get(target) == '-gold'
1140 if not gold_supported:
1141 srcpath = srcpath[:-len('-gold')]
1142 ld_path = os.path.join(srcpath, 'ld')
1143 assert os.path.exists(ld_path), '%s: linker is missing!' % ld_path
1144
Mike Frysinger78b7a812014-11-26 19:45:23 -05001145 srcpath = srcpath[len(output_dir):]
Mike Frysinger35247af2012-11-16 18:58:06 -05001146 gccpath = os.path.join('/usr', 'libexec', 'gcc')
1147 for prog in os.listdir(output_dir + srcpath):
1148 # Skip binaries already wrapped.
1149 if not prog.endswith('.real'):
1150 GeneratePathWrapper(output_dir, binpath + prog,
1151 os.path.join(srcpath, prog))
1152 GeneratePathWrapper(output_dir, os.path.join(gccpath, prog),
1153 os.path.join(srcpath, prog))
1154
David James27ac4ae2012-12-03 23:16:15 -08001155 libpath = os.path.join('/usr', toolchain.GetHostTuple(), target, 'lib')
Mike Frysingerd4d40fd2014-11-06 17:30:57 -05001156 envd = os.path.join(output_dir, 'etc', 'env.d', 'binutils', '*')
1157 if gold_supported:
1158 envd += '-gold'
Rahul Chaudhry4891b4d2017-03-08 10:31:27 -08001159 else:
1160 # If gold is not enabled as the default linker and 2 env.d
1161 # files exist, pick the one without the '-gold' suffix.
1162 envds = sorted(glob.glob(envd))
1163 if len(envds) == 2 and envds[1] == envds[0] + '-gold':
1164 envd = envds[0]
Mike Frysinger35247af2012-11-16 18:58:06 -05001165 srcpath = _EnvdGetVar(envd, 'LIBPATH')
1166 os.symlink(os.path.relpath(srcpath, os.path.dirname(libpath)),
1167 output_dir + libpath)
1168
1169
1170def _ProcessGccConfig(target, output_dir):
1171 """Do what gcc-config would have done"""
1172 binpath = '/bin'
1173 envd = os.path.join(output_dir, 'etc', 'env.d', 'gcc', '*')
1174 srcpath = _EnvdGetVar(envd, 'GCC_PATH')
1175 for prog in os.listdir(output_dir + srcpath):
1176 # Skip binaries already wrapped.
1177 if (not prog.endswith('.real') and
1178 not prog.endswith('.elf') and
1179 prog.startswith(target)):
1180 GeneratePathWrapper(output_dir, os.path.join(binpath, prog),
1181 os.path.join(srcpath, prog))
1182 return srcpath
1183
1184
Frank Henigman179ec7c2015-02-06 03:01:09 -05001185def _ProcessSysrootWrappers(_target, output_dir, srcpath):
1186 """Remove chroot-specific things from our sysroot wrappers"""
Mike Frysinger35247af2012-11-16 18:58:06 -05001187 # Disable ccache since we know it won't work outside of chroot.
Tobias Boschddd16492019-08-14 09:29:54 -07001188
Tobias Boschddd16492019-08-14 09:29:54 -07001189 # Use the version of the wrapper that does not use ccache.
Frank Henigman179ec7c2015-02-06 03:01:09 -05001190 for sysroot_wrapper in glob.glob(os.path.join(
Tobias Boschddd16492019-08-14 09:29:54 -07001191 output_dir + srcpath, 'sysroot_wrapper*.ccache')):
1192 # Can't update the wrapper in place to not affect the chroot,
1193 # but only the extracted toolchain.
1194 os.unlink(sysroot_wrapper)
1195 shutil.copy(sysroot_wrapper[:-6] + 'noccache', sysroot_wrapper)
Tobias Bosch7bc907a2019-10-09 11:52:40 -07001196 shutil.copy(sysroot_wrapper[:-6] + 'noccache.elf', sysroot_wrapper + '.elf')
Mike Frysinger35247af2012-11-16 18:58:06 -05001197
1198
Yunlian Jiang5ad6b512017-09-20 09:27:45 -07001199def _CreateMainLibDir(target, output_dir):
1200 """Create some lib dirs so that compiler can get the right Gcc paths"""
1201 osutils.SafeMakedirs(os.path.join(output_dir, 'usr', target, 'lib'))
1202 osutils.SafeMakedirs(os.path.join(output_dir, 'usr', target, 'usr/lib'))
1203
1204
Mike Frysinger35247af2012-11-16 18:58:06 -05001205def _ProcessDistroCleanups(target, output_dir):
1206 """Clean up the tree and remove all distro-specific requirements
1207
1208 Args:
1209 target: The toolchain target name
1210 output_dir: The output directory to clean up
1211 """
1212 _ProcessBinutilsConfig(target, output_dir)
1213 gcc_path = _ProcessGccConfig(target, output_dir)
Frank Henigman179ec7c2015-02-06 03:01:09 -05001214 _ProcessSysrootWrappers(target, output_dir, gcc_path)
Yunlian Jiang5ad6b512017-09-20 09:27:45 -07001215 _CreateMainLibDir(target, output_dir)
Mike Frysinger35247af2012-11-16 18:58:06 -05001216
1217 osutils.RmDir(os.path.join(output_dir, 'etc'))
1218
1219
1220def CreatePackagableRoot(target, output_dir, ldpaths, root='/'):
1221 """Setup a tree from the packages for the specified target
1222
1223 This populates a path with all the files from toolchain packages so that
1224 a tarball can easily be generated from the result.
1225
1226 Args:
1227 target: The target to create a packagable root from
1228 output_dir: The output directory to place all the files
1229 ldpaths: A dict of static ldpath information
1230 root: The root path to pull all packages/files from
1231 """
1232 # Find all the files owned by the packages for this target.
1233 paths, elfs = _GetFilesForTarget(target, root=root)
1234
1235 # Link in all the package's files, any ELF dependencies, and wrap any
1236 # executable ELFs with helper scripts.
1237 def MoveUsrBinToBin(path):
Han Shen699ea192016-03-02 10:42:47 -08001238 """Move /usr/bin to /bin so people can just use that toplevel dir
1239
George Burgess IVca1d7612020-10-01 00:38:32 -07001240 Note we do not apply this to clang or rust - there is correlation between
1241 clang's search path for libraries / inclusion and its installation path.
Han Shen699ea192016-03-02 10:42:47 -08001242 """
George Burgess IVca1d7612020-10-01 00:38:32 -07001243 if (path.startswith('/usr/bin/') and
1244 not any(x in path for x in ('clang', 'rust', 'cargo'))):
Han Shen699ea192016-03-02 10:42:47 -08001245 return path[4:]
1246 return path
Mike Frysinger35247af2012-11-16 18:58:06 -05001247 _BuildInitialPackageRoot(output_dir, paths, elfs, ldpaths,
1248 path_rewrite_func=MoveUsrBinToBin, root=root)
1249
1250 # The packages, when part of the normal distro, have helper scripts
1251 # that setup paths and such. Since we are making this standalone, we
1252 # need to preprocess all that ourselves.
1253 _ProcessDistroCleanups(target, output_dir)
1254
1255
1256def CreatePackages(targets_wanted, output_dir, root='/'):
1257 """Create redistributable cross-compiler packages for the specified targets
1258
1259 This creates toolchain packages that should be usable in conjunction with
1260 a downloaded sysroot (created elsewhere).
1261
1262 Tarballs (one per target) will be created in $PWD.
1263
1264 Args:
Don Garrett25f309a2014-03-19 14:02:12 -07001265 targets_wanted: The targets to package up.
1266 output_dir: The directory to put the packages in.
1267 root: The root path to pull all packages/files from.
Mike Frysinger35247af2012-11-16 18:58:06 -05001268 """
Ralph Nathan03047282015-03-23 11:09:32 -07001269 logging.info('Writing tarballs to %s', output_dir)
Mike Frysinger35247af2012-11-16 18:58:06 -05001270 osutils.SafeMakedirs(output_dir)
1271 ldpaths = lddtree.LoadLdpaths(root)
1272 targets = ExpandTargets(targets_wanted)
1273
Mike Frysinger221bd822017-09-29 02:51:47 -04001274 with osutils.TempDir(prefix='create-packages') as tempdir:
1275 logging.debug('Using tempdir: %s', tempdir)
1276
Mike Frysinger35247af2012-11-16 18:58:06 -05001277 # We have to split the root generation from the compression stages. This is
1278 # because we hardlink in all the files (to avoid overhead of reading/writing
1279 # the copies multiple times). But tar gets angry if a file's hardlink count
1280 # changes from when it starts reading a file to when it finishes.
1281 with parallel.BackgroundTaskRunner(CreatePackagableRoot) as queue:
1282 for target in targets:
1283 output_target_dir = os.path.join(tempdir, target)
1284 queue.put([target, output_target_dir, ldpaths, root])
1285
1286 # Build the tarball.
1287 with parallel.BackgroundTaskRunner(cros_build_lib.CreateTarball) as queue:
1288 for target in targets:
1289 tar_file = os.path.join(output_dir, target + '.tar.xz')
1290 queue.put([tar_file, os.path.join(tempdir, target)])
1291
1292
Mike Frysinger07534cf2017-09-12 17:40:21 -04001293def GetParser():
1294 """Return a command line parser."""
Mike Frysinger0c808452014-11-06 17:30:23 -05001295 parser = commandline.ArgumentParser(description=__doc__)
1296 parser.add_argument('-u', '--nousepkg',
1297 action='store_false', dest='usepkg', default=True,
1298 help='Use prebuilt packages if possible')
1299 parser.add_argument('-d', '--deleteold',
1300 action='store_true', dest='deleteold', default=False,
1301 help='Unmerge deprecated packages')
1302 parser.add_argument('-t', '--targets',
1303 dest='targets', default='sdk',
Mike Frysinger80de5012019-08-01 14:10:53 -04001304 help='Comma separated list of tuples. Special keywords '
Don Garrettc0c74002015-10-09 12:58:19 -07001305 "'host', 'sdk', 'boards', and 'all' are "
Gilad Arnold8195b532015-04-07 10:56:30 +03001306 "allowed. Defaults to 'sdk'.")
1307 parser.add_argument('--include-boards', default='', metavar='BOARDS',
1308 help='Comma separated list of boards whose toolchains we '
1309 'will always include. Default: none')
Mike Frysinger0c808452014-11-06 17:30:23 -05001310 parser.add_argument('--hostonly',
1311 dest='hostonly', default=False, action='store_true',
1312 help='Only setup the host toolchain. '
1313 'Useful for bootstrapping chroot')
Bertrand SIMONNETcae9d5f2015-03-09 15:58:01 -07001314 parser.add_argument('--show-board-cfg', '--show-cfg',
1315 dest='cfg_name', default=None,
Don Garrettc0c74002015-10-09 12:58:19 -07001316 help='Board to list toolchains tuples for')
Mike Frysinger91f52882017-09-13 00:16:58 -04001317 parser.add_argument('--show-packages', default=None,
1318 help='List all packages the specified target uses')
Mike Frysinger0c808452014-11-06 17:30:23 -05001319 parser.add_argument('--create-packages',
1320 action='store_true', default=False,
1321 help='Build redistributable packages')
1322 parser.add_argument('--output-dir', default=os.getcwd(), type='path',
1323 help='Output directory')
1324 parser.add_argument('--reconfig', default=False, action='store_true',
1325 help='Reload crossdev config and reselect toolchains')
Gilad Arnold2dab78c2015-05-21 14:43:33 -07001326 parser.add_argument('--sysroot', type='path',
1327 help='The sysroot in which to install the toolchains')
Mike Frysinger07534cf2017-09-12 17:40:21 -04001328 return parser
Zdenek Behan508dcce2011-12-05 15:39:32 +01001329
Mike Frysinger07534cf2017-09-12 17:40:21 -04001330
1331def main(argv):
1332 parser = GetParser()
Mike Frysinger0c808452014-11-06 17:30:23 -05001333 options = parser.parse_args(argv)
1334 options.Freeze()
Zdenek Behan508dcce2011-12-05 15:39:32 +01001335
Mike Frysinger35247af2012-11-16 18:58:06 -05001336 # Figure out what we're supposed to do and reject conflicting options.
Mike Frysinger91f52882017-09-13 00:16:58 -04001337 conflicting_options = (
1338 options.cfg_name,
1339 options.show_packages,
1340 options.create_packages,
1341 )
1342 if sum(bool(x) for x in conflicting_options) > 1:
1343 parser.error('conflicting options: create-packages & show-packages & '
1344 'show-board-cfg')
Mike Frysinger984d0622012-06-01 16:08:44 -04001345
Gilad Arnold8195b532015-04-07 10:56:30 +03001346 targets_wanted = set(options.targets.split(','))
1347 boards_wanted = (set(options.include_boards.split(','))
1348 if options.include_boards else set())
Mike Frysinger35247af2012-11-16 18:58:06 -05001349
Bertrand SIMONNETcae9d5f2015-03-09 15:58:01 -07001350 if options.cfg_name:
1351 ShowConfig(options.cfg_name)
Mike Frysinger91f52882017-09-13 00:16:58 -04001352 elif options.show_packages is not None:
1353 cros_build_lib.AssertInsideChroot()
1354 target = options.show_packages
1355 Crossdev.Load(False)
1356 for package in GetTargetPackages(target):
1357 print(GetPortagePackage(target, package))
Mike Frysinger35247af2012-11-16 18:58:06 -05001358 elif options.create_packages:
1359 cros_build_lib.AssertInsideChroot()
1360 Crossdev.Load(False)
Gilad Arnold8195b532015-04-07 10:56:30 +03001361 CreatePackages(targets_wanted, options.output_dir)
Mike Frysinger35247af2012-11-16 18:58:06 -05001362 else:
1363 cros_build_lib.AssertInsideChroot()
1364 # This has to be always run as root.
1365 if os.geteuid() != 0:
1366 cros_build_lib.Die('this script must be run as root')
1367
1368 Crossdev.Load(options.reconfig)
Gilad Arnold2dab78c2015-05-21 14:43:33 -07001369 root = options.sysroot or '/'
Mike Frysinger35247af2012-11-16 18:58:06 -05001370 UpdateToolchains(options.usepkg, options.deleteold, options.hostonly,
Gilad Arnold8195b532015-04-07 10:56:30 +03001371 options.reconfig, targets_wanted, boards_wanted,
Don Garrettc0c74002015-10-09 12:58:19 -07001372 root=root)
Mike Frysinger35247af2012-11-16 18:58:06 -05001373 Crossdev.Save()
1374
1375 return 0