blob: 7262b5a3bed20baeb4156aec18b40f16080556e8 [file] [log] [blame]
Mike Frysingere58c0e22017-10-04 15:43:30 -04001# -*- coding: utf-8 -*-
Zdenek Behan508dcce2011-12-05 15:39:32 +01002# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
Mike Frysinger750c5f52014-09-16 16:16:57 -04006"""This script manages the installed toolchains in the chroot."""
Zdenek Behan508dcce2011-12-05 15:39:32 +01007
Mike Frysinger383367e2014-09-16 15:06:17 -04008from __future__ import print_function
9
Mike Frysinger3ed47722017-08-08 14:59:08 -040010import errno
Mike Frysinger35247af2012-11-16 18:58:06 -050011import glob
Mike Frysinger3ed47722017-08-08 14:59:08 -040012import hashlib
Mike Frysinger7ccee992012-06-01 21:27:59 -040013import json
Zdenek Behan508dcce2011-12-05 15:39:32 +010014import os
Rahul Chaudhrya8127bb2016-07-22 15:53:17 -070015import re
Tobias Boschddd16492019-08-14 09:29:54 -070016import shutil
Zdenek Behan508dcce2011-12-05 15:39:32 +010017
Aviv Keshetb7519e12016-10-04 00:50:00 -070018from chromite.lib import constants
Mike Frysinger506e75f2012-12-17 14:21:13 -050019from chromite.lib import commandline
Brian Harring503f3ab2012-03-09 21:39:41 -080020from chromite.lib import cros_build_lib
Ralph Nathan03047282015-03-23 11:09:32 -070021from chromite.lib import cros_logging as logging
Brian Harringaf019fb2012-05-10 15:06:13 -070022from chromite.lib import osutils
Mike Frysinger35247af2012-11-16 18:58:06 -050023from chromite.lib import parallel
David James27ac4ae2012-12-03 23:16:15 -080024from chromite.lib import toolchain
Mike Frysinger35247af2012-11-16 18:58:06 -050025
26# Needs to be after chromite imports.
27import lddtree
Zdenek Behan508dcce2011-12-05 15:39:32 +010028
Mike Frysinger31596002012-12-03 23:54:24 -050029if cros_build_lib.IsInsideChroot():
30 # Only import portage after we've checked that we're inside the chroot.
31 # Outside may not have portage, in which case the above may not happen.
32 # We'll check in main() if the operation needs portage.
Mike Frysinger27e21b72018-07-12 14:20:21 -040033 # pylint: disable=import-error
Mike Frysinger31596002012-12-03 23:54:24 -050034 import portage
Zdenek Behan508dcce2011-12-05 15:39:32 +010035
36
Matt Tennantf1e30972012-03-02 16:30:07 -080037EMERGE_CMD = os.path.join(constants.CHROMITE_BIN_DIR, 'parallel_emerge')
Zdenek Behan508dcce2011-12-05 15:39:32 +010038PACKAGE_STABLE = '[stable]'
Zdenek Behan4eb6fd22012-03-12 17:00:56 +010039
40CHROMIUMOS_OVERLAY = '/usr/local/portage/chromiumos'
Christopher Wileyb22c0712015-06-02 10:37:03 -070041ECLASS_OVERLAY = '/usr/local/portage/eclass-overlay'
Zdenek Behan4eb6fd22012-03-12 17:00:56 +010042STABLE_OVERLAY = '/usr/local/portage/stable'
43CROSSDEV_OVERLAY = '/usr/local/portage/crossdev'
Zdenek Behan508dcce2011-12-05 15:39:32 +010044
45
Mike Frysinger66bfde52017-09-12 16:42:57 -040046# The exact list of host toolchain packages we care about. These are the
47# packages that bots/devs install only from binpkgs and rely on the SDK bot
48# (chromiumos-sdk) to validate+uprev.
49#
Mike Frysinger66bfde52017-09-12 16:42:57 -040050# We don't use crossdev to manage the host toolchain for us, especially since
51# we diverge significantly now (with llvm/clang/etc...), and we don't need or
52# want crossdev managing /etc/portage config files for the sdk
53HOST_PACKAGES = (
54 'dev-lang/go',
Yunlian Jiang2cb91dc2018-03-08 10:56:27 -080055 'dev-libs/elfutils',
Mike Frysinger66bfde52017-09-12 16:42:57 -040056 'sys-devel/binutils',
57 'sys-devel/clang',
58 'sys-devel/gcc',
Yunlian Jiangf5721f32017-10-31 11:43:11 -070059 'sys-devel/lld',
Mike Frysinger66bfde52017-09-12 16:42:57 -040060 'sys-devel/llvm',
61 'sys-kernel/linux-headers',
62 'sys-libs/glibc',
63 'sys-libs/libcxx',
64 'sys-libs/libcxxabi',
Manoj Guptade64cb22019-03-31 18:48:58 -070065 'sys-libs/llvm-libunwind',
Mike Frysinger66bfde52017-09-12 16:42:57 -040066)
67
Mike Frysinger785b0c32017-09-13 01:35:59 -040068# These packages are also installed into the host SDK. However, they require
69# the cross-compilers to be installed first (because they need them to actually
70# build), so we have to delay their installation.
71HOST_POST_CROSS_PACKAGES = (
Manoj Gupta65f88442018-04-12 22:42:19 -070072 'dev-lang/rust',
73 'dev-util/cargo',
Mike Frysinger61a24392017-10-17 17:14:27 -040074 'virtual/target-sdk-post-cross',
Patrick Georgi043ce6e2019-02-20 22:27:09 +010075 'dev-embedded/coreboot-sdk',
Mike Frysinger785b0c32017-09-13 01:35:59 -040076)
77
78# New packages that we're in the process of adding to the SDK. Since the SDK
79# bot hasn't had a chance to run yet, there are no binary packages available,
80# so we have to list them here and wait. Once it completes, entries here can
81# be removed so they'll end up on bots & dev's systems.
82NEW_PACKAGES = (
Mike Frysinger785b0c32017-09-13 01:35:59 -040083)
84
Rahul Chaudhry4b803052015-05-13 15:25:56 -070085# Enable the Go compiler for these targets.
86TARGET_GO_ENABLED = (
87 'x86_64-cros-linux-gnu',
Rahul Chaudhry4b803052015-05-13 15:25:56 -070088 'armv7a-cros-linux-gnueabi',
Yunlian Jiang16c1a422017-10-04 10:33:22 -070089 'armv7a-cros-linux-gnueabihf',
Rahul Chaudhry4d416582017-10-25 12:31:58 -070090 'aarch64-cros-linux-gnu',
Rahul Chaudhry4b803052015-05-13 15:25:56 -070091)
92CROSSDEV_GO_ARGS = ['--ex-pkg', 'dev-lang/go']
93
Manoj Gupta1b5642e2017-03-08 16:44:12 -080094# Enable llvm's compiler-rt for these targets.
95TARGET_COMPILER_RT_ENABLED = (
96 'armv7a-cros-linux-gnueabi',
Yunlian Jiang1b77ee42017-10-06 13:44:29 -070097 'armv7a-cros-linux-gnueabihf',
Manoj Gupta946abb42017-04-12 14:27:19 -070098 'aarch64-cros-linux-gnu',
Manoj Gupta21f3a082018-03-06 21:25:39 -080099 'armv7m-cros-eabi',
Manoj Gupta1b5642e2017-03-08 16:44:12 -0800100)
101CROSSDEV_COMPILER_RT_ARGS = ['--ex-pkg', 'sys-libs/compiler-rt']
102
Manoj Gupta946abb42017-04-12 14:27:19 -0700103TARGET_LLVM_PKGS_ENABLED = (
104 'armv7a-cros-linux-gnueabi',
Yunlian Jiang16c1a422017-10-04 10:33:22 -0700105 'armv7a-cros-linux-gnueabihf',
Manoj Gupta946abb42017-04-12 14:27:19 -0700106 'aarch64-cros-linux-gnu',
107 'x86_64-cros-linux-gnu',
108)
109
110LLVM_PKGS_TABLE = {
Yunlian Jiangbb2a3d32017-04-26 14:44:09 -0700111 'ex_libcxxabi' : ['--ex-pkg', 'sys-libs/libcxxabi'],
112 'ex_libcxx' : ['--ex-pkg', 'sys-libs/libcxx'],
Chirantan Ekbotee694cad2018-07-12 15:07:24 -0700113 'ex_llvm-libunwind' : ['--ex-pkg', 'sys-libs/llvm-libunwind'],
Manoj Gupta946abb42017-04-12 14:27:19 -0700114}
115
Zdenek Behan508dcce2011-12-05 15:39:32 +0100116# Overrides for {gcc,binutils}-config, pick a package with particular suffix.
117CONFIG_TARGET_SUFFIXES = {
Mike Frysingerd6e2df02014-11-26 02:55:04 -0500118 'binutils' : {
Manoj Guptaa91e38e2018-11-15 11:07:48 -0800119 'aarch64-cros-linux-gnu' : '-gold',
Mike Frysinger8a83c622015-05-28 00:35:05 -0400120 'armv6j-cros-linux-gnueabi': '-gold',
Han Shen43b84422015-02-19 11:38:13 -0800121 'armv7a-cros-linux-gnueabi': '-gold',
Yunlian Jiang16c1a422017-10-04 10:33:22 -0700122 'armv7a-cros-linux-gnueabihf': '-gold',
Mike Frysingerd6e2df02014-11-26 02:55:04 -0500123 'i686-pc-linux-gnu' : '-gold',
124 'x86_64-cros-linux-gnu' : '-gold',
125 },
Zdenek Behan508dcce2011-12-05 15:39:32 +0100126}
Zdenek Behan508dcce2011-12-05 15:39:32 +0100127
128
David James66a09c42012-11-05 13:31:38 -0800129class Crossdev(object):
130 """Class for interacting with crossdev and caching its output."""
131
132 _CACHE_FILE = os.path.join(CROSSDEV_OVERLAY, '.configured.json')
133 _CACHE = {}
Han Shene23782f2016-02-18 12:20:00 -0800134 # Packages that needs separate handling, in addition to what we have from
135 # crossdev.
136 MANUAL_PKGS = {
Manoj Guptaf0a602e2017-08-02 10:25:45 -0700137 'clang': 'sys-devel',
Yunlian Jiang18ae9982017-11-03 09:15:31 -0700138 'lld': 'sys-devel',
Han Shene23782f2016-02-18 12:20:00 -0800139 'llvm': 'sys-devel',
Manoj Gupta20cfc6c2017-07-19 15:49:55 -0700140 'libcxxabi': 'sys-libs',
141 'libcxx': 'sys-libs',
Yunlian Jiangda3ce5f2018-04-25 14:10:01 -0700142 'elfutils': 'dev-libs',
Chirantan Ekbotee694cad2018-07-12 15:07:24 -0700143 'llvm-libunwind': 'sys-libs',
Han Shene23782f2016-02-18 12:20:00 -0800144 }
David James66a09c42012-11-05 13:31:38 -0800145
146 @classmethod
147 def Load(cls, reconfig):
Mike Frysinger3ed47722017-08-08 14:59:08 -0400148 """Load crossdev cache from disk.
149
150 We invalidate the cache when crossdev updates or this script changes.
151 """
David James90239b92012-11-05 15:31:34 -0800152 crossdev_version = GetStablePackageVersion('sys-devel/crossdev', True)
Mike Frysinger3ed47722017-08-08 14:59:08 -0400153 # If we run the compiled/cached .pyc file, we'll read/hash that when we
154 # really always want to track the source .py file.
155 script = os.path.abspath(__file__)
156 if script.endswith('.pyc'):
157 script = script[:-1]
158 setup_toolchains_hash = hashlib.md5(osutils.ReadFile(script)).hexdigest()
159
160 cls._CACHE = {
161 'crossdev_version': crossdev_version,
162 'setup_toolchains_hash': setup_toolchains_hash,
163 }
164
165 logging.debug('cache: checking file: %s', cls._CACHE_FILE)
166 if reconfig:
167 logging.debug('cache: forcing regen due to reconfig')
168 return
169
170 try:
171 file_data = osutils.ReadFile(cls._CACHE_FILE)
172 except IOError as e:
173 if e.errno != errno.ENOENT:
174 logging.warning('cache: reading failed: %s', e)
175 osutils.SafeUnlink(cls._CACHE_FILE)
176 return
177
178 try:
179 data = json.loads(file_data)
180 except ValueError as e:
181 logging.warning('cache: ignoring invalid content: %s', e)
182 return
183
184 if crossdev_version != data.get('crossdev_version'):
185 logging.debug('cache: rebuilding after crossdev upgrade')
186 elif setup_toolchains_hash != data.get('setup_toolchains_hash'):
187 logging.debug('cache: rebuilding after cros_setup_toolchains change')
188 else:
189 logging.debug('cache: content is up-to-date!')
190 cls._CACHE = data
David James66a09c42012-11-05 13:31:38 -0800191
192 @classmethod
193 def Save(cls):
194 """Store crossdev cache on disk."""
195 # Save the cache from the successful run.
196 with open(cls._CACHE_FILE, 'w') as f:
197 json.dump(cls._CACHE, f)
198
199 @classmethod
200 def GetConfig(cls, target):
201 """Returns a map of crossdev provided variables about a tuple."""
202 CACHE_ATTR = '_target_tuple_map'
203
204 val = cls._CACHE.setdefault(CACHE_ATTR, {})
205 if not target in val:
Mike Frysinger785b0c32017-09-13 01:35:59 -0400206 if target.startswith('host'):
Mike Frysinger66bfde52017-09-12 16:42:57 -0400207 conf = {
208 'crosspkgs': [],
209 'target': toolchain.GetHostTuple(),
210 }
Mike Frysinger785b0c32017-09-13 01:35:59 -0400211 if target == 'host':
212 packages_list = HOST_PACKAGES
213 else:
214 packages_list = HOST_POST_CROSS_PACKAGES
Mike Frysinger66bfde52017-09-12 16:42:57 -0400215 manual_pkgs = dict((pkg, cat) for cat, pkg in
Mike Frysinger785b0c32017-09-13 01:35:59 -0400216 [x.split('/') for x in packages_list])
Mike Frysinger66bfde52017-09-12 16:42:57 -0400217 else:
218 # Build the crossdev command.
219 cmd = ['crossdev', '--show-target-cfg', '--ex-gdb']
220 if target in TARGET_COMPILER_RT_ENABLED:
221 cmd.extend(CROSSDEV_COMPILER_RT_ARGS)
222 if target in TARGET_GO_ENABLED:
223 cmd.extend(CROSSDEV_GO_ARGS)
224 if target in TARGET_LLVM_PKGS_ENABLED:
225 for pkg in LLVM_PKGS_TABLE:
226 cmd.extend(LLVM_PKGS_TABLE[pkg])
227 cmd.extend(['-t', target])
228 # Catch output of crossdev.
229 out = cros_build_lib.RunCommand(
230 cmd, print_cmd=False, redirect_stdout=True).output.splitlines()
231 # List of tuples split at the first '=', converted into dict.
232 conf = dict((k, cros_build_lib.ShellUnquote(v))
233 for k, v in (x.split('=', 1) for x in out))
234 conf['crosspkgs'] = conf['crosspkgs'].split()
Han Shene23782f2016-02-18 12:20:00 -0800235
Mike Frysinger66bfde52017-09-12 16:42:57 -0400236 manual_pkgs = cls.MANUAL_PKGS
237
238 for pkg, cat in manual_pkgs.items():
Mike Frysinger3bba5032016-09-20 14:15:04 -0400239 conf[pkg + '_pn'] = pkg
240 conf[pkg + '_category'] = cat
241 if pkg not in conf['crosspkgs']:
242 conf['crosspkgs'].append(pkg)
Han Shene23782f2016-02-18 12:20:00 -0800243
244 val[target] = conf
245
David James66a09c42012-11-05 13:31:38 -0800246 return val[target]
247
248 @classmethod
249 def UpdateTargets(cls, targets, usepkg, config_only=False):
250 """Calls crossdev to initialize a cross target.
251
252 Args:
Don Garrett25f309a2014-03-19 14:02:12 -0700253 targets: The list of targets to initialize using crossdev.
254 usepkg: Copies the commandline opts.
255 config_only: Just update.
David James66a09c42012-11-05 13:31:38 -0800256 """
257 configured_targets = cls._CACHE.setdefault('configured_targets', [])
258
259 cmdbase = ['crossdev', '--show-fail-log']
260 cmdbase.extend(['--env', 'FEATURES=splitdebug'])
261 # Pick stable by default, and override as necessary.
262 cmdbase.extend(['-P', '--oneshot'])
263 if usepkg:
264 cmdbase.extend(['-P', '--getbinpkg',
265 '-P', '--usepkgonly',
266 '--without-headers'])
267
Christopher Wileyb22c0712015-06-02 10:37:03 -0700268 overlays = ' '.join((CHROMIUMOS_OVERLAY, ECLASS_OVERLAY, STABLE_OVERLAY))
David James66a09c42012-11-05 13:31:38 -0800269 cmdbase.extend(['--overlays', overlays])
270 cmdbase.extend(['--ov-output', CROSSDEV_OVERLAY])
271
Yunlian Jiang4ff84172018-05-12 13:03:01 -0700272 # Build target by the reversed alphabetical order to make sure
273 # armv7a-cros-linux-gnueabihf builds before armv7a-cros-linux-gnueabi
Yunlian Jiang85c606a2017-10-10 20:58:53 -0700274 # because some dependency issue. This can be reverted once we
275 # migrated to armv7a-cros-linux-gnueabihf. crbug.com/711369
Yunlian Jiang4ff84172018-05-12 13:03:01 -0700276 for target in sorted(targets, reverse=True):
David James66a09c42012-11-05 13:31:38 -0800277 if config_only and target in configured_targets:
278 continue
279
280 cmd = cmdbase + ['-t', target]
281
282 for pkg in GetTargetPackages(target):
283 if pkg == 'gdb':
284 # Gdb does not have selectable versions.
285 cmd.append('--ex-gdb')
Manoj Guptab8181562017-05-21 10:18:59 -0700286 elif pkg == 'ex_compiler-rt':
287 cmd.extend(CROSSDEV_COMPILER_RT_ARGS)
Rahul Chaudhry4b803052015-05-13 15:25:56 -0700288 elif pkg == 'ex_go':
289 # Go does not have selectable versions.
290 cmd.extend(CROSSDEV_GO_ARGS)
Manoj Gupta946abb42017-04-12 14:27:19 -0700291 elif pkg in LLVM_PKGS_TABLE:
292 cmd.extend(LLVM_PKGS_TABLE[pkg])
Han Shene23782f2016-02-18 12:20:00 -0800293 elif pkg in cls.MANUAL_PKGS:
294 pass
Rahul Chaudhry4b803052015-05-13 15:25:56 -0700295 else:
296 # The first of the desired versions is the "primary" one.
297 version = GetDesiredPackageVersions(target, pkg)[0]
298 cmd.extend(['--%s' % pkg, version])
David James66a09c42012-11-05 13:31:38 -0800299
300 cmd.extend(targets[target]['crossdev'].split())
301 if config_only:
302 # In this case we want to just quietly reinit
303 cmd.append('--init-target')
304 cros_build_lib.RunCommand(cmd, print_cmd=False, redirect_stdout=True)
305 else:
306 cros_build_lib.RunCommand(cmd)
307
308 configured_targets.append(target)
309
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100310
Zdenek Behanf4d18a02012-03-22 15:45:05 +0100311def GetTargetPackages(target):
312 """Returns a list of packages for a given target."""
David James66a09c42012-11-05 13:31:38 -0800313 conf = Crossdev.GetConfig(target)
Zdenek Behanf4d18a02012-03-22 15:45:05 +0100314 # Undesired packages are denoted by empty ${pkg}_pn variable.
Han Shene23782f2016-02-18 12:20:00 -0800315 return [x for x in conf['crosspkgs'] if conf.get(x+'_pn')]
Zdenek Behanf4d18a02012-03-22 15:45:05 +0100316
317
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100318# Portage helper functions:
319def GetPortagePackage(target, package):
320 """Returns a package name for the given target."""
David James66a09c42012-11-05 13:31:38 -0800321 conf = Crossdev.GetConfig(target)
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100322 # Portage category:
Mike Frysinger785b0c32017-09-13 01:35:59 -0400323 if target.startswith('host') or package in Crossdev.MANUAL_PKGS:
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100324 category = conf[package + '_category']
Zdenek Behan508dcce2011-12-05 15:39:32 +0100325 else:
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100326 category = conf['category']
327 # Portage package:
328 pn = conf[package + '_pn']
329 # Final package name:
Mike Frysinger422faf42014-11-12 23:16:48 -0500330 assert category
331 assert pn
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100332 return '%s/%s' % (category, pn)
333
334
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700335def PortageTrees(root):
336 """Return the portage trees for a given root."""
337 if root == '/':
338 return portage.db['/']
339 # The portage logic requires the path always end in a slash.
340 root = root.rstrip('/') + '/'
341 return portage.create_trees(target_root=root, config_root=root)[root]
342
343
344def GetInstalledPackageVersions(atom, root='/'):
Zdenek Behan508dcce2011-12-05 15:39:32 +0100345 """Extracts the list of current versions of a target, package pair.
346
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500347 Args:
348 atom: The atom to operate on (e.g. sys-devel/gcc)
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700349 root: The root to check for installed packages.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100350
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500351 Returns:
352 The list of versions of the package currently installed.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100353 """
Zdenek Behan508dcce2011-12-05 15:39:32 +0100354 versions = []
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700355 for pkg in PortageTrees(root)['vartree'].dbapi.match(atom, use_cache=0):
Zdenek Behan508dcce2011-12-05 15:39:32 +0100356 version = portage.versions.cpv_getversion(pkg)
357 versions.append(version)
358 return versions
359
360
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700361def GetStablePackageVersion(atom, installed, root='/'):
Zdenek Behan508dcce2011-12-05 15:39:32 +0100362 """Extracts the current stable version for a given package.
363
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500364 Args:
365 atom: The target/package to operate on eg. i686-pc-linux-gnu,gcc
366 installed: Whether we want installed packages or ebuilds
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700367 root: The root to use when querying packages.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100368
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500369 Returns:
370 A string containing the latest version.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100371 """
David James90239b92012-11-05 15:31:34 -0800372 pkgtype = 'vartree' if installed else 'porttree'
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700373 cpv = portage.best(PortageTrees(root)[pkgtype].dbapi.match(atom, use_cache=0))
David James90239b92012-11-05 15:31:34 -0800374 return portage.versions.cpv_getversion(cpv) if cpv else None
Zdenek Behan508dcce2011-12-05 15:39:32 +0100375
376
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700377def VersionListToNumeric(target, package, versions, installed, root='/'):
Zdenek Behan508dcce2011-12-05 15:39:32 +0100378 """Resolves keywords in a given version list for a particular package.
379
380 Resolving means replacing PACKAGE_STABLE with the actual number.
381
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500382 Args:
383 target: The target to operate on (e.g. i686-pc-linux-gnu)
384 package: The target/package to operate on (e.g. gcc)
385 versions: List of versions to resolve
386 installed: Query installed packages
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700387 root: The install root to use; ignored if |installed| is False.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100388
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500389 Returns:
390 List of purely numeric versions equivalent to argument
Zdenek Behan508dcce2011-12-05 15:39:32 +0100391 """
392 resolved = []
David James90239b92012-11-05 15:31:34 -0800393 atom = GetPortagePackage(target, package)
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700394 if not installed:
395 root = '/'
Zdenek Behan508dcce2011-12-05 15:39:32 +0100396 for version in versions:
397 if version == PACKAGE_STABLE:
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700398 resolved.append(GetStablePackageVersion(atom, installed, root=root))
Mike Frysinger7f0e1982017-09-12 17:08:50 -0400399 else:
Zdenek Behan508dcce2011-12-05 15:39:32 +0100400 resolved.append(version)
401 return resolved
402
403
404def GetDesiredPackageVersions(target, package):
405 """Produces the list of desired versions for each target, package pair.
406
407 The first version in the list is implicitly treated as primary, ie.
408 the version that will be initialized by crossdev and selected.
409
410 If the version is PACKAGE_STABLE, it really means the current version which
411 is emerged by using the package atom with no particular version key.
412 Since crossdev unmasks all packages by default, this will actually
413 mean 'unstable' in most cases.
414
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500415 Args:
416 target: The target to operate on (e.g. i686-pc-linux-gnu)
417 package: The target/package to operate on (e.g. gcc)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100418
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500419 Returns:
420 A list composed of either a version string, PACKAGE_STABLE
Zdenek Behan508dcce2011-12-05 15:39:32 +0100421 """
Mike Frysingerd23be272017-09-12 17:18:44 -0400422 if package in GetTargetPackages(target):
423 return [PACKAGE_STABLE]
424 else:
425 return []
Zdenek Behan508dcce2011-12-05 15:39:32 +0100426
427
428def TargetIsInitialized(target):
429 """Verifies if the given list of targets has been correctly initialized.
430
431 This determines whether we have to call crossdev while emerging
432 toolchain packages or can do it using emerge. Emerge is naturally
433 preferred, because all packages can be updated in a single pass.
434
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500435 Args:
436 target: The target to operate on (e.g. i686-pc-linux-gnu)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100437
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500438 Returns:
439 True if |target| is completely initialized, else False
Zdenek Behan508dcce2011-12-05 15:39:32 +0100440 """
441 # Check if packages for the given target all have a proper version.
442 try:
Zdenek Behanf4d18a02012-03-22 15:45:05 +0100443 for package in GetTargetPackages(target):
David James66a09c42012-11-05 13:31:38 -0800444 atom = GetPortagePackage(target, package)
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100445 # Do we even want this package && is it initialized?
Mike Frysinger7f0e1982017-09-12 17:08:50 -0400446 if not (GetStablePackageVersion(atom, True) and
447 GetStablePackageVersion(atom, False)):
Zdenek Behan508dcce2011-12-05 15:39:32 +0100448 return False
449 return True
450 except cros_build_lib.RunCommandError:
451 # Fails - The target has likely never been initialized before.
452 return False
453
454
455def RemovePackageMask(target):
456 """Removes a package.mask file for the given platform.
457
458 The pre-existing package.mask files can mess with the keywords.
459
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500460 Args:
461 target: The target to operate on (e.g. i686-pc-linux-gnu)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100462 """
463 maskfile = os.path.join('/etc/portage/package.mask', 'cross-' + target)
Brian Harringaf019fb2012-05-10 15:06:13 -0700464 osutils.SafeUnlink(maskfile)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100465
466
Zdenek Behan508dcce2011-12-05 15:39:32 +0100467# Main functions performing the actual update steps.
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700468def RebuildLibtool(root='/'):
Mike Frysingerc880a962013-11-08 13:59:06 -0500469 """Rebuild libtool as needed
470
471 Libtool hardcodes full paths to internal gcc files, so whenever we upgrade
472 gcc, libtool will break. We can't use binary packages either as those will
473 most likely be compiled against the previous version of gcc.
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700474
475 Args:
476 root: The install root where we want libtool rebuilt.
Mike Frysingerc880a962013-11-08 13:59:06 -0500477 """
478 needs_update = False
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700479 with open(os.path.join(root, 'usr/bin/libtool')) as f:
Mike Frysingerc880a962013-11-08 13:59:06 -0500480 for line in f:
481 # Look for a line like:
482 # sys_lib_search_path_spec="..."
483 # It'll be a list of paths and gcc will be one of them.
484 if line.startswith('sys_lib_search_path_spec='):
485 line = line.rstrip()
486 for path in line.split('=', 1)[1].strip('"').split():
Mike Frysinger3bba5032016-09-20 14:15:04 -0400487 root_path = os.path.join(root, path.lstrip(os.path.sep))
488 logging.debug('Libtool: checking %s', root_path)
489 if not os.path.exists(root_path):
490 logging.info('Rebuilding libtool after gcc upgrade')
491 logging.info(' %s', line)
492 logging.info(' missing path: %s', path)
Mike Frysingerc880a962013-11-08 13:59:06 -0500493 needs_update = True
494 break
495
496 if needs_update:
497 break
498
499 if needs_update:
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700500 cmd = [EMERGE_CMD, '--oneshot']
501 if root != '/':
502 cmd.extend(['--sysroot=%s' % root, '--root=%s' % root])
503 cmd.append('sys-devel/libtool')
Mike Frysingerc880a962013-11-08 13:59:06 -0500504 cros_build_lib.RunCommand(cmd)
Mike Frysinger3bba5032016-09-20 14:15:04 -0400505 else:
506 logging.debug('Libtool is up-to-date; no need to rebuild')
Mike Frysingerc880a962013-11-08 13:59:06 -0500507
508
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700509def UpdateTargets(targets, usepkg, root='/'):
Zdenek Behan508dcce2011-12-05 15:39:32 +0100510 """Determines which packages need update/unmerge and defers to portage.
511
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500512 Args:
513 targets: The list of targets to update
514 usepkg: Copies the commandline option
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700515 root: The install root in which we want packages updated.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100516 """
Zdenek Behan508dcce2011-12-05 15:39:32 +0100517 # For each target, we do two things. Figure out the list of updates,
518 # and figure out the appropriate keywords/masks. Crossdev will initialize
519 # these, but they need to be regenerated on every update.
Mike Frysinger3bba5032016-09-20 14:15:04 -0400520 logging.info('Determining required toolchain updates...')
David James90239b92012-11-05 15:31:34 -0800521 mergemap = {}
Zdenek Behan508dcce2011-12-05 15:39:32 +0100522 for target in targets:
Mike Frysinger3bba5032016-09-20 14:15:04 -0400523 logging.debug('Updating target %s', target)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100524 # Record the highest needed version for each target, for masking purposes.
525 RemovePackageMask(target)
Zdenek Behanf4d18a02012-03-22 15:45:05 +0100526 for package in GetTargetPackages(target):
Zdenek Behan508dcce2011-12-05 15:39:32 +0100527 # Portage name for the package
Mike Frysinger7f0e1982017-09-12 17:08:50 -0400528 logging.debug(' Checking package %s', package)
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100529 pkg = GetPortagePackage(target, package)
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700530 current = GetInstalledPackageVersions(pkg, root=root)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100531 desired = GetDesiredPackageVersions(target, package)
Zdenek Behan699ddd32012-04-13 07:14:08 +0200532 desired_num = VersionListToNumeric(target, package, desired, False)
Mike Frysinger785b0c32017-09-13 01:35:59 -0400533 if pkg in NEW_PACKAGES and usepkg:
534 # Skip this binary package (for now).
535 continue
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100536 mergemap[pkg] = set(desired_num).difference(current)
Mike Frysinger7f0e1982017-09-12 17:08:50 -0400537 logging.debug(' %s -> %s', current, desired_num)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100538
Mike Frysinger7f0e1982017-09-12 17:08:50 -0400539 packages = [pkg for pkg, vers in mergemap.items() if vers]
Zdenek Behan508dcce2011-12-05 15:39:32 +0100540 if not packages:
Mike Frysinger3bba5032016-09-20 14:15:04 -0400541 logging.info('Nothing to update!')
David Jamesf8c672f2012-11-06 13:38:11 -0800542 return False
Zdenek Behan508dcce2011-12-05 15:39:32 +0100543
Mike Frysinger3bba5032016-09-20 14:15:04 -0400544 logging.info('Updating packages:')
545 logging.info('%s', packages)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100546
Zdenek Behan7e33b4e2012-03-12 17:00:56 +0100547 cmd = [EMERGE_CMD, '--oneshot', '--update']
Zdenek Behan508dcce2011-12-05 15:39:32 +0100548 if usepkg:
549 cmd.extend(['--getbinpkg', '--usepkgonly'])
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700550 if root != '/':
551 cmd.extend(['--sysroot=%s' % root, '--root=%s' % root])
Zdenek Behan508dcce2011-12-05 15:39:32 +0100552
553 cmd.extend(packages)
554 cros_build_lib.RunCommand(cmd)
David Jamesf8c672f2012-11-06 13:38:11 -0800555 return True
Zdenek Behan508dcce2011-12-05 15:39:32 +0100556
557
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700558def CleanTargets(targets, root='/'):
559 """Unmerges old packages that are assumed unnecessary.
560
561 Args:
562 targets: The list of targets to clean up.
563 root: The install root in which we want packages cleaned up.
564 """
Zdenek Behan508dcce2011-12-05 15:39:32 +0100565 unmergemap = {}
566 for target in targets:
Mike Frysinger3bba5032016-09-20 14:15:04 -0400567 logging.debug('Cleaning target %s', target)
Zdenek Behanf4d18a02012-03-22 15:45:05 +0100568 for package in GetTargetPackages(target):
Mike Frysinger3bba5032016-09-20 14:15:04 -0400569 logging.debug(' Cleaning package %s', package)
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100570 pkg = GetPortagePackage(target, package)
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700571 current = GetInstalledPackageVersions(pkg, root=root)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100572 desired = GetDesiredPackageVersions(target, package)
Gilad Arnold2fcb0aa2015-05-21 14:51:41 -0700573 # NOTE: This refers to installed packages (vartree) rather than the
574 # Portage version (porttree and/or bintree) when determining the current
575 # version. While this isn't the most accurate thing to do, it is probably
576 # a good simple compromise, which should have the desired result of
577 # uninstalling everything but the latest installed version. In
578 # particular, using the bintree (--usebinpkg) requires a non-trivial
579 # binhost sync and is probably more complex than useful.
Zdenek Behan699ddd32012-04-13 07:14:08 +0200580 desired_num = VersionListToNumeric(target, package, desired, True)
581 if not set(desired_num).issubset(current):
Mike Frysinger3bba5032016-09-20 14:15:04 -0400582 logging.warning('Error detecting stable version for %s, '
583 'skipping clean!', pkg)
Zdenek Behan699ddd32012-04-13 07:14:08 +0200584 return
Zdenek Behan508dcce2011-12-05 15:39:32 +0100585 unmergemap[pkg] = set(current).difference(desired_num)
586
587 # Cleaning doesn't care about consistency and rebuilding package.* files.
588 packages = []
Mike Frysinger0bdbc102019-06-13 15:27:29 -0400589 for pkg, vers in unmergemap.items():
Zdenek Behan508dcce2011-12-05 15:39:32 +0100590 packages.extend('=%s-%s' % (pkg, ver) for ver in vers if ver != '9999')
591
592 if packages:
Mike Frysinger3bba5032016-09-20 14:15:04 -0400593 logging.info('Cleaning packages:')
594 logging.info('%s', packages)
Zdenek Behan7e33b4e2012-03-12 17:00:56 +0100595 cmd = [EMERGE_CMD, '--unmerge']
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700596 if root != '/':
597 cmd.extend(['--sysroot=%s' % root, '--root=%s' % root])
Zdenek Behan508dcce2011-12-05 15:39:32 +0100598 cmd.extend(packages)
599 cros_build_lib.RunCommand(cmd)
600 else:
Mike Frysinger3bba5032016-09-20 14:15:04 -0400601 logging.info('Nothing to clean!')
Zdenek Behan508dcce2011-12-05 15:39:32 +0100602
603
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700604def SelectActiveToolchains(targets, suffixes, root='/'):
Zdenek Behan508dcce2011-12-05 15:39:32 +0100605 """Runs gcc-config and binutils-config to select the desired.
606
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500607 Args:
608 targets: The targets to select
609 suffixes: Optional target-specific hacks
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700610 root: The root where we want to select toolchain versions.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100611 """
612 for package in ['gcc', 'binutils']:
613 for target in targets:
Mike Frysinger785b0c32017-09-13 01:35:59 -0400614 # See if this package is part of this target.
615 if package not in GetTargetPackages(target):
616 logging.debug('%s: %s is not used', target, package)
617 continue
618
Zdenek Behan508dcce2011-12-05 15:39:32 +0100619 # Pick the first version in the numbered list as the selected one.
620 desired = GetDesiredPackageVersions(target, package)
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700621 desired_num = VersionListToNumeric(target, package, desired, True,
622 root=root)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100623 desired = desired_num[0]
624 # *-config does not play revisions, strip them, keep just PV.
625 desired = portage.versions.pkgsplit('%s-%s' % (package, desired))[1]
626
Mike Frysinger785b0c32017-09-13 01:35:59 -0400627 if target.startswith('host'):
Zdenek Behan508dcce2011-12-05 15:39:32 +0100628 # *-config is the only tool treating host identically (by tuple).
David James27ac4ae2012-12-03 23:16:15 -0800629 target = toolchain.GetHostTuple()
Zdenek Behan508dcce2011-12-05 15:39:32 +0100630
631 # And finally, attach target to it.
632 desired = '%s-%s' % (target, desired)
633
634 # Target specific hacks
635 if package in suffixes:
636 if target in suffixes[package]:
637 desired += suffixes[package][target]
638
David James7ec5efc2012-11-06 09:39:49 -0800639 extra_env = {'CHOST': target}
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700640 if root != '/':
641 extra_env['ROOT'] = root
David James7ec5efc2012-11-06 09:39:49 -0800642 cmd = ['%s-config' % package, '-c', target]
Mike Frysingerd6e2df02014-11-26 02:55:04 -0500643 result = cros_build_lib.RunCommand(
644 cmd, print_cmd=False, redirect_stdout=True, extra_env=extra_env)
645 current = result.output.splitlines()[0]
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700646
647 # Do not reconfig when the current is live or nothing needs to be done.
648 extra_env = {'ROOT': root} if root != '/' else None
Zdenek Behan508dcce2011-12-05 15:39:32 +0100649 if current != desired and current != '9999':
Mike Frysingerd6e2df02014-11-26 02:55:04 -0500650 cmd = [package + '-config', desired]
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700651 cros_build_lib.RunCommand(cmd, print_cmd=False, extra_env=extra_env)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100652
653
Mike Frysinger35247af2012-11-16 18:58:06 -0500654def ExpandTargets(targets_wanted):
655 """Expand any possible toolchain aliases into full targets
656
657 This will expand 'all' and 'sdk' into the respective toolchain tuples.
658
659 Args:
660 targets_wanted: The targets specified by the user.
Mike Frysinger1a736a82013-12-12 01:50:59 -0500661
Mike Frysinger35247af2012-11-16 18:58:06 -0500662 Returns:
Gilad Arnold8195b532015-04-07 10:56:30 +0300663 Dictionary of concrete targets and their toolchain tuples.
Mike Frysinger35247af2012-11-16 18:58:06 -0500664 """
Mike Frysinger35247af2012-11-16 18:58:06 -0500665 targets_wanted = set(targets_wanted)
Don Garrettc0c74002015-10-09 12:58:19 -0700666 if targets_wanted == set(['boards']):
667 # Only pull targets from the included boards.
Gilad Arnold8195b532015-04-07 10:56:30 +0300668 return {}
669
670 all_targets = toolchain.GetAllTargets()
Mike Frysinger35247af2012-11-16 18:58:06 -0500671 if targets_wanted == set(['all']):
Gilad Arnold8195b532015-04-07 10:56:30 +0300672 return all_targets
673 if targets_wanted == set(['sdk']):
Mike Frysinger35247af2012-11-16 18:58:06 -0500674 # Filter out all the non-sdk toolchains as we don't want to mess
675 # with those in all of our builds.
Gilad Arnold8195b532015-04-07 10:56:30 +0300676 return toolchain.FilterToolchains(all_targets, 'sdk', True)
677
678 # Verify user input.
679 nonexistent = targets_wanted.difference(all_targets)
680 if nonexistent:
681 raise ValueError('Invalid targets: %s', ','.join(nonexistent))
682 return {t: all_targets[t] for t in targets_wanted}
Mike Frysinger35247af2012-11-16 18:58:06 -0500683
684
David Jamesf8c672f2012-11-06 13:38:11 -0800685def UpdateToolchains(usepkg, deleteold, hostonly, reconfig,
Don Garrettc0c74002015-10-09 12:58:19 -0700686 targets_wanted, boards_wanted, root='/'):
Zdenek Behan508dcce2011-12-05 15:39:32 +0100687 """Performs all steps to create a synchronized toolchain enviroment.
688
Mike Frysingerdc649ae2014-11-26 19:29:24 -0500689 Args:
690 usepkg: Use prebuilt packages
691 deleteold: Unmerge deprecated packages
692 hostonly: Only setup the host toolchain
693 reconfig: Reload crossdev config and reselect toolchains
694 targets_wanted: All the targets to update
695 boards_wanted: Load targets from these boards
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700696 root: The root in which to install the toolchains.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100697 """
David Jamesf8c672f2012-11-06 13:38:11 -0800698 targets, crossdev_targets, reconfig_targets = {}, {}, {}
Zdenek Behan7e33b4e2012-03-12 17:00:56 +0100699 if not hostonly:
700 # For hostonly, we can skip most of the below logic, much of which won't
701 # work on bare systems where this is useful.
Mike Frysinger35247af2012-11-16 18:58:06 -0500702 targets = ExpandTargets(targets_wanted)
Mike Frysinger7ccee992012-06-01 21:27:59 -0400703
Don Garrettc0c74002015-10-09 12:58:19 -0700704 # Now re-add any targets that might be from this board. This is to
Gilad Arnold8195b532015-04-07 10:56:30 +0300705 # allow unofficial boards to declare their own toolchains.
Mike Frysinger7ccee992012-06-01 21:27:59 -0400706 for board in boards_wanted:
David James27ac4ae2012-12-03 23:16:15 -0800707 targets.update(toolchain.GetToolchainsForBoard(board))
Zdenek Behan508dcce2011-12-05 15:39:32 +0100708
Zdenek Behan7e33b4e2012-03-12 17:00:56 +0100709 # First check and initialize all cross targets that need to be.
Mike Frysinger7ccee992012-06-01 21:27:59 -0400710 for target in targets:
711 if TargetIsInitialized(target):
712 reconfig_targets[target] = targets[target]
713 else:
714 crossdev_targets[target] = targets[target]
Zdenek Behan7e33b4e2012-03-12 17:00:56 +0100715 if crossdev_targets:
Mike Frysinger3bba5032016-09-20 14:15:04 -0400716 logging.info('The following targets need to be re-initialized:')
717 logging.info('%s', crossdev_targets)
David James66a09c42012-11-05 13:31:38 -0800718 Crossdev.UpdateTargets(crossdev_targets, usepkg)
Zdenek Behan8be29ba2012-05-29 23:10:34 +0200719 # Those that were not initialized may need a config update.
David James66a09c42012-11-05 13:31:38 -0800720 Crossdev.UpdateTargets(reconfig_targets, usepkg, config_only=True)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100721
Mike Frysinger66814c32017-10-09 18:11:46 -0400722 # If we're building a subset of toolchains for a board, we might not have
723 # all the tuples that the packages expect. We don't define the "full" set
724 # of tuples currently other than "whatever the full sdk has normally".
725 if usepkg or set(('all', 'sdk')) & targets_wanted:
726 # Since we have cross-compilers now, we can update these packages.
727 targets['host-post-cross'] = {}
Mike Frysinger785b0c32017-09-13 01:35:59 -0400728
Zdenek Behan7e33b4e2012-03-12 17:00:56 +0100729 # We want host updated.
Mike Frysinger7ccee992012-06-01 21:27:59 -0400730 targets['host'] = {}
Zdenek Behan7e33b4e2012-03-12 17:00:56 +0100731
732 # Now update all packages.
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700733 if UpdateTargets(targets, usepkg, root=root) or crossdev_targets or reconfig:
734 SelectActiveToolchains(targets, CONFIG_TARGET_SUFFIXES, root=root)
David James7ec5efc2012-11-06 09:39:49 -0800735
736 if deleteold:
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700737 CleanTargets(targets, root=root)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100738
Mike Frysingerc880a962013-11-08 13:59:06 -0500739 # Now that we've cleared out old versions, see if we need to rebuild
740 # anything. Can't do this earlier as it might not be broken.
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700741 RebuildLibtool(root=root)
Mike Frysingerc880a962013-11-08 13:59:06 -0500742
Zdenek Behan508dcce2011-12-05 15:39:32 +0100743
Bertrand SIMONNETcae9d5f2015-03-09 15:58:01 -0700744def ShowConfig(name):
745 """Show the toolchain tuples used by |name|
Mike Frysinger35247af2012-11-16 18:58:06 -0500746
747 Args:
Don Garrettc0c74002015-10-09 12:58:19 -0700748 name: The board name to query.
Mike Frysinger35247af2012-11-16 18:58:06 -0500749 """
Don Garrettc0c74002015-10-09 12:58:19 -0700750 toolchains = toolchain.GetToolchainsForBoard(name)
Mike Frysinger35247af2012-11-16 18:58:06 -0500751 # Make sure we display the default toolchain first.
Mike Frysinger3bba5032016-09-20 14:15:04 -0400752 # Note: Do not use logging here as this is meant to be used by other tools.
Mike Frysinger383367e2014-09-16 15:06:17 -0400753 print(','.join(
David James27ac4ae2012-12-03 23:16:15 -0800754 toolchain.FilterToolchains(toolchains, 'default', True).keys() +
Mike Frysinger383367e2014-09-16 15:06:17 -0400755 toolchain.FilterToolchains(toolchains, 'default', False).keys()))
Mike Frysinger35247af2012-11-16 18:58:06 -0500756
757
Mike Frysinger35247af2012-11-16 18:58:06 -0500758def GeneratePathWrapper(root, wrappath, path):
759 """Generate a shell script to execute another shell script
760
761 Since we can't symlink a wrapped ELF (see GenerateLdsoWrapper) because the
762 argv[0] won't be pointing to the correct path, generate a shell script that
763 just executes another program with its full path.
764
765 Args:
766 root: The root tree to generate scripts inside of
767 wrappath: The full path (inside |root|) to create the wrapper
768 path: The target program which this wrapper will execute
769 """
770 replacements = {
771 'path': path,
772 'relroot': os.path.relpath('/', os.path.dirname(wrappath)),
773 }
Takuto Ikuta58403972018-08-16 18:52:51 +0900774
775 # Do not use exec here, because exec invokes script with absolute path in
776 # argv0. Keeping relativeness allows us to remove abs path from compile result
777 # and leads directory independent build cache sharing in some distributed
778 # build system.
Mike Frysinger35247af2012-11-16 18:58:06 -0500779 wrapper = """#!/bin/sh
Takuto Ikuta58403972018-08-16 18:52:51 +0900780basedir=$(dirname "$0")
781"${basedir}/%(relroot)s%(path)s" "$@"
782exit "$?"
Mike Frysinger35247af2012-11-16 18:58:06 -0500783""" % replacements
784 root_wrapper = root + wrappath
785 if os.path.islink(root_wrapper):
786 os.unlink(root_wrapper)
787 else:
788 osutils.SafeMakedirs(os.path.dirname(root_wrapper))
789 osutils.WriteFile(root_wrapper, wrapper)
Mike Frysinger60ec1012013-10-21 00:11:10 -0400790 os.chmod(root_wrapper, 0o755)
Mike Frysinger35247af2012-11-16 18:58:06 -0500791
792
Rahul Chaudhrya8127bb2016-07-22 15:53:17 -0700793def FixClangXXWrapper(root, path):
794 """Fix wrapper shell scripts and symlinks for invoking clang++
795
796 In a typical installation, clang++ symlinks to clang, which symlinks to the
797 elf executable. The executable distinguishes between clang and clang++ based
798 on argv[0].
799
800 When invoked through the LdsoWrapper, argv[0] always contains the path to the
801 executable elf file, making clang/clang++ invocations indistinguishable.
802
803 This function detects if the elf executable being wrapped is clang-X.Y, and
804 fixes wrappers/symlinks as necessary so that clang++ will work correctly.
805
806 The calling sequence now becomes:
807 -) clang++ invocation turns into clang++-3.9 (which is a copy of clang-3.9,
808 the Ldsowrapper).
809 -) clang++-3.9 uses the Ldso to invoke clang++-3.9.elf, which is a symlink
810 to the original clang-3.9 elf.
811 -) The difference this time is that inside the elf file execution, $0 is
812 set as .../usr/bin/clang++-3.9.elf, which contains 'clang++' in the name.
813
Manoj Guptaae268142018-04-27 23:28:36 -0700814 Update: Starting since clang 7, the clang and clang++ are symlinks to
815 clang-7 binary, not clang-7.0. The pattern match is extended to handle
816 both clang-7 and clang-7.0 cases for now. (https://crbug.com/837889)
817
Rahul Chaudhrya8127bb2016-07-22 15:53:17 -0700818 Args:
819 root: The root tree to generate scripts / symlinks inside of
820 path: The target elf for which LdsoWrapper was created
821 """
Manoj Guptaae268142018-04-27 23:28:36 -0700822 if re.match(r'/usr/bin/clang-\d+(\.\d+)*$', path):
Rahul Chaudhrya8127bb2016-07-22 15:53:17 -0700823 logging.info('fixing clang++ invocation for %s', path)
824 clangdir = os.path.dirname(root + path)
825 clang = os.path.basename(path)
826 clangxx = clang.replace('clang', 'clang++')
827
828 # Create a symlink clang++-X.Y.elf to point to clang-X.Y.elf
829 os.symlink(clang + '.elf', os.path.join(clangdir, clangxx + '.elf'))
830
831 # Create a hardlink clang++-X.Y pointing to clang-X.Y
832 os.link(os.path.join(clangdir, clang), os.path.join(clangdir, clangxx))
833
834 # Adjust the clang++ symlink to point to clang++-X.Y
835 os.unlink(os.path.join(clangdir, 'clang++'))
836 os.symlink(clangxx, os.path.join(clangdir, 'clang++'))
837
838
Mike Frysinger35247af2012-11-16 18:58:06 -0500839def FileIsCrosSdkElf(elf):
840 """Determine if |elf| is an ELF that we execute in the cros_sdk
841
842 We don't need this to be perfect, just quick. It makes sure the ELF
843 is a 64bit LSB x86_64 ELF. That is the native type of cros_sdk.
844
845 Args:
846 elf: The file to check
Mike Frysinger1a736a82013-12-12 01:50:59 -0500847
Mike Frysinger35247af2012-11-16 18:58:06 -0500848 Returns:
849 True if we think |elf| is a native ELF
850 """
851 with open(elf) as f:
852 data = f.read(20)
853 # Check the magic number, EI_CLASS, EI_DATA, and e_machine.
854 return (data[0:4] == '\x7fELF' and
855 data[4] == '\x02' and
856 data[5] == '\x01' and
857 data[18] == '\x3e')
858
859
860def IsPathPackagable(ptype, path):
861 """Should the specified file be included in a toolchain package?
862
863 We only need to handle files as we'll create dirs as we need them.
864
865 Further, trim files that won't be useful:
866 - non-english translations (.mo) since it'd require env vars
867 - debug files since these are for the host compiler itself
868 - info/man pages as they're big, and docs are online, and the
869 native docs should work fine for the most part (`man gcc`)
870
871 Args:
872 ptype: A string describing the path type (i.e. 'file' or 'dir' or 'sym')
873 path: The full path to inspect
Mike Frysinger1a736a82013-12-12 01:50:59 -0500874
Mike Frysinger35247af2012-11-16 18:58:06 -0500875 Returns:
876 True if we want to include this path in the package
877 """
878 return not (ptype in ('dir',) or
879 path.startswith('/usr/lib/debug/') or
880 os.path.splitext(path)[1] == '.mo' or
881 ('/man/' in path or '/info/' in path))
882
883
884def ReadlinkRoot(path, root):
885 """Like os.readlink(), but relative to a |root|
886
887 Args:
888 path: The symlink to read
889 root: The path to use for resolving absolute symlinks
Mike Frysinger1a736a82013-12-12 01:50:59 -0500890
Mike Frysinger35247af2012-11-16 18:58:06 -0500891 Returns:
892 A fully resolved symlink path
893 """
894 while os.path.islink(root + path):
895 path = os.path.join(os.path.dirname(path), os.readlink(root + path))
896 return path
897
898
899def _GetFilesForTarget(target, root='/'):
900 """Locate all the files to package for |target|
901
902 This does not cover ELF dependencies.
903
904 Args:
905 target: The toolchain target name
906 root: The root path to pull all packages from
Mike Frysinger1a736a82013-12-12 01:50:59 -0500907
Mike Frysinger35247af2012-11-16 18:58:06 -0500908 Returns:
909 A tuple of a set of all packable paths, and a set of all paths which
910 are also native ELFs
911 """
912 paths = set()
913 elfs = set()
914
915 # Find all the files owned by the packages for this target.
916 for pkg in GetTargetPackages(target):
Mike Frysinger35247af2012-11-16 18:58:06 -0500917
Rahul Chaudhry4b803052015-05-13 15:25:56 -0700918 # Skip Go compiler from redistributable packages.
919 # The "go" executable has GOROOT=/usr/lib/go/${CTARGET} hardcoded
920 # into it. Due to this, the toolchain cannot be unpacked anywhere
921 # else and be readily useful. To enable packaging Go, we need to:
922 # -) Tweak the wrappers/environment to override GOROOT
923 # automatically based on the unpack location.
924 # -) Make sure the ELF dependency checking and wrapping logic
925 # below skips the Go toolchain executables and libraries.
926 # -) Make sure the packaging process maintains the relative
927 # timestamps of precompiled standard library packages.
928 # (see dev-lang/go ebuild for details).
929 if pkg == 'ex_go':
930 continue
931
Yunlian Jiang36f35242018-04-27 10:18:40 -0700932 # Use armv7a-cros-linux-gnueabi/compiler-rt for
933 # armv7a-cros-linux-gnueabihf/compiler-rt.
934 # Currently the armv7a-cros-linux-gnueabi is actually
935 # the same as armv7a-cros-linux-gnueabihf with different names.
936 # Because of that, for compiler-rt, it generates the same binary in
937 # the same location. To avoid the installation conflict, we do not
938 # install anything for 'armv7a-cros-linux-gnueabihf'. This would cause
939 # problem if other people try to use standalone armv7a-cros-linux-gnueabihf
940 # toolchain.
941 if 'compiler-rt' in pkg and 'armv7a-cros-linux-gnueabi' in target:
942 atom = GetPortagePackage(target, pkg)
943 cat, pn = atom.split('/')
944 ver = GetInstalledPackageVersions(atom, root=root)[0]
Yunlian Jiang36f35242018-04-27 10:18:40 -0700945 dblink = portage.dblink(cat, pn + '-' + ver, myroot=root,
946 settings=portage.settings)
947 contents = dblink.getcontents()
948 if not contents:
949 if 'hf' in target:
950 new_target = 'armv7a-cros-linux-gnueabi'
951 else:
952 new_target = 'armv7a-cros-linux-gnueabihf'
953 atom = GetPortagePackage(new_target, pkg)
954 else:
955 atom = GetPortagePackage(target, pkg)
956
Mike Frysinger35247af2012-11-16 18:58:06 -0500957 cat, pn = atom.split('/')
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700958 ver = GetInstalledPackageVersions(atom, root=root)[0]
Ralph Nathan03047282015-03-23 11:09:32 -0700959 logging.info('packaging %s-%s', atom, ver)
Mike Frysinger35247af2012-11-16 18:58:06 -0500960
Mike Frysinger35247af2012-11-16 18:58:06 -0500961 dblink = portage.dblink(cat, pn + '-' + ver, myroot=root,
962 settings=portage.settings)
963 contents = dblink.getcontents()
964 for obj in contents:
965 ptype = contents[obj][0]
966 if not IsPathPackagable(ptype, obj):
967 continue
968
969 if ptype == 'obj':
970 # For native ELFs, we need to pull in their dependencies too.
971 if FileIsCrosSdkElf(obj):
972 elfs.add(obj)
973 paths.add(obj)
974
975 return paths, elfs
976
977
978def _BuildInitialPackageRoot(output_dir, paths, elfs, ldpaths,
Mike Frysingerd6e2df02014-11-26 02:55:04 -0500979 path_rewrite_func=lambda x: x, root='/'):
Mike Frysinger35247af2012-11-16 18:58:06 -0500980 """Link in all packable files and their runtime dependencies
981
982 This also wraps up executable ELFs with helper scripts.
983
984 Args:
985 output_dir: The output directory to store files
986 paths: All the files to include
987 elfs: All the files which are ELFs (a subset of |paths|)
988 ldpaths: A dict of static ldpath information
989 path_rewrite_func: User callback to rewrite paths in output_dir
990 root: The root path to pull all packages/files from
991 """
992 # Link in all the files.
Mike Frysinger221bd822017-09-29 02:51:47 -0400993 sym_paths = {}
Mike Frysinger35247af2012-11-16 18:58:06 -0500994 for path in paths:
995 new_path = path_rewrite_func(path)
996 dst = output_dir + new_path
997 osutils.SafeMakedirs(os.path.dirname(dst))
998
999 # Is this a symlink which we have to rewrite or wrap?
1000 # Delay wrap check until after we have created all paths.
1001 src = root + path
1002 if os.path.islink(src):
1003 tgt = os.readlink(src)
1004 if os.path.sep in tgt:
Mike Frysinger221bd822017-09-29 02:51:47 -04001005 sym_paths[lddtree.normpath(ReadlinkRoot(src, root))] = new_path
Mike Frysinger35247af2012-11-16 18:58:06 -05001006
1007 # Rewrite absolute links to relative and then generate the symlink
1008 # ourselves. All other symlinks can be hardlinked below.
1009 if tgt[0] == '/':
1010 tgt = os.path.relpath(tgt, os.path.dirname(new_path))
1011 os.symlink(tgt, dst)
1012 continue
1013
1014 os.link(src, dst)
1015
Mike Frysinger35247af2012-11-16 18:58:06 -05001016 # Locate all the dependencies for all the ELFs. Stick them all in the
1017 # top level "lib" dir to make the wrapper simpler. This exact path does
1018 # not matter since we execute ldso directly, and we tell the ldso the
1019 # exact path to search for its libraries.
1020 libdir = os.path.join(output_dir, 'lib')
1021 osutils.SafeMakedirs(libdir)
1022 donelibs = set()
Mike Frysinger4331fc62017-09-28 14:03:25 -04001023 glibc_re = re.compile(r'/lib(c|pthread)-[0-9.]+\.so$')
Mike Frysinger35247af2012-11-16 18:58:06 -05001024 for elf in elfs:
Mike Frysingerea7688e2014-07-31 22:40:33 -04001025 e = lddtree.ParseELF(elf, root=root, ldpaths=ldpaths)
Mike Frysinger35247af2012-11-16 18:58:06 -05001026 interp = e['interp']
Yunlian Jiang196e8f42017-09-20 12:29:47 -07001027 # Do not create wrapper for libc. crbug.com/766827
1028 if interp and not glibc_re.search(elf):
Mike Frysinger35247af2012-11-16 18:58:06 -05001029 # Generate a wrapper if it is executable.
Mike Frysingerc2ec0902013-03-26 01:28:45 -04001030 interp = os.path.join('/lib', os.path.basename(interp))
1031 lddtree.GenerateLdsoWrapper(output_dir, path_rewrite_func(elf), interp,
1032 libpaths=e['rpath'] + e['runpath'])
Rahul Chaudhrya8127bb2016-07-22 15:53:17 -07001033 FixClangXXWrapper(output_dir, path_rewrite_func(elf))
Mike Frysinger35247af2012-11-16 18:58:06 -05001034
Mike Frysinger221bd822017-09-29 02:51:47 -04001035 # Wrap any symlinks to the wrapper.
1036 if elf in sym_paths:
1037 link = sym_paths[elf]
1038 GeneratePathWrapper(output_dir, link, elf)
1039
Mike Frysinger0bdbc102019-06-13 15:27:29 -04001040 for lib, lib_data in e['libs'].items():
Mike Frysinger35247af2012-11-16 18:58:06 -05001041 if lib in donelibs:
1042 continue
1043
1044 src = path = lib_data['path']
1045 if path is None:
Ralph Nathan446aee92015-03-23 14:44:56 -07001046 logging.warning('%s: could not locate %s', elf, lib)
Mike Frysinger35247af2012-11-16 18:58:06 -05001047 continue
1048 donelibs.add(lib)
1049
1050 # Needed libs are the SONAME, but that is usually a symlink, not a
1051 # real file. So link in the target rather than the symlink itself.
1052 # We have to walk all the possible symlinks (SONAME could point to a
1053 # symlink which points to a symlink), and we have to handle absolute
1054 # ourselves (since we have a "root" argument).
1055 dst = os.path.join(libdir, os.path.basename(path))
1056 src = ReadlinkRoot(src, root)
1057
1058 os.link(root + src, dst)
1059
1060
1061def _EnvdGetVar(envd, var):
1062 """Given a Gentoo env.d file, extract a var from it
1063
1064 Args:
1065 envd: The env.d file to load (may be a glob path)
1066 var: The var to extract
Mike Frysinger1a736a82013-12-12 01:50:59 -05001067
Mike Frysinger35247af2012-11-16 18:58:06 -05001068 Returns:
1069 The value of |var|
1070 """
1071 envds = glob.glob(envd)
1072 assert len(envds) == 1, '%s: should have exactly 1 env.d file' % envd
1073 envd = envds[0]
1074 return cros_build_lib.LoadKeyValueFile(envd)[var]
1075
1076
1077def _ProcessBinutilsConfig(target, output_dir):
1078 """Do what binutils-config would have done"""
1079 binpath = os.path.join('/bin', target + '-')
Mike Frysingerd4d40fd2014-11-06 17:30:57 -05001080
1081 # Locate the bin dir holding the gold linker.
1082 binutils_bin_path = os.path.join(output_dir, 'usr', toolchain.GetHostTuple(),
1083 target, 'binutils-bin')
1084 globpath = os.path.join(binutils_bin_path, '*-gold')
Mike Frysinger35247af2012-11-16 18:58:06 -05001085 srcpath = glob.glob(globpath)
Mike Frysingerd4d40fd2014-11-06 17:30:57 -05001086 if not srcpath:
1087 # Maybe this target doesn't support gold.
1088 globpath = os.path.join(binutils_bin_path, '*')
1089 srcpath = glob.glob(globpath)
1090 assert len(srcpath) == 1, ('%s: matched more than one path (but not *-gold)'
1091 % globpath)
1092 srcpath = srcpath[0]
1093 ld_path = os.path.join(srcpath, 'ld')
1094 assert os.path.exists(ld_path), '%s: linker is missing!' % ld_path
1095 ld_path = os.path.join(srcpath, 'ld.bfd')
1096 assert os.path.exists(ld_path), '%s: linker is missing!' % ld_path
1097 ld_path = os.path.join(srcpath, 'ld.gold')
1098 assert not os.path.exists(ld_path), ('%s: exists, but gold dir does not!'
1099 % ld_path)
1100
1101 # Nope, no gold support to be found.
1102 gold_supported = False
Ralph Nathan446aee92015-03-23 14:44:56 -07001103 logging.warning('%s: binutils lacks support for the gold linker', target)
Mike Frysingerd4d40fd2014-11-06 17:30:57 -05001104 else:
1105 assert len(srcpath) == 1, '%s: did not match exactly 1 path' % globpath
Mike Frysinger78b7a812014-11-26 19:45:23 -05001106 srcpath = srcpath[0]
Mike Frysingerd4d40fd2014-11-06 17:30:57 -05001107
Rahul Chaudhry4891b4d2017-03-08 10:31:27 -08001108 # Package the binutils-bin directory without the '-gold' suffix
1109 # if gold is not enabled as the default linker for this target.
1110 gold_supported = CONFIG_TARGET_SUFFIXES['binutils'].get(target) == '-gold'
1111 if not gold_supported:
1112 srcpath = srcpath[:-len('-gold')]
1113 ld_path = os.path.join(srcpath, 'ld')
1114 assert os.path.exists(ld_path), '%s: linker is missing!' % ld_path
1115
Mike Frysinger78b7a812014-11-26 19:45:23 -05001116 srcpath = srcpath[len(output_dir):]
Mike Frysinger35247af2012-11-16 18:58:06 -05001117 gccpath = os.path.join('/usr', 'libexec', 'gcc')
1118 for prog in os.listdir(output_dir + srcpath):
1119 # Skip binaries already wrapped.
1120 if not prog.endswith('.real'):
1121 GeneratePathWrapper(output_dir, binpath + prog,
1122 os.path.join(srcpath, prog))
1123 GeneratePathWrapper(output_dir, os.path.join(gccpath, prog),
1124 os.path.join(srcpath, prog))
1125
David James27ac4ae2012-12-03 23:16:15 -08001126 libpath = os.path.join('/usr', toolchain.GetHostTuple(), target, 'lib')
Mike Frysingerd4d40fd2014-11-06 17:30:57 -05001127 envd = os.path.join(output_dir, 'etc', 'env.d', 'binutils', '*')
1128 if gold_supported:
1129 envd += '-gold'
Rahul Chaudhry4891b4d2017-03-08 10:31:27 -08001130 else:
1131 # If gold is not enabled as the default linker and 2 env.d
1132 # files exist, pick the one without the '-gold' suffix.
1133 envds = sorted(glob.glob(envd))
1134 if len(envds) == 2 and envds[1] == envds[0] + '-gold':
1135 envd = envds[0]
Mike Frysinger35247af2012-11-16 18:58:06 -05001136 srcpath = _EnvdGetVar(envd, 'LIBPATH')
1137 os.symlink(os.path.relpath(srcpath, os.path.dirname(libpath)),
1138 output_dir + libpath)
1139
1140
1141def _ProcessGccConfig(target, output_dir):
1142 """Do what gcc-config would have done"""
1143 binpath = '/bin'
1144 envd = os.path.join(output_dir, 'etc', 'env.d', 'gcc', '*')
1145 srcpath = _EnvdGetVar(envd, 'GCC_PATH')
1146 for prog in os.listdir(output_dir + srcpath):
1147 # Skip binaries already wrapped.
1148 if (not prog.endswith('.real') and
1149 not prog.endswith('.elf') and
1150 prog.startswith(target)):
1151 GeneratePathWrapper(output_dir, os.path.join(binpath, prog),
1152 os.path.join(srcpath, prog))
1153 return srcpath
1154
1155
Frank Henigman179ec7c2015-02-06 03:01:09 -05001156def _ProcessSysrootWrappers(_target, output_dir, srcpath):
1157 """Remove chroot-specific things from our sysroot wrappers"""
Mike Frysinger35247af2012-11-16 18:58:06 -05001158 # Disable ccache since we know it won't work outside of chroot.
Tobias Boschddd16492019-08-14 09:29:54 -07001159
1160 # Update the new go wrapper.
1161 # Use the version of the wrapper that does not use ccache.
Frank Henigman179ec7c2015-02-06 03:01:09 -05001162 for sysroot_wrapper in glob.glob(os.path.join(
Tobias Boschddd16492019-08-14 09:29:54 -07001163 output_dir + srcpath, 'sysroot_wrapper*.ccache')):
1164 # Can't update the wrapper in place to not affect the chroot,
1165 # but only the extracted toolchain.
1166 os.unlink(sysroot_wrapper)
1167 shutil.copy(sysroot_wrapper[:-6] + 'noccache', sysroot_wrapper)
1168
1169 # Update the old python wrapper
1170 # TODO(crbug/773875): Remove this logic once the go wrapper
1171 # is rolled out.
1172 old_wrapper_paths = [os.path.join(output_dir + srcpath,
1173 'sysroot_wrapper'),
1174 os.path.join(output_dir + srcpath,
1175 'sysroot_wrapper.hardened')]
1176 for sysroot_wrapper in old_wrapper_paths:
1177 if not os.path.exists(sysroot_wrapper):
1178 continue
Frank Henigman179ec7c2015-02-06 03:01:09 -05001179 contents = osutils.ReadFile(sysroot_wrapper).splitlines()
Douglas Andersoncc828a52017-10-13 13:07:25 -07001180
1181 # In order to optimize startup time in the chroot we run python a little
1182 # differently there. Put it back to the more portable way here.
Mike Frysingerdcad4e02018-08-03 16:20:02 -04001183 # See https://crbug.com/773138 for some details.
Douglas Andersoncc828a52017-10-13 13:07:25 -07001184 if contents[0] == '#!/usr/bin/python2 -S':
1185 contents[0] = '#!/usr/bin/env python2'
1186
Mike Frysinger79cca962019-06-13 15:26:53 -04001187 for num, line in enumerate(contents):
1188 if '@CCACHE_DEFAULT@' in line:
1189 assert 'True' in line
1190 contents[num] = line.replace('True', 'False')
Frank Henigman179ec7c2015-02-06 03:01:09 -05001191 break
Tobias Boschddd16492019-08-14 09:29:54 -07001192 # Can't update the wrapper in place to not affect the chroot,
1193 # but only the extracted toolchain.
Frank Henigman179ec7c2015-02-06 03:01:09 -05001194 os.unlink(sysroot_wrapper)
1195 osutils.WriteFile(sysroot_wrapper, '\n'.join(contents))
1196 os.chmod(sysroot_wrapper, 0o755)
Mike Frysinger35247af2012-11-16 18:58:06 -05001197
1198
Yunlian Jiang5ad6b512017-09-20 09:27:45 -07001199def _CreateMainLibDir(target, output_dir):
1200 """Create some lib dirs so that compiler can get the right Gcc paths"""
1201 osutils.SafeMakedirs(os.path.join(output_dir, 'usr', target, 'lib'))
1202 osutils.SafeMakedirs(os.path.join(output_dir, 'usr', target, 'usr/lib'))
1203
1204
Mike Frysinger35247af2012-11-16 18:58:06 -05001205def _ProcessDistroCleanups(target, output_dir):
1206 """Clean up the tree and remove all distro-specific requirements
1207
1208 Args:
1209 target: The toolchain target name
1210 output_dir: The output directory to clean up
1211 """
1212 _ProcessBinutilsConfig(target, output_dir)
1213 gcc_path = _ProcessGccConfig(target, output_dir)
Frank Henigman179ec7c2015-02-06 03:01:09 -05001214 _ProcessSysrootWrappers(target, output_dir, gcc_path)
Yunlian Jiang5ad6b512017-09-20 09:27:45 -07001215 _CreateMainLibDir(target, output_dir)
Mike Frysinger35247af2012-11-16 18:58:06 -05001216
1217 osutils.RmDir(os.path.join(output_dir, 'etc'))
1218
1219
1220def CreatePackagableRoot(target, output_dir, ldpaths, root='/'):
1221 """Setup a tree from the packages for the specified target
1222
1223 This populates a path with all the files from toolchain packages so that
1224 a tarball can easily be generated from the result.
1225
1226 Args:
1227 target: The target to create a packagable root from
1228 output_dir: The output directory to place all the files
1229 ldpaths: A dict of static ldpath information
1230 root: The root path to pull all packages/files from
1231 """
1232 # Find all the files owned by the packages for this target.
1233 paths, elfs = _GetFilesForTarget(target, root=root)
1234
1235 # Link in all the package's files, any ELF dependencies, and wrap any
1236 # executable ELFs with helper scripts.
1237 def MoveUsrBinToBin(path):
Han Shen699ea192016-03-02 10:42:47 -08001238 """Move /usr/bin to /bin so people can just use that toplevel dir
1239
1240 Note we do not apply this to clang - there is correlation between clang's
1241 search path for libraries / inclusion and its installation path.
1242 """
1243 if path.startswith('/usr/bin/') and path.find('clang') == -1:
1244 return path[4:]
1245 return path
Mike Frysinger35247af2012-11-16 18:58:06 -05001246 _BuildInitialPackageRoot(output_dir, paths, elfs, ldpaths,
1247 path_rewrite_func=MoveUsrBinToBin, root=root)
1248
1249 # The packages, when part of the normal distro, have helper scripts
1250 # that setup paths and such. Since we are making this standalone, we
1251 # need to preprocess all that ourselves.
1252 _ProcessDistroCleanups(target, output_dir)
1253
1254
1255def CreatePackages(targets_wanted, output_dir, root='/'):
1256 """Create redistributable cross-compiler packages for the specified targets
1257
1258 This creates toolchain packages that should be usable in conjunction with
1259 a downloaded sysroot (created elsewhere).
1260
1261 Tarballs (one per target) will be created in $PWD.
1262
1263 Args:
Don Garrett25f309a2014-03-19 14:02:12 -07001264 targets_wanted: The targets to package up.
1265 output_dir: The directory to put the packages in.
1266 root: The root path to pull all packages/files from.
Mike Frysinger35247af2012-11-16 18:58:06 -05001267 """
Ralph Nathan03047282015-03-23 11:09:32 -07001268 logging.info('Writing tarballs to %s', output_dir)
Mike Frysinger35247af2012-11-16 18:58:06 -05001269 osutils.SafeMakedirs(output_dir)
1270 ldpaths = lddtree.LoadLdpaths(root)
1271 targets = ExpandTargets(targets_wanted)
1272
Mike Frysinger221bd822017-09-29 02:51:47 -04001273 with osutils.TempDir(prefix='create-packages') as tempdir:
1274 logging.debug('Using tempdir: %s', tempdir)
1275
Mike Frysinger35247af2012-11-16 18:58:06 -05001276 # We have to split the root generation from the compression stages. This is
1277 # because we hardlink in all the files (to avoid overhead of reading/writing
1278 # the copies multiple times). But tar gets angry if a file's hardlink count
1279 # changes from when it starts reading a file to when it finishes.
1280 with parallel.BackgroundTaskRunner(CreatePackagableRoot) as queue:
1281 for target in targets:
1282 output_target_dir = os.path.join(tempdir, target)
1283 queue.put([target, output_target_dir, ldpaths, root])
1284
1285 # Build the tarball.
1286 with parallel.BackgroundTaskRunner(cros_build_lib.CreateTarball) as queue:
1287 for target in targets:
1288 tar_file = os.path.join(output_dir, target + '.tar.xz')
1289 queue.put([tar_file, os.path.join(tempdir, target)])
1290
1291
Mike Frysinger07534cf2017-09-12 17:40:21 -04001292def GetParser():
1293 """Return a command line parser."""
Mike Frysinger0c808452014-11-06 17:30:23 -05001294 parser = commandline.ArgumentParser(description=__doc__)
1295 parser.add_argument('-u', '--nousepkg',
1296 action='store_false', dest='usepkg', default=True,
1297 help='Use prebuilt packages if possible')
1298 parser.add_argument('-d', '--deleteold',
1299 action='store_true', dest='deleteold', default=False,
1300 help='Unmerge deprecated packages')
1301 parser.add_argument('-t', '--targets',
1302 dest='targets', default='sdk',
Mike Frysinger80de5012019-08-01 14:10:53 -04001303 help='Comma separated list of tuples. Special keywords '
Don Garrettc0c74002015-10-09 12:58:19 -07001304 "'host', 'sdk', 'boards', and 'all' are "
Gilad Arnold8195b532015-04-07 10:56:30 +03001305 "allowed. Defaults to 'sdk'.")
1306 parser.add_argument('--include-boards', default='', metavar='BOARDS',
1307 help='Comma separated list of boards whose toolchains we '
1308 'will always include. Default: none')
Mike Frysinger0c808452014-11-06 17:30:23 -05001309 parser.add_argument('--hostonly',
1310 dest='hostonly', default=False, action='store_true',
1311 help='Only setup the host toolchain. '
1312 'Useful for bootstrapping chroot')
Bertrand SIMONNETcae9d5f2015-03-09 15:58:01 -07001313 parser.add_argument('--show-board-cfg', '--show-cfg',
1314 dest='cfg_name', default=None,
Don Garrettc0c74002015-10-09 12:58:19 -07001315 help='Board to list toolchains tuples for')
Mike Frysinger91f52882017-09-13 00:16:58 -04001316 parser.add_argument('--show-packages', default=None,
1317 help='List all packages the specified target uses')
Mike Frysinger0c808452014-11-06 17:30:23 -05001318 parser.add_argument('--create-packages',
1319 action='store_true', default=False,
1320 help='Build redistributable packages')
1321 parser.add_argument('--output-dir', default=os.getcwd(), type='path',
1322 help='Output directory')
1323 parser.add_argument('--reconfig', default=False, action='store_true',
1324 help='Reload crossdev config and reselect toolchains')
Gilad Arnold2dab78c2015-05-21 14:43:33 -07001325 parser.add_argument('--sysroot', type='path',
1326 help='The sysroot in which to install the toolchains')
Mike Frysinger07534cf2017-09-12 17:40:21 -04001327 return parser
Zdenek Behan508dcce2011-12-05 15:39:32 +01001328
Mike Frysinger07534cf2017-09-12 17:40:21 -04001329
1330def main(argv):
1331 parser = GetParser()
Mike Frysinger0c808452014-11-06 17:30:23 -05001332 options = parser.parse_args(argv)
1333 options.Freeze()
Zdenek Behan508dcce2011-12-05 15:39:32 +01001334
Mike Frysinger35247af2012-11-16 18:58:06 -05001335 # Figure out what we're supposed to do and reject conflicting options.
Mike Frysinger91f52882017-09-13 00:16:58 -04001336 conflicting_options = (
1337 options.cfg_name,
1338 options.show_packages,
1339 options.create_packages,
1340 )
1341 if sum(bool(x) for x in conflicting_options) > 1:
1342 parser.error('conflicting options: create-packages & show-packages & '
1343 'show-board-cfg')
Mike Frysinger984d0622012-06-01 16:08:44 -04001344
Gilad Arnold8195b532015-04-07 10:56:30 +03001345 targets_wanted = set(options.targets.split(','))
1346 boards_wanted = (set(options.include_boards.split(','))
1347 if options.include_boards else set())
Mike Frysinger35247af2012-11-16 18:58:06 -05001348
Bertrand SIMONNETcae9d5f2015-03-09 15:58:01 -07001349 if options.cfg_name:
1350 ShowConfig(options.cfg_name)
Mike Frysinger91f52882017-09-13 00:16:58 -04001351 elif options.show_packages is not None:
1352 cros_build_lib.AssertInsideChroot()
1353 target = options.show_packages
1354 Crossdev.Load(False)
1355 for package in GetTargetPackages(target):
1356 print(GetPortagePackage(target, package))
Mike Frysinger35247af2012-11-16 18:58:06 -05001357 elif options.create_packages:
1358 cros_build_lib.AssertInsideChroot()
1359 Crossdev.Load(False)
Gilad Arnold8195b532015-04-07 10:56:30 +03001360 CreatePackages(targets_wanted, options.output_dir)
Mike Frysinger35247af2012-11-16 18:58:06 -05001361 else:
1362 cros_build_lib.AssertInsideChroot()
1363 # This has to be always run as root.
1364 if os.geteuid() != 0:
1365 cros_build_lib.Die('this script must be run as root')
1366
1367 Crossdev.Load(options.reconfig)
Gilad Arnold2dab78c2015-05-21 14:43:33 -07001368 root = options.sysroot or '/'
Mike Frysinger35247af2012-11-16 18:58:06 -05001369 UpdateToolchains(options.usepkg, options.deleteold, options.hostonly,
Gilad Arnold8195b532015-04-07 10:56:30 +03001370 options.reconfig, targets_wanted, boards_wanted,
Don Garrettc0c74002015-10-09 12:58:19 -07001371 root=root)
Mike Frysinger35247af2012-11-16 18:58:06 -05001372 Crossdev.Save()
1373
1374 return 0