blob: 749aa5e1d90b7ff05e2b001eb156253c499be5c1 [file] [log] [blame]
Mike Frysingerf1ba7ad2022-09-12 05:42:57 -04001# Copyright 2012 The ChromiumOS Authors
Zdenek Behan508dcce2011-12-05 15:39:32 +01002# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
Mike Frysinger750c5f52014-09-16 16:16:57 -04005"""This script manages the installed toolchains in the chroot."""
Zdenek Behan508dcce2011-12-05 15:39:32 +01006
Mike Frysinger3ed47722017-08-08 14:59:08 -04007import errno
Mike Frysinger35247af2012-11-16 18:58:06 -05008import glob
Mike Frysinger3ed47722017-08-08 14:59:08 -04009import hashlib
Mike Frysinger7ccee992012-06-01 21:27:59 -040010import json
Chris McDonald59650c32021-07-20 15:29:28 -060011import logging
Zdenek Behan508dcce2011-12-05 15:39:32 +010012import os
Rahul Chaudhrya8127bb2016-07-22 15:53:17 -070013import re
Tobias Boschddd16492019-08-14 09:29:54 -070014import shutil
Zdenek Behan508dcce2011-12-05 15:39:32 +010015
Chris McDonald59650c32021-07-20 15:29:28 -060016from chromite.third_party import lddtree
17
Mike Frysinger506e75f2012-12-17 14:21:13 -050018from chromite.lib import commandline
Mike Frysinger95452702021-01-23 00:07:22 -050019from chromite.lib import constants
Brian Harring503f3ab2012-03-09 21:39:41 -080020from chromite.lib import cros_build_lib
Brian Harringaf019fb2012-05-10 15:06:13 -070021from chromite.lib import osutils
Mike Frysinger35247af2012-11-16 18:58:06 -050022from chromite.lib import parallel
David James27ac4ae2012-12-03 23:16:15 -080023from chromite.lib import toolchain
Mike Frysingere652ba12019-09-08 00:57:43 -040024from chromite.utils import key_value_store
Mike Frysinger35247af2012-11-16 18:58:06 -050025
Zdenek Behan508dcce2011-12-05 15:39:32 +010026
Mike Frysinger31596002012-12-03 23:54:24 -050027if cros_build_lib.IsInsideChroot():
Alex Klein1699fab2022-09-08 08:46:06 -060028 # Only import portage after we've checked that we're inside the chroot.
29 # Outside may not have portage, in which case the above may not happen.
30 # We'll check in main() if the operation needs portage.
31 # pylint: disable=import-error
32 import portage
Zdenek Behan508dcce2011-12-05 15:39:32 +010033
34
Alex Klein1699fab2022-09-08 08:46:06 -060035EMERGE_CMD = os.path.join(constants.CHROMITE_BIN_DIR, "parallel_emerge")
36PACKAGE_STABLE = "[stable]"
Zdenek Behan4eb6fd22012-03-12 17:00:56 +010037
Mike Frysinger4cd80b62022-05-04 20:39:01 -040038CHROMIUMOS_OVERLAY = os.path.join(
Alex Klein1699fab2022-09-08 08:46:06 -060039 constants.CHROOT_SOURCE_ROOT, constants.CHROMIUMOS_OVERLAY_DIR
40)
Mike Frysinger4cd80b62022-05-04 20:39:01 -040041ECLASS_OVERLAY = os.path.join(
Alex Klein1699fab2022-09-08 08:46:06 -060042 constants.CHROOT_SOURCE_ROOT, constants.ECLASS_OVERLAY_DIR
43)
Mike Frysinger4cd80b62022-05-04 20:39:01 -040044STABLE_OVERLAY = os.path.join(
Alex Klein1699fab2022-09-08 08:46:06 -060045 constants.CHROOT_SOURCE_ROOT, constants.PORTAGE_STABLE_OVERLAY_DIR
46)
47CROSSDEV_OVERLAY = "/usr/local/portage/crossdev"
Zdenek Behan508dcce2011-12-05 15:39:32 +010048
49
Mike Frysinger66bfde52017-09-12 16:42:57 -040050# The exact list of host toolchain packages we care about. These are the
51# packages that bots/devs install only from binpkgs and rely on the SDK bot
52# (chromiumos-sdk) to validate+uprev.
53#
Mike Frysinger66bfde52017-09-12 16:42:57 -040054# We don't use crossdev to manage the host toolchain for us, especially since
55# we diverge significantly now (with llvm/clang/etc...), and we don't need or
56# want crossdev managing /etc/portage config files for the sdk
57HOST_PACKAGES = (
Alex Klein1699fab2022-09-08 08:46:06 -060058 "dev-lang/go",
59 "dev-lang/rust-bootstrap",
60 "dev-lang/rust-host",
61 "dev-libs/elfutils",
62 "sys-devel/binutils",
63 "sys-devel/gcc",
64 "sys-devel/llvm",
65 "sys-kernel/linux-headers",
66 "sys-libs/glibc",
67 "sys-libs/libcxx",
68 "sys-libs/llvm-libunwind",
Mike Frysinger66bfde52017-09-12 16:42:57 -040069)
70
Mike Frysinger785b0c32017-09-13 01:35:59 -040071# These packages are also installed into the host SDK. However, they require
72# the cross-compilers to be installed first (because they need them to actually
73# build), so we have to delay their installation.
74HOST_POST_CROSS_PACKAGES = (
Alex Klein1699fab2022-09-08 08:46:06 -060075 "dev-lang/rust",
76 "virtual/target-sdk-post-cross",
77 "dev-embedded/coreboot-sdk",
78 "dev-embedded/hps-sdk",
79 "dev-embedded/ti50-sdk",
Mike Frysinger785b0c32017-09-13 01:35:59 -040080)
81
82# New packages that we're in the process of adding to the SDK. Since the SDK
83# bot hasn't had a chance to run yet, there are no binary packages available,
84# so we have to list them here and wait. Once it completes, entries here can
85# be removed so they'll end up on bots & dev's systems.
George Burgess IV66e199c2021-05-05 15:38:40 -070086NEW_PACKAGES = ()
Mike Frysinger785b0c32017-09-13 01:35:59 -040087
Rahul Chaudhry4b803052015-05-13 15:25:56 -070088# Enable the Go compiler for these targets.
89TARGET_GO_ENABLED = (
Alex Klein1699fab2022-09-08 08:46:06 -060090 "x86_64-cros-linux-gnu",
91 "armv7a-cros-linux-gnueabi",
92 "armv7a-cros-linux-gnueabihf",
93 "aarch64-cros-linux-gnu",
Rahul Chaudhry4b803052015-05-13 15:25:56 -070094)
Alex Klein1699fab2022-09-08 08:46:06 -060095CROSSDEV_GO_ARGS = ["--ex-pkg", "dev-lang/go"]
Rahul Chaudhry4b803052015-05-13 15:25:56 -070096
Alex Klein1699fab2022-09-08 08:46:06 -060097CROSSDEV_LIBXCRYPT_ARGS = ["--ex-pkg", "sys-libs/libxcrypt"]
Adrian Ratiubf0b9af2022-05-02 14:48:15 +030098
Manoj Gupta1b5642e2017-03-08 16:44:12 -080099# Enable llvm's compiler-rt for these targets.
100TARGET_COMPILER_RT_ENABLED = (
Alex Klein1699fab2022-09-08 08:46:06 -0600101 "armv7a-cros-linux-gnueabi",
102 "armv7a-cros-linux-gnueabihf",
103 "aarch64-cros-linux-gnu",
104 "arm-none-eabi",
105 "armv7m-cros-eabi",
Manoj Gupta1b5642e2017-03-08 16:44:12 -0800106)
Alex Klein1699fab2022-09-08 08:46:06 -0600107CROSSDEV_COMPILER_RT_ARGS = ["--ex-pkg", "sys-libs/compiler-rt"]
Manoj Gupta1b5642e2017-03-08 16:44:12 -0800108
Manoj Gupta946abb42017-04-12 14:27:19 -0700109TARGET_LLVM_PKGS_ENABLED = (
Alex Klein1699fab2022-09-08 08:46:06 -0600110 "armv7m-cros-eabi",
111 "armv7a-cros-linux-gnueabi",
112 "armv7a-cros-linux-gnueabihf",
113 "aarch64-cros-linux-gnu",
114 "i686-cros-linux-gnu",
115 "x86_64-cros-linux-gnu",
Manoj Gupta946abb42017-04-12 14:27:19 -0700116)
117
118LLVM_PKGS_TABLE = {
Alex Klein1699fab2022-09-08 08:46:06 -0600119 "ex_llvm-libunwind": ["--ex-pkg", "sys-libs/llvm-libunwind"],
120 "ex_libcxx": ["--ex-pkg", "sys-libs/libcxx"],
Manoj Gupta946abb42017-04-12 14:27:19 -0700121}
122
Alex Klein1699fab2022-09-08 08:46:06 -0600123
David James66a09c42012-11-05 13:31:38 -0800124class Crossdev(object):
Alex Klein1699fab2022-09-08 08:46:06 -0600125 """Class for interacting with crossdev and caching its output."""
David James66a09c42012-11-05 13:31:38 -0800126
Alex Klein1699fab2022-09-08 08:46:06 -0600127 _CACHE_FILE = os.path.join(CROSSDEV_OVERLAY, ".configured.json")
128 _CACHE = {}
129 # Packages that needs separate handling, in addition to what we have from
130 # crossdev.
131 MANUAL_PKGS = {
Alex Klein1699fab2022-09-08 08:46:06 -0600132 "llvm": "sys-devel",
133 "llvm-libunwind": "sys-libs",
134 "libcxx": "sys-libs",
135 "elfutils": "dev-libs",
George Burgess IV6c298782023-02-14 14:28:04 -0700136 # b/269306499: note that rust and rust-host are shipped as a part of
137 # this tarball on a best-effort basis. If you would like them to be
138 # fully supported (with an SLA), please reach out to
139 # chromeos-toolchain@google.com and chat with us.
140 "rust": "dev-lang",
141 "rust-host": "dev-lang",
Mike Frysinger3ed47722017-08-08 14:59:08 -0400142 }
143
Alex Klein1699fab2022-09-08 08:46:06 -0600144 @classmethod
145 def Load(cls, reconfig):
146 """Load crossdev cache from disk.
Mike Frysinger3ed47722017-08-08 14:59:08 -0400147
Alex Klein1699fab2022-09-08 08:46:06 -0600148 We invalidate the cache when crossdev updates or this script changes.
149 """
150 crossdev_version = GetStablePackageVersion("sys-devel/crossdev", True)
151 # If we run the compiled/cached .pyc file, we'll read/hash that when we
152 # really always want to track the source .py file.
153 script = os.path.abspath(__file__)
154 if script.endswith(".pyc"):
155 script = script[:-1]
156 setup_toolchains_hash = hashlib.md5(
157 osutils.ReadFile(script, mode="rb")
158 ).hexdigest()
Mike Frysinger3ed47722017-08-08 14:59:08 -0400159
Alex Klein1699fab2022-09-08 08:46:06 -0600160 cls._CACHE = {
161 "crossdev_version": crossdev_version,
162 "setup_toolchains_hash": setup_toolchains_hash,
Mike Frysinger66bfde52017-09-12 16:42:57 -0400163 }
Alex Klein1699fab2022-09-08 08:46:06 -0600164
165 logging.debug("cache: checking file: %s", cls._CACHE_FILE)
166 if reconfig:
167 logging.debug("cache: forcing regen due to reconfig")
168 return
169
170 try:
171 file_data = osutils.ReadFile(cls._CACHE_FILE)
172 except IOError as e:
173 if e.errno != errno.ENOENT:
174 logging.warning("cache: reading failed: %s", e)
175 osutils.SafeUnlink(cls._CACHE_FILE)
176 return
177
178 try:
179 data = json.loads(file_data)
180 except ValueError as e:
181 logging.warning("cache: ignoring invalid content: %s", e)
182 return
183
184 if crossdev_version != data.get("crossdev_version"):
185 logging.debug("cache: rebuilding after crossdev upgrade")
186 elif setup_toolchains_hash != data.get("setup_toolchains_hash"):
187 logging.debug(
188 "cache: rebuilding after cros_setup_toolchains change"
189 )
Mike Frysinger785b0c32017-09-13 01:35:59 -0400190 else:
Alex Klein1699fab2022-09-08 08:46:06 -0600191 logging.debug("cache: content is up-to-date!")
192 cls._CACHE = data
Han Shene23782f2016-02-18 12:20:00 -0800193
Alex Klein1699fab2022-09-08 08:46:06 -0600194 @classmethod
195 def Save(cls):
196 """Store crossdev cache on disk."""
197 # Save the cache from the successful run.
Mike Frysinger31fdddd2023-02-24 15:50:55 -0500198 with open(cls._CACHE_FILE, "w", encoding="utf-8") as f:
Alex Klein1699fab2022-09-08 08:46:06 -0600199 json.dump(cls._CACHE, f)
Mike Frysinger66bfde52017-09-12 16:42:57 -0400200
Alex Klein1699fab2022-09-08 08:46:06 -0600201 @classmethod
202 def GetConfig(cls, target):
203 """Returns a map of crossdev provided variables about a tuple."""
204 CACHE_ATTR = "_target_tuple_map"
Han Shene23782f2016-02-18 12:20:00 -0800205
Alex Klein1699fab2022-09-08 08:46:06 -0600206 val = cls._CACHE.setdefault(CACHE_ATTR, {})
207 if not target in val:
208 if target.startswith("host"):
209 conf = {
210 "crosspkgs": [],
211 "target": toolchain.GetHostTuple(),
212 }
213 if target == "host":
214 packages_list = HOST_PACKAGES
215 else:
216 packages_list = HOST_POST_CROSS_PACKAGES
217 manual_pkgs = dict(
218 (pkg, cat)
219 for cat, pkg in [x.split("/") for x in packages_list]
220 )
221 else:
222 # Build the crossdev command.
223 cmd = ["crossdev", "--show-target-cfg", "--ex-gdb"]
224 # Enable libxcrypt for all linux-gnu targets.
225 if "cros-linux-gnu" in target:
226 cmd.extend(CROSSDEV_LIBXCRYPT_ARGS)
227 if target in TARGET_COMPILER_RT_ENABLED:
228 cmd.extend(CROSSDEV_COMPILER_RT_ARGS)
229 if target in TARGET_LLVM_PKGS_ENABLED:
230 for pkg in LLVM_PKGS_TABLE:
231 cmd.extend(LLVM_PKGS_TABLE[pkg])
232 if target in TARGET_GO_ENABLED:
233 cmd.extend(CROSSDEV_GO_ARGS)
234 cmd.extend(["-t", target])
235 # Catch output of crossdev.
236 out = cros_build_lib.run(
237 cmd, print_cmd=False, stdout=True, encoding="utf-8"
238 ).stdout.splitlines()
239 # List of tuples split at the first '=', converted into dict.
240 conf = dict(
241 (k, cros_build_lib.ShellUnquote(v))
242 for k, v in (x.split("=", 1) for x in out)
243 )
244 conf["crosspkgs"] = conf["crosspkgs"].split()
Han Shene23782f2016-02-18 12:20:00 -0800245
Alex Klein1699fab2022-09-08 08:46:06 -0600246 manual_pkgs = cls.MANUAL_PKGS
David James66a09c42012-11-05 13:31:38 -0800247
Alex Klein1699fab2022-09-08 08:46:06 -0600248 for pkg, cat in manual_pkgs.items():
249 conf[pkg + "_pn"] = pkg
250 conf[pkg + "_category"] = cat
251 if pkg not in conf["crosspkgs"]:
252 conf["crosspkgs"].append(pkg)
David James66a09c42012-11-05 13:31:38 -0800253
Alex Klein1699fab2022-09-08 08:46:06 -0600254 val[target] = conf
David James66a09c42012-11-05 13:31:38 -0800255
Alex Klein1699fab2022-09-08 08:46:06 -0600256 return val[target]
Manoj Gupta4d016f62021-10-19 16:39:34 -0700257
Alex Klein1699fab2022-09-08 08:46:06 -0600258 @classmethod
259 def UpdateTargets(cls, targets, usepkg, config_only=False):
260 """Calls crossdev to initialize a cross target.
Manoj Gupta4d016f62021-10-19 16:39:34 -0700261
Alex Klein1699fab2022-09-08 08:46:06 -0600262 Args:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700263 targets: The dict of targets to initialize using crossdev.
264 usepkg: Copies the commandline opts.
265 config_only: Just update.
Alex Klein1699fab2022-09-08 08:46:06 -0600266 """
267 configured_targets = cls._CACHE.setdefault("configured_targets", [])
268 started_targets = set()
David James66a09c42012-11-05 13:31:38 -0800269
Alex Klein1699fab2022-09-08 08:46:06 -0600270 # Schedule all of the targets in parallel, and let them run.
271 with parallel.BackgroundTaskRunner(cls._UpdateTarget) as queue:
272 for target_name in targets:
273 # We already started this target in this loop.
274 if target_name in started_targets:
275 continue
276 # The target is already configured.
277 if config_only and target_name in configured_targets:
278 continue
279 queue.put(
280 [target_name, targets[target_name], usepkg, config_only]
281 )
282 started_targets.add(target_name)
David James66a09c42012-11-05 13:31:38 -0800283
Alex Klein1699fab2022-09-08 08:46:06 -0600284 @classmethod
285 def _UpdateTarget(cls, target_name, target, usepkg, config_only):
286 """Calls crossdev to initialize a cross target.
David James66a09c42012-11-05 13:31:38 -0800287
Alex Klein1699fab2022-09-08 08:46:06 -0600288 Args:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700289 target_name: The name of the target to initialize.
290 target: The target info for initializing.
291 usepkg: Copies the commandline opts.
292 config_only: Just update.
Alex Klein1699fab2022-09-08 08:46:06 -0600293 """
294 configured_targets = cls._CACHE.setdefault("configured_targets", [])
295 cmdbase = ["crossdev", "--show-fail-log"]
296 cmdbase.extend(["--env", "FEATURES=splitdebug"])
297 # Pick stable by default, and override as necessary.
298 cmdbase.extend(["-P", "--oneshot"])
299 if usepkg:
300 cmdbase.extend(
301 ["-P", "--getbinpkg", "-P", "--usepkgonly", "--without-headers"]
302 )
David James66a09c42012-11-05 13:31:38 -0800303
Alex Klein1699fab2022-09-08 08:46:06 -0600304 overlays = " ".join(
305 (CHROMIUMOS_OVERLAY, ECLASS_OVERLAY, STABLE_OVERLAY)
306 )
307 cmdbase.extend(["--overlays", overlays])
308 cmdbase.extend(["--ov-output", CROSSDEV_OVERLAY])
Manoj Gupta4d016f62021-10-19 16:39:34 -0700309
Alex Klein1699fab2022-09-08 08:46:06 -0600310 cmd = cmdbase + ["-t", target_name]
Manoj Gupta4d016f62021-10-19 16:39:34 -0700311
Alex Klein1699fab2022-09-08 08:46:06 -0600312 for pkg in GetTargetPackages(target_name):
313 if pkg == "gdb":
314 # Gdb does not have selectable versions.
315 cmd.append("--ex-gdb")
316 elif pkg == "ex_libxcrypt":
317 cmd.extend(CROSSDEV_LIBXCRYPT_ARGS)
318 elif pkg == "ex_compiler-rt":
319 cmd.extend(CROSSDEV_COMPILER_RT_ARGS)
320 elif pkg == "ex_go":
321 # Go does not have selectable versions.
322 cmd.extend(CROSSDEV_GO_ARGS)
323 elif pkg in LLVM_PKGS_TABLE:
324 cmd.extend(LLVM_PKGS_TABLE[pkg])
325 elif pkg in cls.MANUAL_PKGS:
326 pass
327 else:
328 # The first of the desired versions is the "primary" one.
329 version = GetDesiredPackageVersions(target_name, pkg)[0]
330 cmd.extend(["--%s" % pkg, version])
331
332 cmd.extend(target["crossdev"].split())
333
334 if config_only:
335 # In this case we want to just quietly reinit
336 cmd.append("--init-target")
337 cros_build_lib.run(cmd, print_cmd=False, stdout=True)
338 else:
339 cros_build_lib.run(cmd)
340
341 configured_targets.append(target_name)
David James66a09c42012-11-05 13:31:38 -0800342
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100343
Zdenek Behanf4d18a02012-03-22 15:45:05 +0100344def GetTargetPackages(target):
Alex Klein1699fab2022-09-08 08:46:06 -0600345 """Returns a list of packages for a given target."""
346 conf = Crossdev.GetConfig(target)
347 # Undesired packages are denoted by empty ${pkg}_pn variable.
348 return [x for x in conf["crosspkgs"] if conf.get(x + "_pn")]
Zdenek Behanf4d18a02012-03-22 15:45:05 +0100349
350
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100351# Portage helper functions:
352def GetPortagePackage(target, package):
Alex Klein1699fab2022-09-08 08:46:06 -0600353 """Returns a package name for the given target."""
354 conf = Crossdev.GetConfig(target)
355 # Portage category:
356 if target.startswith("host") or package in Crossdev.MANUAL_PKGS:
357 category = conf[package + "_category"]
358 else:
359 category = conf["category"]
360 # Portage package:
361 pn = conf[package + "_pn"]
362 # Final package name:
363 assert category
364 assert pn
365 return "%s/%s" % (category, pn)
Zdenek Behan4eb6fd22012-03-12 17:00:56 +0100366
367
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700368def PortageTrees(root):
Alex Klein1699fab2022-09-08 08:46:06 -0600369 """Return the portage trees for a given root."""
370 if root == "/":
371 return portage.db["/"]
372 # The portage logic requires the path always end in a slash.
373 root = root.rstrip("/") + "/"
374 return portage.create_trees(target_root=root, config_root=root)[root]
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700375
376
Alex Klein1699fab2022-09-08 08:46:06 -0600377def GetInstalledPackageVersions(atom, root="/"):
378 """Extracts the list of current versions of a target, package pair.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100379
Alex Klein1699fab2022-09-08 08:46:06 -0600380 Args:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700381 atom: The atom to operate on (e.g. sys-devel/gcc)
382 root: The root to check for installed packages.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100383
Alex Klein1699fab2022-09-08 08:46:06 -0600384 Returns:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700385 The list of versions of the package currently installed.
Alex Klein1699fab2022-09-08 08:46:06 -0600386 """
387 versions = []
388 for pkg in PortageTrees(root)["vartree"].dbapi.match(atom, use_cache=0):
389 version = portage.versions.cpv_getversion(pkg)
390 versions.append(version)
391 return versions
Zdenek Behan508dcce2011-12-05 15:39:32 +0100392
393
Alex Klein1699fab2022-09-08 08:46:06 -0600394def GetStablePackageVersion(atom, installed, root="/"):
395 """Extracts the current stable version for a given package.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100396
Alex Klein1699fab2022-09-08 08:46:06 -0600397 Args:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700398 atom: The target/package to operate on e.g. i686-cros-linux-gnu/gcc
399 installed: Whether we want installed packages or ebuilds
400 root: The root to use when querying packages.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100401
Alex Klein1699fab2022-09-08 08:46:06 -0600402 Returns:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700403 A string containing the latest version.
Alex Klein1699fab2022-09-08 08:46:06 -0600404 """
405 pkgtype = "vartree" if installed else "porttree"
406 cpv = portage.best(
407 PortageTrees(root)[pkgtype].dbapi.match(atom, use_cache=0)
408 )
409 return portage.versions.cpv_getversion(cpv) if cpv else None
Zdenek Behan508dcce2011-12-05 15:39:32 +0100410
411
Alex Klein1699fab2022-09-08 08:46:06 -0600412def VersionListToNumeric(target, package, versions, installed, root="/"):
413 """Resolves keywords in a given version list for a particular package.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100414
Alex Klein1699fab2022-09-08 08:46:06 -0600415 Resolving means replacing PACKAGE_STABLE with the actual number.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100416
Alex Klein1699fab2022-09-08 08:46:06 -0600417 Args:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700418 target: The target to operate on (e.g. i686-cros-linux-gnu)
419 package: The target/package to operate on (e.g. gcc)
420 versions: List of versions to resolve
421 installed: Query installed packages
422 root: The install root to use; ignored if |installed| is False.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100423
Alex Klein1699fab2022-09-08 08:46:06 -0600424 Returns:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700425 List of purely numeric versions equivalent to argument
Alex Klein1699fab2022-09-08 08:46:06 -0600426 """
427 resolved = []
428 atom = GetPortagePackage(target, package)
429 if not installed:
430 root = "/"
431 for version in versions:
432 if version == PACKAGE_STABLE:
433 resolved.append(GetStablePackageVersion(atom, installed, root=root))
434 else:
435 resolved.append(version)
436 return resolved
Zdenek Behan508dcce2011-12-05 15:39:32 +0100437
438
439def GetDesiredPackageVersions(target, package):
Alex Klein1699fab2022-09-08 08:46:06 -0600440 """Produces the list of desired versions for each target, package pair.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100441
Alex Klein1699fab2022-09-08 08:46:06 -0600442 The first version in the list is implicitly treated as primary, ie.
443 the version that will be initialized by crossdev and selected.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100444
Alex Klein1699fab2022-09-08 08:46:06 -0600445 If the version is PACKAGE_STABLE, it really means the current version which
446 is emerged by using the package atom with no particular version key.
447 Since crossdev unmasks all packages by default, this will actually
448 mean 'unstable' in most cases.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100449
Alex Klein1699fab2022-09-08 08:46:06 -0600450 Args:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700451 target: The target to operate on (e.g. i686-cros-linux-gnu)
452 package: The target/package to operate on (e.g. gcc)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100453
Alex Klein1699fab2022-09-08 08:46:06 -0600454 Returns:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700455 A list composed of either a version string, PACKAGE_STABLE
Alex Klein1699fab2022-09-08 08:46:06 -0600456 """
457 if package in GetTargetPackages(target):
458 return [PACKAGE_STABLE]
459 else:
460 return []
Zdenek Behan508dcce2011-12-05 15:39:32 +0100461
462
463def TargetIsInitialized(target):
Alex Klein1699fab2022-09-08 08:46:06 -0600464 """Verifies if the given list of targets has been correctly initialized.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100465
Alex Klein1699fab2022-09-08 08:46:06 -0600466 This determines whether we have to call crossdev while emerging
467 toolchain packages or can do it using emerge. Emerge is naturally
468 preferred, because all packages can be updated in a single pass.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100469
Alex Klein1699fab2022-09-08 08:46:06 -0600470 Args:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700471 target: The target to operate on (e.g. i686-cros-linux-gnu)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100472
Alex Klein1699fab2022-09-08 08:46:06 -0600473 Returns:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700474 True if |target| is completely initialized, else False
Alex Klein1699fab2022-09-08 08:46:06 -0600475 """
476 # Check if packages for the given target all have a proper version.
477 try:
478 for package in GetTargetPackages(target):
479 atom = GetPortagePackage(target, package)
480 # Do we even want this package && is it initialized?
481 if not (
482 GetStablePackageVersion(atom, True)
483 and GetStablePackageVersion(atom, False)
484 ):
485 return False
486 return True
487 except cros_build_lib.RunCommandError:
488 # Fails - The target has likely never been initialized before.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100489 return False
Zdenek Behan508dcce2011-12-05 15:39:32 +0100490
491
492def RemovePackageMask(target):
Alex Klein1699fab2022-09-08 08:46:06 -0600493 """Removes a package.mask file for the given platform.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100494
Alex Klein1699fab2022-09-08 08:46:06 -0600495 The pre-existing package.mask files can mess with the keywords.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100496
Alex Klein1699fab2022-09-08 08:46:06 -0600497 Args:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700498 target: The target to operate on (e.g. i686-cros-linux-gnu)
Alex Klein1699fab2022-09-08 08:46:06 -0600499 """
500 maskfile = os.path.join("/etc/portage/package.mask", "cross-" + target)
501 osutils.SafeUnlink(maskfile)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100502
503
Zdenek Behan508dcce2011-12-05 15:39:32 +0100504# Main functions performing the actual update steps.
Alex Klein1699fab2022-09-08 08:46:06 -0600505def RebuildLibtool(root="/"):
506 """Rebuild libtool as needed
Mike Frysingerc880a962013-11-08 13:59:06 -0500507
Alex Klein1699fab2022-09-08 08:46:06 -0600508 Libtool hardcodes full paths to internal gcc files, so whenever we upgrade
509 gcc, libtool will break. We can't use binary packages either as those will
510 most likely be compiled against the previous version of gcc.
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700511
Alex Klein1699fab2022-09-08 08:46:06 -0600512 Args:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700513 root: The install root where we want libtool rebuilt.
Alex Klein1699fab2022-09-08 08:46:06 -0600514 """
515 needs_update = False
Mike Frysinger31fdddd2023-02-24 15:50:55 -0500516 with open(os.path.join(root, "usr/bin/libtool"), encoding="utf-8") as f:
Alex Klein1699fab2022-09-08 08:46:06 -0600517 for line in f:
518 # Look for a line like:
519 # sys_lib_search_path_spec="..."
520 # It'll be a list of paths and gcc will be one of them.
521 if line.startswith("sys_lib_search_path_spec="):
522 line = line.rstrip()
523 for path in line.split("=", 1)[1].strip('"').split():
524 root_path = os.path.join(root, path.lstrip(os.path.sep))
525 logging.debug("Libtool: checking %s", root_path)
526 if not os.path.exists(root_path):
527 logging.info("Rebuilding libtool after gcc upgrade")
528 logging.info(" %s", line)
529 logging.info(" missing path: %s", path)
530 needs_update = True
531 break
Mike Frysingerc880a962013-11-08 13:59:06 -0500532
Alex Klein1699fab2022-09-08 08:46:06 -0600533 if needs_update:
534 break
Mike Frysingerc880a962013-11-08 13:59:06 -0500535
Alex Klein1699fab2022-09-08 08:46:06 -0600536 if needs_update:
537 cmd = [EMERGE_CMD, "--oneshot"]
538 if root != "/":
539 cmd.extend(["--sysroot=%s" % root, "--root=%s" % root])
540 cmd.append("sys-devel/libtool")
541 cros_build_lib.run(cmd)
542 else:
543 logging.debug("Libtool is up-to-date; no need to rebuild")
Mike Frysingerc880a962013-11-08 13:59:06 -0500544
545
Alex Klein1699fab2022-09-08 08:46:06 -0600546def UpdateTargets(targets, usepkg, root="/"):
547 """Determines which packages need update/unmerge and defers to portage.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100548
Alex Klein1699fab2022-09-08 08:46:06 -0600549 Args:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700550 targets: The list of targets to update
551 usepkg: Copies the commandline option
552 root: The install root in which we want packages updated.
Alex Klein1699fab2022-09-08 08:46:06 -0600553 """
554 # For each target, we do two things. Figure out the list of updates,
555 # and figure out the appropriate keywords/masks. Crossdev will initialize
556 # these, but they need to be regenerated on every update.
557 logging.info("Determining required toolchain updates...")
558 mergemap = {}
559 # Used to keep track of post-cross packages. These are allowed to have
560 # implicit dependencies on toolchain packages, and therefore need to
561 # be built last.
562 post_cross_pkgs = set()
Zdenek Behan508dcce2011-12-05 15:39:32 +0100563 for target in targets:
Alex Klein1699fab2022-09-08 08:46:06 -0600564 is_post_cross_target = target.endswith("-post-cross")
565 logging.debug("Updating target %s", target)
Alex Klein0e92b2c2023-01-13 11:54:15 -0700566 # Record the highest needed version for each target, for masking
567 # purposes.
Alex Klein1699fab2022-09-08 08:46:06 -0600568 RemovePackageMask(target)
569 for package in GetTargetPackages(target):
570 # Portage name for the package
571 logging.debug(" Checking package %s", package)
572 pkg = GetPortagePackage(target, package)
573 current = GetInstalledPackageVersions(pkg, root=root)
574 desired = GetDesiredPackageVersions(target, package)
575 desired_num = VersionListToNumeric(target, package, desired, False)
576 if pkg in NEW_PACKAGES and usepkg:
577 # Skip this binary package (for now).
578 continue
579 mergemap[pkg] = set(desired_num).difference(current)
580 logging.debug(" %s -> %s", current, desired_num)
581 if is_post_cross_target:
582 post_cross_pkgs.add(pkg)
Mike Frysinger785b0c32017-09-13 01:35:59 -0400583
Alex Klein1699fab2022-09-08 08:46:06 -0600584 packages = [pkg for pkg, vers in mergemap.items() if vers]
585 if not packages:
586 logging.info("Nothing to update!")
587 return False
Zdenek Behan508dcce2011-12-05 15:39:32 +0100588
Alex Klein1699fab2022-09-08 08:46:06 -0600589 logging.info("Updating packages:")
590 logging.info("%s", packages)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100591
Alex Klein1699fab2022-09-08 08:46:06 -0600592 cmd = [EMERGE_CMD, "--oneshot", "--update"]
593 if usepkg:
594 cmd.extend(["--getbinpkg", "--usepkgonly"])
595 if root != "/":
596 cmd.extend(["--sysroot=%s" % root, "--root=%s" % root])
Zdenek Behan508dcce2011-12-05 15:39:32 +0100597
Alex Klein1699fab2022-09-08 08:46:06 -0600598 if usepkg:
599 # Since we are not building from source, we can handle
600 # all packages in one go.
601 cmd.extend(packages)
602 cros_build_lib.run(cmd)
603 else:
604 pre_cross_items = [
605 pkg for pkg in packages if pkg not in post_cross_pkgs
606 ]
607 if pre_cross_items:
608 cros_build_lib.run(cmd + pre_cross_items)
609 post_cross_items = [pkg for pkg in packages if pkg in post_cross_pkgs]
610 if post_cross_items:
611 cros_build_lib.run(cmd + post_cross_items)
612 return True
Gilad Arnold2dab78c2015-05-21 14:43:33 -0700613
Alex Klein1699fab2022-09-08 08:46:06 -0600614
615def CleanTargets(targets, root="/"):
616 """Unmerges old packages that are assumed unnecessary.
617
618 Args:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700619 targets: The list of targets to clean up.
620 root: The install root in which we want packages cleaned up.
Alex Klein1699fab2022-09-08 08:46:06 -0600621 """
622 unmergemap = {}
623 for target in targets:
624 logging.debug("Cleaning target %s", target)
625 for package in GetTargetPackages(target):
626 logging.debug(" Cleaning package %s", package)
627 pkg = GetPortagePackage(target, package)
628 current = GetInstalledPackageVersions(pkg, root=root)
629 desired = GetDesiredPackageVersions(target, package)
630 # NOTE: This refers to installed packages (vartree) rather than the
Alex Klein0e92b2c2023-01-13 11:54:15 -0700631 # Portage version (porttree and/or bintree) when determining the
632 # current version. While this isn't the most accurate thing to do,
633 # it is probably a good simple compromise, which should have the
634 # desired result of uninstalling everything but the latest installed
635 # version. In particular, using the bintree (--usebinpkg) requires a
636 # non-trivial binhost sync and is probably more complex than useful.
Alex Klein1699fab2022-09-08 08:46:06 -0600637 desired_num = VersionListToNumeric(target, package, desired, True)
638 if not set(desired_num).issubset(current):
639 logging.warning(
640 "Error detecting stable version for %s, " "skipping clean!",
641 pkg,
642 )
643 return
644 unmergemap[pkg] = set(current).difference(desired_num)
645
646 # Cleaning doesn't care about consistency and rebuilding package.* files.
647 packages = []
648 for pkg, vers in unmergemap.items():
649 packages.extend("=%s-%s" % (pkg, ver) for ver in vers if ver != "9999")
650
651 if packages:
652 logging.info("Cleaning packages:")
653 logging.info("%s", packages)
654 cmd = [EMERGE_CMD, "--unmerge"]
655 if root != "/":
656 cmd.extend(["--sysroot=%s" % root, "--root=%s" % root])
657 cmd.extend(packages)
658 cros_build_lib.run(cmd)
659 else:
660 logging.info("Nothing to clean!")
661
662
663def SelectActiveToolchains(targets, root="/"):
664 """Runs gcc-config and binutils-config to select the desired.
665
666 Args:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700667 targets: The targets to select
668 root: The root where we want to select toolchain versions.
Alex Klein1699fab2022-09-08 08:46:06 -0600669 """
670 for package in ["gcc", "binutils"]:
671 for target in targets:
672 # See if this package is part of this target.
673 if package not in GetTargetPackages(target):
674 logging.debug("%s: %s is not used", target, package)
675 continue
676
677 # Pick the first version in the numbered list as the selected one.
678 desired = GetDesiredPackageVersions(target, package)
679 desired_num = VersionListToNumeric(
680 target, package, desired, True, root=root
681 )
682 desired = desired_num[0]
683 # *-config does not play revisions, strip them, keep just PV.
684 desired = portage.versions.pkgsplit("%s-%s" % (package, desired))[1]
685
686 if target.startswith("host"):
Alex Klein0e92b2c2023-01-13 11:54:15 -0700687 # *-config is the only tool treating host identically (by
688 # tuple).
Alex Klein1699fab2022-09-08 08:46:06 -0600689 target = toolchain.GetHostTuple()
690
691 # And finally, attach target to it.
692 desired = "%s-%s" % (target, desired)
693
694 extra_env = {"CHOST": target}
695 if root != "/":
696 extra_env["ROOT"] = root
697 cmd = ["%s-config" % package, "-c", target]
698 result = cros_build_lib.run(
699 cmd,
700 print_cmd=False,
701 stdout=True,
702 encoding="utf-8",
703 extra_env=extra_env,
704 )
705 current = result.stdout.splitlines()[0]
706
Alex Klein0e92b2c2023-01-13 11:54:15 -0700707 # Do not reconfig when the current is live or nothing needs to be
708 # done.
Alex Klein1699fab2022-09-08 08:46:06 -0600709 extra_env = {"ROOT": root} if root != "/" else None
710 if current != desired and current != "9999":
711 cmd = [package + "-config", desired]
712 cros_build_lib.run(cmd, print_cmd=False, extra_env=extra_env)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100713
714
Mike Frysinger35247af2012-11-16 18:58:06 -0500715def ExpandTargets(targets_wanted):
Alex Klein1699fab2022-09-08 08:46:06 -0600716 """Expand any possible toolchain aliases into full targets
Mike Frysinger35247af2012-11-16 18:58:06 -0500717
Alex Klein1699fab2022-09-08 08:46:06 -0600718 This will expand 'all' and 'sdk' into the respective toolchain tuples.
Mike Frysinger35247af2012-11-16 18:58:06 -0500719
Alex Klein1699fab2022-09-08 08:46:06 -0600720 Args:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700721 targets_wanted: The targets specified by the user.
Mike Frysinger1a736a82013-12-12 01:50:59 -0500722
Alex Klein1699fab2022-09-08 08:46:06 -0600723 Returns:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700724 Dictionary of concrete targets and their toolchain tuples.
Alex Klein1699fab2022-09-08 08:46:06 -0600725 """
726 targets_wanted = set(targets_wanted)
727 if targets_wanted == set(["boards"]):
728 # Only pull targets from the included boards.
729 return {}
Gilad Arnold8195b532015-04-07 10:56:30 +0300730
Alex Klein1699fab2022-09-08 08:46:06 -0600731 all_targets = toolchain.GetAllTargets()
732 if targets_wanted == set(["all"]):
733 return all_targets
734 if targets_wanted == set(["sdk"]):
735 # Filter out all the non-sdk toolchains as we don't want to mess
736 # with those in all of our builds.
737 return toolchain.FilterToolchains(all_targets, "sdk", True)
Gilad Arnold8195b532015-04-07 10:56:30 +0300738
Alex Klein1699fab2022-09-08 08:46:06 -0600739 # Verify user input.
740 nonexistent = targets_wanted.difference(all_targets)
741 if nonexistent:
742 raise ValueError("Invalid targets: %s" % (",".join(nonexistent),))
743 return {t: all_targets[t] for t in targets_wanted}
Mike Frysinger35247af2012-11-16 18:58:06 -0500744
745
Alex Klein1699fab2022-09-08 08:46:06 -0600746def UpdateToolchains(
747 usepkg,
748 deleteold,
749 hostonly,
750 reconfig,
751 targets_wanted,
752 boards_wanted,
753 root="/",
754):
755 """Performs all steps to create a synchronized toolchain enviroment.
Zdenek Behan508dcce2011-12-05 15:39:32 +0100756
Alex Klein1699fab2022-09-08 08:46:06 -0600757 Args:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700758 usepkg: Use prebuilt packages
759 deleteold: Unmerge deprecated packages
760 hostonly: Only setup the host toolchain
761 reconfig: Reload crossdev config and reselect toolchains
762 targets_wanted: All the targets to update
763 boards_wanted: Load targets from these boards
764 root: The root in which to install the toolchains.
Alex Klein1699fab2022-09-08 08:46:06 -0600765 """
766 targets, crossdev_targets, reconfig_targets = {}, {}, {}
767 if not hostonly:
768 # For hostonly, we can skip most of the below logic, much of which won't
769 # work on bare systems where this is useful.
770 targets = ExpandTargets(targets_wanted)
Mike Frysinger7ccee992012-06-01 21:27:59 -0400771
Alex Klein1699fab2022-09-08 08:46:06 -0600772 # Filter out toolchains that don't (yet) have a binpkg available.
773 if usepkg:
774 for target in list(targets.keys()):
775 if not targets[target]["have-binpkg"]:
776 del targets[target]
Mike Frysingerd246fb92021-10-26 16:08:39 -0400777
Alex Klein1699fab2022-09-08 08:46:06 -0600778 # Now re-add any targets that might be from this board. This is to
779 # allow unofficial boards to declare their own toolchains.
780 for board in boards_wanted:
781 targets.update(toolchain.GetToolchainsForBoard(board))
Zdenek Behan508dcce2011-12-05 15:39:32 +0100782
Alex Klein1699fab2022-09-08 08:46:06 -0600783 # First check and initialize all cross targets that need to be.
784 for target in targets:
785 if TargetIsInitialized(target):
786 reconfig_targets[target] = targets[target]
787 else:
788 crossdev_targets[target] = targets[target]
789 if crossdev_targets:
790 logging.info("The following targets need to be re-initialized:")
791 logging.info("%s", crossdev_targets)
792 Crossdev.UpdateTargets(crossdev_targets, usepkg)
793 # Those that were not initialized may need a config update.
794 Crossdev.UpdateTargets(reconfig_targets, usepkg, config_only=True)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100795
Alex Klein0e92b2c2023-01-13 11:54:15 -0700796 # If we're building a subset of toolchains for a board, we might not
797 # have all the tuples that the packages expect. We don't define the
798 # "full" set of tuples currently other than "whatever the full sdk has
799 # normally".
Alex Klein1699fab2022-09-08 08:46:06 -0600800 if usepkg or set(("all", "sdk")) & targets_wanted:
801 # Since we have cross-compilers now, we can update these packages.
802 targets["host-post-cross"] = {}
Mike Frysinger785b0c32017-09-13 01:35:59 -0400803
Alex Klein1699fab2022-09-08 08:46:06 -0600804 # We want host updated.
805 targets["host"] = {}
Zdenek Behan7e33b4e2012-03-12 17:00:56 +0100806
Alex Klein1699fab2022-09-08 08:46:06 -0600807 # Now update all packages.
808 if (
809 UpdateTargets(targets, usepkg, root=root)
810 or crossdev_targets
811 or reconfig
812 ):
813 SelectActiveToolchains(targets, root=root)
David James7ec5efc2012-11-06 09:39:49 -0800814
Alex Klein1699fab2022-09-08 08:46:06 -0600815 if deleteold:
816 CleanTargets(targets, root=root)
Zdenek Behan508dcce2011-12-05 15:39:32 +0100817
Alex Klein1699fab2022-09-08 08:46:06 -0600818 # Now that we've cleared out old versions, see if we need to rebuild
819 # anything. Can't do this earlier as it might not be broken.
820 RebuildLibtool(root=root)
Mike Frysingerc880a962013-11-08 13:59:06 -0500821
Zdenek Behan508dcce2011-12-05 15:39:32 +0100822
Bertrand SIMONNETcae9d5f2015-03-09 15:58:01 -0700823def ShowConfig(name):
Alex Klein1699fab2022-09-08 08:46:06 -0600824 """Show the toolchain tuples used by |name|
Mike Frysinger35247af2012-11-16 18:58:06 -0500825
Alex Klein1699fab2022-09-08 08:46:06 -0600826 Args:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700827 name: The board name to query.
Alex Klein1699fab2022-09-08 08:46:06 -0600828 """
829 toolchains = toolchain.GetToolchainsForBoard(name)
830 # Make sure we display the default toolchain first.
831 # Note: Do not use logging here as this is meant to be used by other tools.
832 print(
833 ",".join(
834 list(toolchain.FilterToolchains(toolchains, "default", True))
835 + list(toolchain.FilterToolchains(toolchains, "default", False))
836 )
837 )
Mike Frysinger35247af2012-11-16 18:58:06 -0500838
839
Mike Frysinger35247af2012-11-16 18:58:06 -0500840def GeneratePathWrapper(root, wrappath, path):
Alex Klein1699fab2022-09-08 08:46:06 -0600841 """Generate a shell script to execute another shell script
Mike Frysinger35247af2012-11-16 18:58:06 -0500842
Alex Klein1699fab2022-09-08 08:46:06 -0600843 Since we can't symlink a wrapped ELF (see GenerateLdsoWrapper) because the
844 argv[0] won't be pointing to the correct path, generate a shell script that
845 just executes another program with its full path.
Mike Frysinger35247af2012-11-16 18:58:06 -0500846
Alex Klein1699fab2022-09-08 08:46:06 -0600847 Args:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700848 root: The root tree to generate scripts inside of
849 wrappath: The full path (inside |root|) to create the wrapper
850 path: The target program which this wrapper will execute
Alex Klein1699fab2022-09-08 08:46:06 -0600851 """
852 replacements = {
853 "path": path,
854 "relroot": os.path.relpath("/", os.path.dirname(wrappath)),
855 }
Takuto Ikuta58403972018-08-16 18:52:51 +0900856
Alex Klein1699fab2022-09-08 08:46:06 -0600857 # Do not use exec here, because exec invokes script with absolute path in
Alex Klein0e92b2c2023-01-13 11:54:15 -0700858 # argv0. Keeping relativeness allows us to remove abs path from compile
859 # result and leads directory independent build cache sharing in some
860 # distributed build system.
Alex Klein1699fab2022-09-08 08:46:06 -0600861 wrapper = (
862 """#!/bin/sh
Takuto Ikuta58403972018-08-16 18:52:51 +0900863basedir=$(dirname "$0")
864"${basedir}/%(relroot)s%(path)s" "$@"
865exit "$?"
Alex Klein1699fab2022-09-08 08:46:06 -0600866"""
867 % replacements
868 )
869 root_wrapper = root + wrappath
870 if os.path.islink(root_wrapper):
871 os.unlink(root_wrapper)
872 else:
873 osutils.SafeMakedirs(os.path.dirname(root_wrapper))
874 osutils.WriteFile(root_wrapper, wrapper)
875 os.chmod(root_wrapper, 0o755)
Mike Frysinger35247af2012-11-16 18:58:06 -0500876
877
Rahul Chaudhrya8127bb2016-07-22 15:53:17 -0700878def FixClangXXWrapper(root, path):
Alex Klein1699fab2022-09-08 08:46:06 -0600879 """Fix wrapper shell scripts and symlinks for invoking clang++
Rahul Chaudhrya8127bb2016-07-22 15:53:17 -0700880
Alex Klein1699fab2022-09-08 08:46:06 -0600881 In a typical installation, clang++ symlinks to clang, which symlinks to the
882 elf executable. The executable distinguishes between clang and clang++ based
883 on argv[0].
Rahul Chaudhrya8127bb2016-07-22 15:53:17 -0700884
Alex Klein0e92b2c2023-01-13 11:54:15 -0700885 When invoked through the LdsoWrapper, argv[0] always contains the path to
886 the executable elf file, making clang/clang++ invocations indistinguishable.
Rahul Chaudhrya8127bb2016-07-22 15:53:17 -0700887
Alex Klein1699fab2022-09-08 08:46:06 -0600888 This function detects if the elf executable being wrapped is clang-X.Y, and
889 fixes wrappers/symlinks as necessary so that clang++ will work correctly.
Rahul Chaudhrya8127bb2016-07-22 15:53:17 -0700890
Alex Klein1699fab2022-09-08 08:46:06 -0600891 The calling sequence now becomes:
892 -) clang++ invocation turns into clang++-3.9 (which is a copy of clang-3.9,
893 the Ldsowrapper).
894 -) clang++-3.9 uses the Ldso to invoke clang++-3.9.elf, which is a symlink
895 to the original clang-3.9 elf.
896 -) The difference this time is that inside the elf file execution, $0 is
897 set as .../usr/bin/clang++-3.9.elf, which contains 'clang++' in the name.
Rahul Chaudhrya8127bb2016-07-22 15:53:17 -0700898
Alex Klein1699fab2022-09-08 08:46:06 -0600899 Update: Starting since clang 7, the clang and clang++ are symlinks to
900 clang-7 binary, not clang-7.0. The pattern match is extended to handle
901 both clang-7 and clang-7.0 cases for now. (https://crbug.com/837889)
Manoj Guptaae268142018-04-27 23:28:36 -0700902
Alex Klein1699fab2022-09-08 08:46:06 -0600903 Args:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700904 root: The root tree to generate scripts / symlinks inside of
905 path: The target elf for which LdsoWrapper was created
Alex Klein1699fab2022-09-08 08:46:06 -0600906 """
907 if re.match(r"/usr/bin/clang-\d+(\.\d+)*$", path):
908 logging.info("fixing clang++ invocation for %s", path)
909 clangdir = os.path.dirname(root + path)
910 clang = os.path.basename(path)
911 clangxx = clang.replace("clang", "clang++")
Rahul Chaudhrya8127bb2016-07-22 15:53:17 -0700912
Alex Klein1699fab2022-09-08 08:46:06 -0600913 # Create a symlink clang++-X.Y.elf to point to clang-X.Y.elf
914 os.symlink(clang + ".elf", os.path.join(clangdir, clangxx + ".elf"))
Rahul Chaudhrya8127bb2016-07-22 15:53:17 -0700915
Alex Klein1699fab2022-09-08 08:46:06 -0600916 # Create a hardlink clang++-X.Y pointing to clang-X.Y
917 os.link(os.path.join(clangdir, clang), os.path.join(clangdir, clangxx))
Rahul Chaudhrya8127bb2016-07-22 15:53:17 -0700918
Alex Klein1699fab2022-09-08 08:46:06 -0600919 # Adjust the clang++ symlink to point to clang++-X.Y
920 os.unlink(os.path.join(clangdir, "clang++"))
921 os.symlink(clangxx, os.path.join(clangdir, "clang++"))
Rahul Chaudhrya8127bb2016-07-22 15:53:17 -0700922
923
Mike Frysinger35247af2012-11-16 18:58:06 -0500924def FileIsCrosSdkElf(elf):
Alex Klein1699fab2022-09-08 08:46:06 -0600925 """Determine if |elf| is an ELF that we execute in the cros_sdk
Mike Frysinger35247af2012-11-16 18:58:06 -0500926
Alex Klein1699fab2022-09-08 08:46:06 -0600927 We don't need this to be perfect, just quick. It makes sure the ELF
928 is a 64bit LSB x86_64 ELF. That is the native type of cros_sdk.
Mike Frysinger35247af2012-11-16 18:58:06 -0500929
Alex Klein1699fab2022-09-08 08:46:06 -0600930 Args:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700931 elf: The file to check
Mike Frysinger1a736a82013-12-12 01:50:59 -0500932
Alex Klein1699fab2022-09-08 08:46:06 -0600933 Returns:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700934 True if we think |elf| is a native ELF
Alex Klein1699fab2022-09-08 08:46:06 -0600935 """
936 with open(elf, "rb") as f:
937 data = f.read(20)
938 # Check the magic number, EI_CLASS, EI_DATA, and e_machine.
939 return (
940 data[0:4] == b"\x7fELF"
941 and data[4:5] == b"\x02"
942 and data[5:6] == b"\x01"
943 and data[18:19] == b"\x3e"
944 )
Mike Frysinger35247af2012-11-16 18:58:06 -0500945
946
947def IsPathPackagable(ptype, path):
Alex Klein1699fab2022-09-08 08:46:06 -0600948 """Should the specified file be included in a toolchain package?
Mike Frysinger35247af2012-11-16 18:58:06 -0500949
Alex Klein1699fab2022-09-08 08:46:06 -0600950 We only need to handle files as we'll create dirs as we need them.
Mike Frysinger35247af2012-11-16 18:58:06 -0500951
Alex Klein1699fab2022-09-08 08:46:06 -0600952 Further, trim files that won't be useful:
953 - non-english translations (.mo) since it'd require env vars
954 - debug files since these are for the host compiler itself
955 - info/man pages as they're big, and docs are online, and the
956 native docs should work fine for the most part (`man gcc`)
Mike Frysinger35247af2012-11-16 18:58:06 -0500957
Alex Klein1699fab2022-09-08 08:46:06 -0600958 Args:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700959 ptype: A string describing the path type (i.e. 'file' or 'dir' or 'sym')
960 path: The full path to inspect
Mike Frysinger1a736a82013-12-12 01:50:59 -0500961
Alex Klein1699fab2022-09-08 08:46:06 -0600962 Returns:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700963 True if we want to include this path in the package
Alex Klein1699fab2022-09-08 08:46:06 -0600964 """
965 return not (
966 ptype in ("dir",)
967 or path.startswith("/usr/lib/debug/")
968 or os.path.splitext(path)[1] == ".mo"
969 or ("/man/" in path or "/info/" in path)
970 )
Mike Frysinger35247af2012-11-16 18:58:06 -0500971
972
973def ReadlinkRoot(path, root):
Alex Klein1699fab2022-09-08 08:46:06 -0600974 """Like os.readlink(), but relative to a |root|
Mike Frysinger35247af2012-11-16 18:58:06 -0500975
Alex Klein1699fab2022-09-08 08:46:06 -0600976 Args:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700977 path: The symlink to read
978 root: The path to use for resolving absolute symlinks
Mike Frysinger1a736a82013-12-12 01:50:59 -0500979
Alex Klein1699fab2022-09-08 08:46:06 -0600980 Returns:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700981 A fully resolved symlink path
Alex Klein1699fab2022-09-08 08:46:06 -0600982 """
983 while os.path.islink(root + path):
984 path = os.path.join(os.path.dirname(path), os.readlink(root + path))
985 return path
Mike Frysinger35247af2012-11-16 18:58:06 -0500986
987
Alex Klein1699fab2022-09-08 08:46:06 -0600988def _GetFilesForTarget(target, root="/"):
989 """Locate all the files to package for |target|
Mike Frysinger35247af2012-11-16 18:58:06 -0500990
Alex Klein1699fab2022-09-08 08:46:06 -0600991 This does not cover ELF dependencies.
Mike Frysinger35247af2012-11-16 18:58:06 -0500992
Alex Klein1699fab2022-09-08 08:46:06 -0600993 Args:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700994 target: The toolchain target name
995 root: The root path to pull all packages from
Mike Frysinger1a736a82013-12-12 01:50:59 -0500996
Alex Klein1699fab2022-09-08 08:46:06 -0600997 Returns:
Alex Klein0e92b2c2023-01-13 11:54:15 -0700998 A tuple of a set of all packable paths, and a set of all paths which
999 are also native ELFs
Alex Klein1699fab2022-09-08 08:46:06 -06001000 """
1001 paths = set()
1002 elfs = set()
Mike Frysinger35247af2012-11-16 18:58:06 -05001003
Alex Klein1699fab2022-09-08 08:46:06 -06001004 # Find all the files owned by the packages for this target.
1005 for pkg in GetTargetPackages(target):
Mike Frysinger35247af2012-11-16 18:58:06 -05001006
Alex Klein1699fab2022-09-08 08:46:06 -06001007 # Skip Go compiler from redistributable packages.
1008 # The "go" executable has GOROOT=/usr/lib/go/${CTARGET} hardcoded
1009 # into it. Due to this, the toolchain cannot be unpacked anywhere
1010 # else and be readily useful. To enable packaging Go, we need to:
1011 # -) Tweak the wrappers/environment to override GOROOT
1012 # automatically based on the unpack location.
1013 # -) Make sure the ELF dependency checking and wrapping logic
1014 # below skips the Go toolchain executables and libraries.
1015 # -) Make sure the packaging process maintains the relative
1016 # timestamps of precompiled standard library packages.
1017 # (see dev-lang/go ebuild for details).
1018 if pkg == "ex_go":
1019 continue
Rahul Chaudhry4b803052015-05-13 15:25:56 -07001020
Alex Klein1699fab2022-09-08 08:46:06 -06001021 # Use armv7a-cros-linux-gnueabi/compiler-rt for
Alex Klein0e92b2c2023-01-13 11:54:15 -07001022 # armv7a-cros-linux-gnueabihf/compiler-rt. Currently the
1023 # armv7a-cros-linux-gnueabi is actually the same as
1024 # armv7a-cros-linux-gnueabihf with different names. Because of that, for
1025 # compiler-rt, it generates the same binary in the same location. To
1026 # avoid the installation conflict, we do not install anything for
1027 # 'armv7a-cros-linux-gnueabihf'. This would cause problem if other
1028 # people try to use standalone armv7a-cros-linux-gnueabihf toolchain.
Alex Klein1699fab2022-09-08 08:46:06 -06001029 if "compiler-rt" in pkg and "armv7a-cros-linux-gnueabi" in target:
1030 atom = GetPortagePackage(target, pkg)
1031 cat, pn = atom.split("/")
1032 ver = GetInstalledPackageVersions(atom, root=root)[0]
1033 dblink = portage.dblink(
1034 cat, pn + "-" + ver, myroot=root, settings=portage.settings
1035 )
1036 contents = dblink.getcontents()
1037 if not contents:
1038 if "hf" in target:
1039 new_target = "armv7a-cros-linux-gnueabi"
1040 else:
1041 new_target = "armv7a-cros-linux-gnueabihf"
1042 atom = GetPortagePackage(new_target, pkg)
Yunlian Jiang36f35242018-04-27 10:18:40 -07001043 else:
Alex Klein1699fab2022-09-08 08:46:06 -06001044 atom = GetPortagePackage(target, pkg)
Yunlian Jiang36f35242018-04-27 10:18:40 -07001045
Alex Klein1699fab2022-09-08 08:46:06 -06001046 cat, pn = atom.split("/")
1047 ver = GetInstalledPackageVersions(atom, root=root)[0]
1048 logging.info("packaging %s-%s", atom, ver)
Mike Frysinger35247af2012-11-16 18:58:06 -05001049
Alex Klein1699fab2022-09-08 08:46:06 -06001050 dblink = portage.dblink(
1051 cat, pn + "-" + ver, myroot=root, settings=portage.settings
1052 )
1053 contents = dblink.getcontents()
1054 for obj in contents:
1055 ptype = contents[obj][0]
1056 if not IsPathPackagable(ptype, obj):
1057 continue
Mike Frysinger35247af2012-11-16 18:58:06 -05001058
Alex Klein1699fab2022-09-08 08:46:06 -06001059 if ptype == "obj":
1060 # For native ELFs, we need to pull in their dependencies too.
1061 if FileIsCrosSdkElf(obj):
1062 logging.debug("Adding ELF %s", obj)
1063 elfs.add(obj)
1064 logging.debug("Adding path %s", obj)
1065 paths.add(obj)
Mike Frysinger35247af2012-11-16 18:58:06 -05001066
Alex Klein1699fab2022-09-08 08:46:06 -06001067 return paths, elfs
Mike Frysinger35247af2012-11-16 18:58:06 -05001068
1069
Alex Klein1699fab2022-09-08 08:46:06 -06001070def _BuildInitialPackageRoot(
1071 output_dir, paths, elfs, ldpaths, path_rewrite_func=lambda x: x, root="/"
1072):
1073 """Link in all packable files and their runtime dependencies
Mike Frysinger35247af2012-11-16 18:58:06 -05001074
Alex Klein1699fab2022-09-08 08:46:06 -06001075 This also wraps up executable ELFs with helper scripts.
Mike Frysinger35247af2012-11-16 18:58:06 -05001076
Alex Klein1699fab2022-09-08 08:46:06 -06001077 Args:
Alex Klein0e92b2c2023-01-13 11:54:15 -07001078 output_dir: The output directory to store files
1079 paths: All the files to include
1080 elfs: All the files which are ELFs (a subset of |paths|)
1081 ldpaths: A dict of static ldpath information
1082 path_rewrite_func: User callback to rewrite paths in output_dir
1083 root: The root path to pull all packages/files from
Alex Klein1699fab2022-09-08 08:46:06 -06001084 """
1085 # Link in all the files.
1086 sym_paths = {}
1087 for path in paths:
1088 new_path = path_rewrite_func(path)
1089 logging.debug("Transformed %s to %s", path, new_path)
1090 dst = output_dir + new_path
1091 osutils.SafeMakedirs(os.path.dirname(dst))
Mike Frysinger35247af2012-11-16 18:58:06 -05001092
Alex Klein1699fab2022-09-08 08:46:06 -06001093 # Is this a symlink which we have to rewrite or wrap?
1094 # Delay wrap check until after we have created all paths.
1095 src = root + path
1096 if os.path.islink(src):
1097 tgt = os.readlink(src)
1098 if os.path.sep in tgt:
1099 sym_paths[lddtree.normpath(ReadlinkRoot(src, root))] = new_path
Mike Frysinger35247af2012-11-16 18:58:06 -05001100
Alex Klein0e92b2c2023-01-13 11:54:15 -07001101 # Rewrite absolute links to relative and then generate the
1102 # symlink ourselves. All other symlinks can be hardlinked below.
Alex Klein1699fab2022-09-08 08:46:06 -06001103 if tgt[0] == "/":
1104 tgt = os.path.relpath(tgt, os.path.dirname(new_path))
1105 os.symlink(tgt, dst)
1106 continue
Mike Frysinger35247af2012-11-16 18:58:06 -05001107
Alex Klein1699fab2022-09-08 08:46:06 -06001108 logging.debug("Linking path %s -> %s", src, dst)
1109 os.link(src, dst)
Mike Frysinger35247af2012-11-16 18:58:06 -05001110
Alex Klein1699fab2022-09-08 08:46:06 -06001111 # Locate all the dependencies for all the ELFs. Stick them all in the
1112 # top level "lib" dir to make the wrapper simpler. This exact path does
1113 # not matter since we execute ldso directly, and we tell the ldso the
1114 # exact path to search for its libraries.
1115 libdir = os.path.join(output_dir, "lib")
1116 osutils.SafeMakedirs(libdir)
1117 donelibs = set()
1118 basenamelibs = set()
Manoj Gupta98e674d2022-10-05 00:31:41 +00001119 glibc_re = re.compile(r"/lib(c|pthread)[0-9.-]*\.so[0-9.-]*")
Alex Klein1699fab2022-09-08 08:46:06 -06001120 for elf in elfs:
1121 e = lddtree.ParseELF(elf, root=root, ldpaths=ldpaths)
1122 logging.debug("Parsed elf %s data: %s", elf, e)
1123 interp = e["interp"]
Mike Frysinger221bd822017-09-29 02:51:47 -04001124
Alex Klein0e92b2c2023-01-13 11:54:15 -07001125 # TODO(b/187786323): Drop this hack once libopcodes linkage is fixed.
Alex Klein1699fab2022-09-08 08:46:06 -06001126 if os.path.basename(elf).startswith("libopcodes-"):
1127 continue
Mike Frysinger35247af2012-11-16 18:58:06 -05001128
Alex Klein0e92b2c2023-01-13 11:54:15 -07001129 # Copy all the dependencies before we copy the program & generate
1130 # wrappers.
Alex Klein1699fab2022-09-08 08:46:06 -06001131 for lib, lib_data in e["libs"].items():
1132 src = path = lib_data["path"]
1133 if path is None:
1134 logging.warning("%s: could not locate %s", elf, lib)
1135 continue
Mike Frysinger9fe02342019-12-12 17:52:53 -05001136
Alex Klein1699fab2022-09-08 08:46:06 -06001137 # No need to try and copy the same source lib multiple times.
1138 if path in donelibs:
1139 continue
1140 donelibs.add(path)
Mike Frysinger9fe02342019-12-12 17:52:53 -05001141
Alex Klein0e92b2c2023-01-13 11:54:15 -07001142 # Die if we try to normalize different source libs with the same
1143 # basename.
Alex Klein1699fab2022-09-08 08:46:06 -06001144 if lib in basenamelibs:
1145 logging.error(
1146 "Multiple sources detected for %s:\n new: %s\n old: %s",
1147 os.path.join("/lib", lib),
1148 path,
1149 " ".join(
1150 x
1151 for x in donelibs
1152 if x != path and os.path.basename(x) == lib
1153 ),
1154 )
1155 # TODO(crbug.com/917193): Make this fatal.
1156 # cros_build_lib.Die('Unable to resolve lib conflicts')
1157 continue
1158 basenamelibs.add(lib)
Mike Frysinger35247af2012-11-16 18:58:06 -05001159
Alex Klein1699fab2022-09-08 08:46:06 -06001160 # Needed libs are the SONAME, but that is usually a symlink, not a
1161 # real file. So link in the target rather than the symlink itself.
1162 # We have to walk all the possible symlinks (SONAME could point to a
1163 # symlink which points to a symlink), and we have to handle absolute
1164 # ourselves (since we have a "root" argument).
1165 dst = os.path.join(libdir, os.path.basename(path))
1166 src = ReadlinkRoot(src, root)
Mike Frysinger35247af2012-11-16 18:58:06 -05001167
Alex Klein1699fab2022-09-08 08:46:06 -06001168 logging.debug("Linking lib %s -> %s", root + src, dst)
1169 os.link(root + src, dst)
Mike Frysinger35247af2012-11-16 18:58:06 -05001170
Alex Klein1699fab2022-09-08 08:46:06 -06001171 # Do not create wrapper for libc. crbug.com/766827
1172 if interp and not glibc_re.search(elf):
1173 # Generate a wrapper if it is executable.
1174 interp = os.path.join("/lib", os.path.basename(interp))
1175 lddtree.GenerateLdsoWrapper(
1176 output_dir,
1177 path_rewrite_func(elf),
1178 interp,
1179 libpaths=e["rpath"] + e["runpath"],
1180 )
1181 FixClangXXWrapper(output_dir, path_rewrite_func(elf))
Mike Frysinger00b129f2021-04-21 18:11:48 -04001182
Alex Klein1699fab2022-09-08 08:46:06 -06001183 # Wrap any symlinks to the wrapper.
1184 if elf in sym_paths:
1185 link = sym_paths[elf]
1186 GeneratePathWrapper(output_dir, link, elf)
Mike Frysinger00b129f2021-04-21 18:11:48 -04001187
Mike Frysinger35247af2012-11-16 18:58:06 -05001188
1189def _EnvdGetVar(envd, var):
Alex Klein1699fab2022-09-08 08:46:06 -06001190 """Given a Gentoo env.d file, extract a var from it
Mike Frysinger35247af2012-11-16 18:58:06 -05001191
Alex Klein1699fab2022-09-08 08:46:06 -06001192 Args:
Alex Klein0e92b2c2023-01-13 11:54:15 -07001193 envd: The env.d file to load (may be a glob path)
1194 var: The var to extract
Mike Frysinger1a736a82013-12-12 01:50:59 -05001195
Alex Klein1699fab2022-09-08 08:46:06 -06001196 Returns:
Alex Klein0e92b2c2023-01-13 11:54:15 -07001197 The value of |var|
Alex Klein1699fab2022-09-08 08:46:06 -06001198 """
1199 envds = glob.glob(envd)
1200 assert len(envds) == 1, "%s: should have exactly 1 env.d file" % envd
1201 envd = envds[0]
1202 return key_value_store.LoadFile(envd)[var]
Mike Frysinger35247af2012-11-16 18:58:06 -05001203
1204
1205def _ProcessBinutilsConfig(target, output_dir):
Alex Klein1699fab2022-09-08 08:46:06 -06001206 """Do what binutils-config would have done"""
1207 binpath = os.path.join("/bin", target + "-")
Mike Frysingerd4d40fd2014-11-06 17:30:57 -05001208
Alex Klein1699fab2022-09-08 08:46:06 -06001209 # Locate the bin dir holding the linker and perform some confidence checks
1210 binutils_bin_path = os.path.join(
1211 output_dir, "usr", toolchain.GetHostTuple(), target, "binutils-bin"
1212 )
1213 globpath = os.path.join(binutils_bin_path, "*")
1214 srcpath = glob.glob(globpath)
1215 assert len(srcpath) == 1, (
1216 "%s: matched more than one path. Is Gold enabled?" % globpath
1217 )
1218 srcpath = srcpath[0]
1219 ld_path = os.path.join(srcpath, "ld")
1220 assert os.path.exists(ld_path), "%s: linker is missing!" % ld_path
1221 ld_path = os.path.join(srcpath, "ld.bfd")
1222 assert os.path.exists(ld_path), "%s: linker is missing!" % ld_path
Rahul Chaudhry4891b4d2017-03-08 10:31:27 -08001223
Alex Klein1699fab2022-09-08 08:46:06 -06001224 srcpath = srcpath[len(output_dir) :]
1225 gccpath = os.path.join("/usr", "libexec", "gcc")
1226 for prog in os.listdir(output_dir + srcpath):
1227 # Skip binaries already wrapped.
1228 if not prog.endswith(".real"):
1229 GeneratePathWrapper(
1230 output_dir, binpath + prog, os.path.join(srcpath, prog)
1231 )
1232 GeneratePathWrapper(
1233 output_dir,
1234 os.path.join(gccpath, prog),
1235 os.path.join(srcpath, prog),
1236 )
Mike Frysinger35247af2012-11-16 18:58:06 -05001237
Alex Klein1699fab2022-09-08 08:46:06 -06001238 libpath = os.path.join("/usr", toolchain.GetHostTuple(), target, "lib")
1239 envd = os.path.join(output_dir, "etc", "env.d", "binutils", "*")
1240 srcpath = _EnvdGetVar(envd, "LIBPATH")
1241 os.symlink(
1242 os.path.relpath(srcpath, os.path.dirname(libpath)), output_dir + libpath
1243 )
Mike Frysinger35247af2012-11-16 18:58:06 -05001244
1245
1246def _ProcessGccConfig(target, output_dir):
Alex Klein1699fab2022-09-08 08:46:06 -06001247 """Do what gcc-config would have done"""
1248 binpath = "/bin"
1249 envd = os.path.join(output_dir, "etc", "env.d", "gcc", "*")
1250 srcpath = _EnvdGetVar(envd, "GCC_PATH")
1251 for prog in os.listdir(output_dir + srcpath):
1252 # Skip binaries already wrapped.
1253 if (
1254 not prog.endswith(".real")
1255 and not prog.endswith(".elf")
1256 and prog.startswith(target)
1257 ):
1258 GeneratePathWrapper(
1259 output_dir,
1260 os.path.join(binpath, prog),
1261 os.path.join(srcpath, prog),
1262 )
1263 return srcpath
Mike Frysinger35247af2012-11-16 18:58:06 -05001264
1265
Frank Henigman179ec7c2015-02-06 03:01:09 -05001266def _ProcessSysrootWrappers(_target, output_dir, srcpath):
Alex Klein1699fab2022-09-08 08:46:06 -06001267 """Remove chroot-specific things from our sysroot wrappers"""
1268 # Disable ccache since we know it won't work outside of chroot.
Tobias Boschddd16492019-08-14 09:29:54 -07001269
Alex Klein1699fab2022-09-08 08:46:06 -06001270 # Use the version of the wrapper that does not use ccache.
1271 for sysroot_wrapper in glob.glob(
1272 os.path.join(output_dir + srcpath, "sysroot_wrapper*.ccache")
1273 ):
1274 # Can't update the wrapper in place to not affect the chroot,
1275 # but only the extracted toolchain.
1276 os.unlink(sysroot_wrapper)
1277 shutil.copy(sysroot_wrapper[:-6] + "noccache", sysroot_wrapper)
1278 shutil.copy(
1279 sysroot_wrapper[:-6] + "noccache.elf", sysroot_wrapper + ".elf"
1280 )
Mike Frysinger35247af2012-11-16 18:58:06 -05001281
1282
Manoj Gupta61bf9db2020-03-23 21:28:04 -07001283def _ProcessClangWrappers(target, output_dir):
Alex Klein1699fab2022-09-08 08:46:06 -06001284 """Remove chroot-specific things from our sysroot wrappers"""
1285 clang_bin_path = "/usr/bin"
1286 # Disable ccache from clang wrappers.
1287 _ProcessSysrootWrappers(target, output_dir, clang_bin_path)
1288 GeneratePathWrapper(
1289 output_dir, f"/bin/{target}-clang", f"/usr/bin/{target}-clang"
1290 )
1291 GeneratePathWrapper(
1292 output_dir, f"/bin/{target}-clang++", f"/usr/bin/{target}-clang++"
1293 )
Manoj Gupta61bf9db2020-03-23 21:28:04 -07001294
1295
Yunlian Jiang5ad6b512017-09-20 09:27:45 -07001296def _CreateMainLibDir(target, output_dir):
Alex Klein1699fab2022-09-08 08:46:06 -06001297 """Create some lib dirs so that compiler can get the right Gcc paths"""
1298 osutils.SafeMakedirs(os.path.join(output_dir, "usr", target, "lib"))
1299 osutils.SafeMakedirs(os.path.join(output_dir, "usr", target, "usr/lib"))
Yunlian Jiang5ad6b512017-09-20 09:27:45 -07001300
1301
Manoj Guptadf8b3872022-01-13 11:57:36 -08001302def _CreateRemoteToolchainFile(output_dir):
Alex Klein1699fab2022-09-08 08:46:06 -06001303 """Create a remote_toolchain_inputs file for reclient/RBE"""
1304 # The inputs file lists all files/shared libraries needed to run clang.
1305 # All inputs are relative to location of clang binary and one input
1306 # location per line of file e.g.
1307 # clang-13.elf
1308 # clang++-13.elf
1309 # relative/path/to/clang/resource/directory
Manoj Guptadf8b3872022-01-13 11:57:36 -08001310
Alex Klein1699fab2022-09-08 08:46:06 -06001311 clang_path = os.path.join(output_dir, "usr/bin")
1312 # Add needed shared libraries and internal files e.g. allowlists.
1313 toolchain_inputs = ["../../lib"]
1314 clang_shared_dirs = glob.glob(
1315 os.path.join(output_dir, "usr/lib64/clang/*/share")
1316 )
1317 for clang_dir in clang_shared_dirs:
1318 toolchain_inputs.append(os.path.relpath(clang_dir, clang_path))
Manoj Guptadf8b3872022-01-13 11:57:36 -08001319
Alex Klein1699fab2022-09-08 08:46:06 -06001320 # Add actual clang binaries/wrappers.
1321 for clang_files in glob.glob(os.path.join(clang_path, "clang*-[0-9]*")):
1322 toolchain_inputs.append(os.path.basename(clang_files))
Manoj Guptadf8b3872022-01-13 11:57:36 -08001323
Mike Frysinger31fdddd2023-02-24 15:50:55 -05001324 with open(
1325 os.path.join(clang_path, "remote_toolchain_inputs"),
1326 "w",
1327 encoding="utf-8",
1328 ) as f:
Alex Klein1699fab2022-09-08 08:46:06 -06001329 f.writelines("%s\n" % line for line in toolchain_inputs)
Manoj Guptadf8b3872022-01-13 11:57:36 -08001330
1331
Mike Frysinger35247af2012-11-16 18:58:06 -05001332def _ProcessDistroCleanups(target, output_dir):
Alex Klein1699fab2022-09-08 08:46:06 -06001333 """Clean up the tree and remove all distro-specific requirements
Mike Frysinger35247af2012-11-16 18:58:06 -05001334
Alex Klein1699fab2022-09-08 08:46:06 -06001335 Args:
Alex Klein0e92b2c2023-01-13 11:54:15 -07001336 target: The toolchain target name
1337 output_dir: The output directory to clean up
Han Shen699ea192016-03-02 10:42:47 -08001338 """
Alex Klein1699fab2022-09-08 08:46:06 -06001339 _ProcessBinutilsConfig(target, output_dir)
1340 gcc_path = _ProcessGccConfig(target, output_dir)
1341 _ProcessSysrootWrappers(target, output_dir, gcc_path)
1342 _ProcessClangWrappers(target, output_dir)
1343 _CreateMainLibDir(target, output_dir)
1344 _CreateRemoteToolchainFile(output_dir)
Mike Frysinger35247af2012-11-16 18:58:06 -05001345
Alex Klein1699fab2022-09-08 08:46:06 -06001346 osutils.RmDir(os.path.join(output_dir, "etc"))
Mike Frysinger35247af2012-11-16 18:58:06 -05001347
1348
Alex Klein1699fab2022-09-08 08:46:06 -06001349def CreatePackagableRoot(target, output_dir, ldpaths, root="/"):
1350 """Setup a tree from the packages for the specified target
Mike Frysinger35247af2012-11-16 18:58:06 -05001351
Alex Klein1699fab2022-09-08 08:46:06 -06001352 This populates a path with all the files from toolchain packages so that
1353 a tarball can easily be generated from the result.
Mike Frysinger35247af2012-11-16 18:58:06 -05001354
Alex Klein1699fab2022-09-08 08:46:06 -06001355 Args:
Alex Klein0e92b2c2023-01-13 11:54:15 -07001356 target: The target to create a packagable root from
1357 output_dir: The output directory to place all the files
1358 ldpaths: A dict of static ldpath information
1359 root: The root path to pull all packages/files from
Alex Klein1699fab2022-09-08 08:46:06 -06001360 """
1361 # Find all the files owned by the packages for this target.
1362 paths, elfs = _GetFilesForTarget(target, root=root)
Mike Frysinger35247af2012-11-16 18:58:06 -05001363
Alex Klein1699fab2022-09-08 08:46:06 -06001364 # Link in all the package's files, any ELF dependencies, and wrap any
1365 # executable ELFs with helper scripts.
1366 def MoveUsrBinToBin(path):
1367 """Move /usr/bin to /bin so people can just use that toplevel dir
Mike Frysinger35247af2012-11-16 18:58:06 -05001368
Alex Klein0e92b2c2023-01-13 11:54:15 -07001369 Note we do not apply this to clang or rust; there is correlation between
Alex Klein1699fab2022-09-08 08:46:06 -06001370 clang's search path for libraries / inclusion and its installation path.
1371 """
1372 NO_MOVE_PATTERNS = ("clang", "rust", "cargo", "sysroot_wrapper")
1373 if path.startswith("/usr/bin/") and not any(
1374 x in path for x in NO_MOVE_PATTERNS
1375 ):
1376 return path[4:]
1377 return path
Mike Frysinger221bd822017-09-29 02:51:47 -04001378
Alex Klein1699fab2022-09-08 08:46:06 -06001379 _BuildInitialPackageRoot(
1380 output_dir,
1381 paths,
1382 elfs,
1383 ldpaths,
1384 path_rewrite_func=MoveUsrBinToBin,
1385 root=root,
1386 )
Mike Frysinger35247af2012-11-16 18:58:06 -05001387
Alex Klein1699fab2022-09-08 08:46:06 -06001388 # The packages, when part of the normal distro, have helper scripts
1389 # that setup paths and such. Since we are making this standalone, we
1390 # need to preprocess all that ourselves.
1391 _ProcessDistroCleanups(target, output_dir)
1392
1393
1394def CreatePackages(targets_wanted, output_dir, root="/"):
1395 """Create redistributable cross-compiler packages for the specified targets
1396
1397 This creates toolchain packages that should be usable in conjunction with
1398 a downloaded sysroot (created elsewhere).
1399
1400 Tarballs (one per target) will be created in $PWD.
1401
1402 Args:
Alex Klein0e92b2c2023-01-13 11:54:15 -07001403 targets_wanted: The targets to package up.
1404 output_dir: The directory to put the packages in.
1405 root: The root path to pull all packages/files from.
Alex Klein1699fab2022-09-08 08:46:06 -06001406 """
1407 logging.info("Writing tarballs to %s", output_dir)
1408 osutils.SafeMakedirs(output_dir)
1409 ldpaths = lddtree.LoadLdpaths(root)
1410 targets = ExpandTargets(targets_wanted)
1411
1412 with osutils.TempDir(prefix="create-packages") as tempdir:
1413 logging.debug("Using tempdir: %s", tempdir)
1414
Alex Klein0e92b2c2023-01-13 11:54:15 -07001415 # We have to split the root generation from the compression stages.
1416 # This is because we hardlink in all the files (to avoid overhead of
1417 # reading/writing the copies multiple times). But tar gets angry if a
1418 # file's hardlink count changes from when it starts reading a file to
1419 # when it finishes.
Alex Klein1699fab2022-09-08 08:46:06 -06001420 with parallel.BackgroundTaskRunner(CreatePackagableRoot) as queue:
1421 for target in targets:
1422 output_target_dir = os.path.join(tempdir, target)
1423 queue.put([target, output_target_dir, ldpaths, root])
1424
1425 # Build the tarball.
1426 with parallel.BackgroundTaskRunner(
1427 cros_build_lib.CreateTarball
1428 ) as queue:
1429 for target in targets:
1430 tar_file = os.path.join(output_dir, target + ".tar.xz")
1431 queue.put([tar_file, os.path.join(tempdir, target)])
Mike Frysinger35247af2012-11-16 18:58:06 -05001432
1433
Mike Frysinger07534cf2017-09-12 17:40:21 -04001434def GetParser():
Alex Klein1699fab2022-09-08 08:46:06 -06001435 """Return a command line parser."""
1436 parser = commandline.ArgumentParser(description=__doc__)
1437 parser.add_argument(
1438 "-u",
1439 "--nousepkg",
1440 action="store_false",
1441 dest="usepkg",
1442 default=True,
1443 help="Do not use prebuilt packages",
1444 )
1445 parser.add_argument(
1446 "-d",
1447 "--deleteold",
1448 action="store_true",
1449 dest="deleteold",
1450 default=False,
1451 help="Unmerge deprecated packages",
1452 )
1453 parser.add_argument(
1454 "-t",
1455 "--targets",
1456 dest="targets",
1457 default="sdk",
1458 help="Comma separated list of tuples. Special keywords "
1459 "'host', 'sdk', 'boards', and 'all' are "
1460 "allowed. Defaults to 'sdk'.",
1461 )
1462 parser.add_argument(
1463 "--include-boards",
1464 default="",
1465 metavar="BOARDS",
1466 help="Comma separated list of boards whose toolchains we "
1467 "will always include. Default: none",
1468 )
1469 parser.add_argument(
1470 "--hostonly",
1471 dest="hostonly",
1472 default=False,
1473 action="store_true",
1474 help="Only setup the host toolchain. "
1475 "Useful for bootstrapping chroot",
1476 )
1477 parser.add_argument(
1478 "--show-board-cfg",
1479 "--show-cfg",
1480 dest="cfg_name",
1481 default=None,
1482 help="Board to list toolchains tuples for",
1483 )
1484 parser.add_argument(
1485 "--show-packages",
1486 default=None,
1487 help="List all packages the specified target uses",
1488 )
1489 parser.add_argument(
1490 "--create-packages",
1491 action="store_true",
1492 default=False,
1493 help="Build redistributable packages",
1494 )
1495 parser.add_argument(
1496 "--output-dir",
1497 default=os.getcwd(),
1498 type="path",
1499 help="Output directory",
1500 )
1501 parser.add_argument(
1502 "--reconfig",
1503 default=False,
1504 action="store_true",
1505 help="Reload crossdev config and reselect toolchains",
1506 )
1507 parser.add_argument(
1508 "--sysroot",
1509 type="path",
1510 help="The sysroot in which to install the toolchains",
1511 )
1512 return parser
Zdenek Behan508dcce2011-12-05 15:39:32 +01001513
Mike Frysinger07534cf2017-09-12 17:40:21 -04001514
1515def main(argv):
Alex Klein1699fab2022-09-08 08:46:06 -06001516 parser = GetParser()
1517 options = parser.parse_args(argv)
1518 options.Freeze()
Zdenek Behan508dcce2011-12-05 15:39:32 +01001519
Alex Klein1699fab2022-09-08 08:46:06 -06001520 # Figure out what we're supposed to do and reject conflicting options.
1521 conflicting_options = (
1522 options.cfg_name,
1523 options.show_packages,
1524 options.create_packages,
1525 )
1526 if sum(bool(x) for x in conflicting_options) > 1:
1527 parser.error(
1528 "conflicting options: create-packages & show-packages & "
1529 "show-board-cfg"
1530 )
Mike Frysinger984d0622012-06-01 16:08:44 -04001531
Alex Klein1699fab2022-09-08 08:46:06 -06001532 targets_wanted = set(options.targets.split(","))
1533 boards_wanted = (
1534 set(options.include_boards.split(","))
1535 if options.include_boards
1536 else set()
1537 )
Mike Frysinger35247af2012-11-16 18:58:06 -05001538
Alex Klein1699fab2022-09-08 08:46:06 -06001539 if options.cfg_name:
1540 ShowConfig(options.cfg_name)
1541 elif options.show_packages is not None:
1542 cros_build_lib.AssertInsideChroot()
1543 target = options.show_packages
1544 Crossdev.Load(False)
1545 for package in GetTargetPackages(target):
1546 print(GetPortagePackage(target, package))
1547 elif options.create_packages:
1548 cros_build_lib.AssertInsideChroot()
1549 Crossdev.Load(False)
1550 CreatePackages(targets_wanted, options.output_dir)
1551 else:
1552 cros_build_lib.AssertInsideChroot()
1553 # This has to be always run as root.
1554 if osutils.IsNonRootUser():
1555 cros_build_lib.Die("this script must be run as root")
Mike Frysinger35247af2012-11-16 18:58:06 -05001556
Alex Klein1699fab2022-09-08 08:46:06 -06001557 Crossdev.Load(options.reconfig)
1558 root = options.sysroot or "/"
1559 UpdateToolchains(
1560 options.usepkg,
1561 options.deleteold,
1562 options.hostonly,
1563 options.reconfig,
1564 targets_wanted,
1565 boards_wanted,
1566 root=root,
1567 )
1568 Crossdev.Save()
Mike Frysinger35247af2012-11-16 18:58:06 -05001569
Alex Klein1699fab2022-09-08 08:46:06 -06001570 return 0