scripts: subtools builder - a starting point.
The starting point is the developer-run script that can build and
export a subtool with a single command.
This CL implements fetching a base SDK, setting it up for building
subtools, and the ability to install packages into the subtools
builder chroot.
BUG=b:277992359
TEST=./bin/build_sdk_subtools shellcheck
Change-Id: Ie30733fcb48f69a10f58fc9fc4b73aa37f7f5142
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/chromite/+/4617263
Reviewed-by: Mike Frysinger <vapier@chromium.org>
Commit-Queue: Trent Apted <tapted@chromium.org>
Tested-by: Trent Apted <tapted@chromium.org>
diff --git a/scripts/build_sdk_subtools.py b/scripts/build_sdk_subtools.py
new file mode 100644
index 0000000..e4dc311
--- /dev/null
+++ b/scripts/build_sdk_subtools.py
@@ -0,0 +1,355 @@
+# Copyright 2023 The ChromiumOS Authors
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""build_sdk_subtools rebuilds binary packages exported by the subtools builder.
+
+The build_sdk_subtools process takes (copies) an amd64-host base SDK, compiles
+and installs additional packages needed by subtools, then creates relocatable
+binary subtool bundles that can be consumed by other build hosts and developer
+machines.
+
+If build_sdk_subtools has already been invoked for the provided chroot, all
+non-toolchain packages in the subtools deptree that have updated revisions or
+changed USE flags will be rebuilt, along with reverse dependencies.
+
+Packages (e.g. an ebuild) provide manifests that describes how files, once
+installed, are to be bundled and exported.
+
+If packages are specified in the command line, only consider the deptree from
+those specific packages rather than all of virtual/target-sdk-subtools.
+"""
+
+import argparse
+import logging
+import os
+from pathlib import Path
+import shutil
+import sys
+from typing import Dict, List, Optional, Protocol, Union
+
+from chromite.lib import build_target_lib
+from chromite.lib import commandline
+from chromite.lib import constants
+from chromite.lib import cros_build_lib
+from chromite.lib import cros_sdk_lib
+from chromite.lib import osutils
+from chromite.lib import portage_util
+from chromite.lib import sysroot_lib
+from chromite.service import sysroot
+
+
+assert sys.version_info >= (3, 8), "build_sdk_subtools uses Python 3.8 features"
+
+
+# Affects where building occurs (e.g. /build/amd64-subtools-host) if not
+# overridden by --output-dir. Note this will be a chroot.
+SUBTOOLS_OUTPUT_DIR = "amd64-subtools-host"
+
+# Version file that identifies a chroot setup as a subtools chroot.
+SUBTOOLS_CHROOT_VERSION_FILE = Path("/etc/cros_subtools_chroot_version")
+
+# Packages that the subtools builder should never rebuild. This is a superset of
+# sysroot._CRITICAL_SDK_PACKAGES.
+EXCLUDE_PACKAGES = (
+ "dev-embedded/hps-sdk",
+ "dev-lang/rust",
+ "dev-lang/go",
+ "sys-libs/glibc",
+ "sys-devel/gcc",
+ "sys-devel/binutils",
+)
+
+# Flag passed to subprocesses in chroots that might not yet be set up as a
+# subtools chroot.
+_RELAUNCH_FOR_SETUP_FLAG = "--relaunch-for-setup"
+
+
+class Options(Protocol):
+ """Protocol to formalize commandline arguments."""
+
+ build_run_config: sysroot.BuildPackagesRunConfig
+ clean: bool
+ setup_chroot: bool
+ update_packages: bool
+ relaunch_for_setup: bool
+ output_dir: Path
+ packages: List[str]
+ jobs: int
+
+ def Freeze(self) -> None:
+ pass
+
+
+def get_parser() -> commandline.ArgumentParser:
+ """Returns the cmdline argparser, populates the options and descriptions."""
+ parser = commandline.ArgumentParser(description=__doc__)
+
+ def add_conforming_bool(flag: str, default: bool, desc: str, no_desc: str):
+ """Adds a boolean argument conforming to chromite recommendations.
+
+ See go/chromite-git/+/HEAD/docs/cli-guidelines.md#Boolean-Options.
+ """
+ desc += " (DEFAULT)" if default else ""
+ no_desc += " (DEFAULT)" if not default else ""
+ dest = flag.replace("-", "_")
+ parser.add_argument(
+ f"--{flag}", action="store_true", default=default, help=desc
+ )
+ parser.add_argument(
+ f"--no-{flag}", action="store_false", dest=dest, help=no_desc
+ )
+
+ add_conforming_bool(
+ "clean",
+ False,
+ "Remove the subtools chroot and re-extract the SDK.",
+ "Re-use an existing subtools chroot.",
+ )
+
+ add_conforming_bool(
+ "setup-chroot",
+ True,
+ "Look for a newer base SDK and set it up as a subtools SDK.",
+ "Don't look for a newer base SDK and assume the chroot is setup.",
+ )
+
+ add_conforming_bool(
+ "update-packages",
+ True,
+ "Update and install packages before looking for things to export.",
+ "Only export packages already installed in the subtools SDK.",
+ )
+
+ parser.add_argument(
+ "--output-dir",
+ type=osutils.ExpandPath,
+ metavar="PATH",
+ help=f"Extract SDK and build in chroot (e.g. {SUBTOOLS_OUTPUT_DIR}).",
+ )
+
+ parser.add_argument(
+ "packages",
+ nargs="*",
+ default=["virtual/target-sdk-subtools"],
+ help="Packages to build before looking for export candidates.",
+ )
+
+ parser.add_argument(
+ "--jobs",
+ "-j",
+ type=int,
+ default=os.cpu_count(),
+ help="Number of packages to build in parallel. (Default: %(default)s)",
+ )
+
+ parser.add_argument(
+ _RELAUNCH_FOR_SETUP_FLAG,
+ action="store_true",
+ default=False,
+ help=argparse.SUPPRESS,
+ )
+
+ # TODO(b/277992359): Consider possibly relevant flags from build_packages:
+ # * --rebuild_revdeps=no: don't rebuild reverse dependencies.
+ # * --skip-toolchain-update? Likely no - the SDK is our toolchain.
+ # * --withdebugsymbols
+ # * --rebuild=no "Automatically rebuild dependencies"
+ # * --backtrack
+ # * --bazel "Use Bazel to build packages"
+
+ return parser
+
+
+def parse_args(argv: Optional[List[str]]) -> Options:
+ """Parse and validate CLI arguments."""
+
+ parser = get_parser()
+ opts: Options = parser.parse_args(argv)
+
+ # Although `BuildPackages` is not used, sharing a config allows better
+ # sharing of subcommands and concepts.
+ opts.build_run_config = sysroot.BuildPackagesRunConfig(
+ packages=opts.packages,
+ jobs=opts.jobs,
+ usepkg=False,
+ clean_build=False,
+ eclean=False,
+ rebuild_dep=False,
+ )
+ opts.Freeze()
+ return opts
+
+
+def _is_inside_subtools_chroot() -> bool:
+ """Returns True if we are inside subtools chroot."""
+ return SUBTOOLS_CHROOT_VERSION_FILE.exists()
+
+
+def _assert_inside_subtools_chroot() -> None:
+ """Die if not _is_inside_subtools_chroot()."""
+ if not _is_inside_subtools_chroot():
+ cros_build_lib.Die("Not in subtools SDK")
+
+
+def _setup_base_sdk(
+ build_target: build_target_lib.BuildTarget,
+ setup_chroot: bool,
+) -> None:
+ """SetupBoard workalike that converts a regular SDK into a subtools chroot.
+
+ Runs inside the /build/amd64-subtools-host subtools SDK chroot.
+ """
+ cros_build_lib.AssertInsideChroot()
+ cros_build_lib.AssertRootUser()
+
+ # "Convert" the SDK into a subtools SDK.
+ if not _is_inside_subtools_chroot():
+ # Copy the sentinel file that chromite uses to indicate the chroot's
+ # duality. The file is copied (not moved) so that other chromite tooling
+ # continues to work.
+ shutil.copy(
+ cros_sdk_lib.CHROOT_VERSION_FILE, SUBTOOLS_CHROOT_VERSION_FILE
+ )
+
+ if setup_chroot:
+ # TODO(b/277992359): Additional setup here, e.g., packages, base layout.
+ logging.info("Setting up subtools SDK in %s.", build_target.root)
+
+
+def _run_system_emerge(
+ emerge_cmd: List[Union[str, Path]],
+ extra_env: Dict[str, str],
+ use_goma: bool,
+ use_remoteexec: bool,
+ reason: str,
+) -> None:
+ """Runs an emerge command, updating the live system."""
+ extra_env = extra_env.copy()
+ with osutils.TempDir() as tempdir:
+ extra_env[constants.CROS_METRICS_DIR_ENVVAR] = tempdir
+ with sysroot.RemoteExecution(use_goma, use_remoteexec):
+ logging.info("Merging %s now.", reason)
+ try:
+ # TODO(b/277992359): Bazel.
+ cros_build_lib.sudo_run(
+ emerge_cmd,
+ preserve_env=True,
+ extra_env=extra_env,
+ )
+ logging.info("Merging %s complete.", reason)
+ except cros_build_lib.RunCommandError as e:
+ failed_pkgs = portage_util.ParseDieHookStatusFile(tempdir)
+ logging.error("Merging %s failed on %s", reason, failed_pkgs)
+ raise sysroot_lib.PackageInstallError(
+ f"Merging {reason} failed",
+ e.result,
+ exception=e,
+ packages=failed_pkgs,
+ )
+
+
+def _build_sdk_packages(config: sysroot.BuildPackagesRunConfig) -> None:
+ """The BuildPackages workalike for installing into the staging SDK."""
+ _assert_inside_subtools_chroot()
+ cros_build_lib.AssertNonRootUser()
+
+ try:
+ # sysroot.BuildPackages can't (yet?) be used here, because it _only_
+ # supports cross-compilation. SDK package management is currently all
+ # handled by src/scripts/sdk_lib/make_chroot.sh (b/191307774).
+
+ emerge = [constants.CHROMITE_BIN_DIR / "parallel_emerge"]
+ extra_env = config.GetExtraEnv()
+ emerge_flags = config.GetEmergeFlags()
+ exclude_pkgs = " ".join(EXCLUDE_PACKAGES)
+ emerge_flags.extend(
+ [
+ f"--useoldpkg-atoms={exclude_pkgs}",
+ f"--rebuild-exclude={exclude_pkgs}",
+ ]
+ )
+ cmd = emerge + emerge_flags + config.GetPackages()
+ _run_system_emerge(
+ cmd,
+ extra_env,
+ config.use_goma,
+ config.use_remoteexec,
+ reason="subtools builder SDK packages",
+ )
+
+ except sysroot_lib.PackageInstallError as e:
+ cros_build_lib.Die(e)
+
+
+def main(argv: Optional[List[str]] = None) -> Optional[int]:
+ opts = parse_args(argv)
+ return build_sdk_subtools(opts, argv if argv else [])
+
+
+def build_sdk_subtools(opts: Options, argv: List[str]) -> int:
+ """Executes SDK subtools builder steps according to `opts`."""
+ # BuildTarget needs a str, but opts.output_dir is osutils.ExpandPath.
+ custom_output_dir = str(opts.output_dir) if opts.output_dir else None
+ build_target = build_target_lib.BuildTarget(
+ name=SUBTOOLS_OUTPUT_DIR, build_root=custom_output_dir
+ )
+
+ # If the process is in the subtools chroot, we must assume it's already set
+ # up (we are in it). So start building.
+ if _is_inside_subtools_chroot() and not opts.relaunch_for_setup:
+ if opts.update_packages:
+ _build_sdk_packages(opts.build_run_config)
+ return 0
+
+ # Otherwise, we have the option to set it up. Then restart inside it. The
+ # setup runs `cros_sdk` to get a base SDK, creates an SDK subprocess to set
+ # it up as a subtools SDK, then restarts inside the subtools SDK.
+ if cros_build_lib.IsInsideChroot():
+ if opts.relaunch_for_setup:
+ # This is the subprocess of the not-in-chroot path used to convert
+ # the base SDK to a subtools SDK (within the chroot).
+ _setup_base_sdk(build_target, opts.setup_chroot)
+ return 0
+ else:
+ cros_build_lib.Die(
+ "build_sdk_subtools must be run outside the chroot."
+ )
+
+ logging.info("Initializing subtools builder in %s", build_target.root)
+ subtools_chroot = os.path.join(
+ constants.DEFAULT_CHROOT_PATH, build_target.root.lstrip("/")
+ )
+ chroot_args = ["--chroot", subtools_chroot]
+
+ if opts.setup_chroot:
+ # Get an SDK. TODO(b/277992359):
+ # - Fetch an SDK version pinned by pupr rather than the default.
+ # - Should this use cros_sdk_lib directly?
+
+ # Pass "--skip-chroot-upgrade": the SDK should initially be used
+ # "as-is", but later steps may upgrade packages in the subtools deptree.
+ cros_sdk_args = ["--create", "--skip-chroot-upgrade"]
+ cros_sdk_args += ["--delete"] if opts.clean else []
+ cros_sdk = cros_build_lib.run(
+ ["cros_sdk"] + chroot_args + cros_sdk_args,
+ check=False,
+ cwd=constants.SOURCE_ROOT,
+ )
+ if cros_sdk.returncode != 0:
+ return cros_sdk.returncode
+
+ # Invoke `_setup_base_sdk()` inside the SDK.
+ setup_base_sdk = cros_build_lib.sudo_run(
+ ["build_sdk_subtools"] + argv + [_RELAUNCH_FOR_SETUP_FLAG],
+ check=False,
+ enter_chroot=True,
+ chroot_args=chroot_args,
+ cwd=constants.SOURCE_ROOT,
+ )
+ if setup_base_sdk.returncode != 0:
+ return setup_base_sdk.returncode
+
+ raise commandline.ChrootRequiredError(
+ ["build_sdk_subtools"] + argv, chroot_args=chroot_args
+ )