scripts/build_sdk_subtools.py - chromium.googlesource.com/chromiumos/chromite - Gitiles

 # Copyright 2023 The ChromiumOS Authors
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.

 """build_sdk_subtools rebuilds binary packages exported by the subtools builder.

 The build_sdk_subtools process takes (copies) an amd64-host base SDK, compiles
 and installs additional packages needed by subtools, then creates relocatable
 binary subtool bundles that can be consumed by other build hosts and developer
 machines.

 If build_sdk_subtools has already been invoked for the provided chroot, all
 non-toolchain packages in the subtools deptree that have updated revisions or
 changed USE flags will be rebuilt, along with reverse dependencies.

 Packages (e.g. an ebuild) provide manifests that describes how files, once
 installed, are to be bundled and exported.

 If packages are specified in the command line, only consider the deptree from
 those specific packages rather than all of virtual/target-sdk-subtools.
 """

 import argparse
 import logging
 import os
 from pathlib import Path
 import shutil
 import sys
 from typing import Dict, List, Optional, Protocol, Union

 from chromite.lib import build_target_lib
 from chromite.lib import commandline
 from chromite.lib import constants
 from chromite.lib import cros_build_lib
 from chromite.lib import cros_sdk_lib
 from chromite.lib import osutils
 from chromite.lib import portage_util
 from chromite.lib import sysroot_lib
 from chromite.service import sysroot


 assert sys.version_info >= (3, 8), "build_sdk_subtools uses Python 3.8 features"


 # Affects where building occurs (e.g. /build/amd64-subtools-host) if not
 # overridden by --output-dir. Note this will be a chroot.
 SUBTOOLS_OUTPUT_DIR = "amd64-subtools-host"

 # Version file that identifies a chroot setup as a subtools chroot.
 SUBTOOLS_CHROOT_VERSION_FILE = Path("/etc/cros_subtools_chroot_version")

 # Packages that the subtools builder should never rebuild. This is a superset of
 # sysroot._CRITICAL_SDK_PACKAGES.
 EXCLUDE_PACKAGES = (
     "dev-embedded/hps-sdk",
     "dev-lang/rust",
     "dev-lang/go",
     "sys-libs/glibc",
     "sys-devel/gcc",
     "sys-devel/binutils",
 )

 # Flag passed to subprocesses in chroots that might not yet be set up as a
 # subtools chroot.
 _RELAUNCH_FOR_SETUP_FLAG = "--relaunch-for-setup"


 class Options(Protocol):
     """Protocol to formalize commandline arguments."""

     build_run_config: sysroot.BuildPackagesRunConfig
     clean: bool
     setup_chroot: bool
     update_packages: bool
     relaunch_for_setup: bool
     output_dir: Path
     packages: List[str]
     jobs: int

     def Freeze(self) -> None:
         pass


 def get_parser() -> commandline.ArgumentParser:
     """Returns the cmdline argparser, populates the options and descriptions."""
     parser = commandline.ArgumentParser(description=__doc__)

     def add_conforming_bool(flag: str, default: bool, desc: str, no_desc: str):
         """Adds a boolean argument conforming to chromite recommendations.

         See go/chromite-git/+/HEAD/docs/cli-guidelines.md#Boolean-Options.
         """
         desc += " (DEFAULT)" if default else ""
         no_desc += " (DEFAULT)" if not default else ""
         dest = flag.replace("-", "_")
         parser.add_argument(
             f"--{flag}", action="store_true", default=default, help=desc
         )
         parser.add_argument(
             f"--no-{flag}", action="store_false", dest=dest, help=no_desc
         )

     add_conforming_bool(
         "clean",
         False,
         "Remove the subtools chroot and re-extract the SDK.",
         "Re-use an existing subtools chroot.",
     )

     add_conforming_bool(
         "setup-chroot",
         True,
         "Look for a newer base SDK and set it up as a subtools SDK.",
         "Don't look for a newer base SDK and assume the chroot is setup.",
     )

     add_conforming_bool(
         "update-packages",
         True,
         "Update and install packages before looking for things to export.",
         "Only export packages already installed in the subtools SDK.",
     )

     parser.add_argument(
         "--output-dir",
         type=osutils.ExpandPath,
         metavar="PATH",
         help=f"Extract SDK and build in chroot (e.g. {SUBTOOLS_OUTPUT_DIR}).",
     )

     parser.add_argument(
         "packages",
         nargs="*",
         default=["virtual/target-sdk-subtools"],
         help="Packages to build before looking for export candidates.",
     )

     parser.add_argument(
         "--jobs",
         "-j",
         type=int,
         default=os.cpu_count(),
         help="Number of packages to build in parallel. (Default: %(default)s)",
     )

     parser.add_argument(
         _RELAUNCH_FOR_SETUP_FLAG,
         action="store_true",
         default=False,
         help=argparse.SUPPRESS,
     )

     # TODO(b/277992359): Consider possibly relevant flags from build_packages:
     #  * --rebuild_revdeps=no: don't rebuild reverse dependencies.
     #  * --skip-toolchain-update? Likely no - the SDK is our toolchain.
     #  * --withdebugsymbols
     #  * --rebuild=no "Automatically rebuild dependencies"
     #  * --backtrack
     #  * --bazel  "Use Bazel to build packages"

     return parser


 def parse_args(argv: Optional[List[str]]) -> Options:
     """Parse and validate CLI arguments."""

     parser = get_parser()
     opts: Options = parser.parse_args(argv)

     # Although `BuildPackages` is not used, sharing a config allows better
     # sharing of subcommands and concepts.
     opts.build_run_config = sysroot.BuildPackagesRunConfig(
         packages=opts.packages,
         jobs=opts.jobs,
         usepkg=False,
         clean_build=False,
         eclean=False,
         rebuild_dep=False,
     )
     opts.Freeze()
     return opts


 def _is_inside_subtools_chroot() -> bool:
     """Returns True if we are inside subtools chroot."""
     return SUBTOOLS_CHROOT_VERSION_FILE.exists()


 def _assert_inside_subtools_chroot() -> None:
     """Die if not _is_inside_subtools_chroot()."""
     if not _is_inside_subtools_chroot():
         cros_build_lib.Die("Not in subtools SDK")


 def _setup_base_sdk(
     build_target: build_target_lib.BuildTarget,
     setup_chroot: bool,
 ) -> None:
     """SetupBoard workalike that converts a regular SDK into a subtools chroot.

     Runs inside the /build/amd64-subtools-host subtools SDK chroot.
     """
     cros_build_lib.AssertInsideChroot()
     cros_build_lib.AssertRootUser()

     # "Convert" the SDK into a subtools SDK.
     if not _is_inside_subtools_chroot():
         # Copy the sentinel file that chromite uses to indicate the chroot's
         # duality. The file is copied (not moved) so that other chromite tooling
         # continues to work.
         shutil.copy(
             cros_sdk_lib.CHROOT_VERSION_FILE, SUBTOOLS_CHROOT_VERSION_FILE
         )

     if setup_chroot:
         # TODO(b/277992359): Additional setup here, e.g., packages, base layout.
         logging.info("Setting up subtools SDK in %s.", build_target.root)


 def _run_system_emerge(
     emerge_cmd: List[Union[str, Path]],
     extra_env: Dict[str, str],
     use_goma: bool,
     use_remoteexec: bool,
     reason: str,
 ) -> None:
     """Runs an emerge command, updating the live system."""
     extra_env = extra_env.copy()
     with osutils.TempDir() as tempdir:
         extra_env[constants.CROS_METRICS_DIR_ENVVAR] = tempdir
         with sysroot.RemoteExecution(use_goma, use_remoteexec):
             logging.info("Merging %s now.", reason)
             try:
                 # TODO(b/277992359): Bazel.
                 cros_build_lib.sudo_run(
                     emerge_cmd,
                     preserve_env=True,
                     extra_env=extra_env,
                 )
                 logging.info("Merging %s complete.", reason)
             except cros_build_lib.RunCommandError as e:
                 failed_pkgs = portage_util.ParseDieHookStatusFile(tempdir)
                 logging.error("Merging %s failed on %s", reason, failed_pkgs)
                 raise sysroot_lib.PackageInstallError(
                     f"Merging {reason} failed",
                     e.result,
                     exception=e,
                     packages=failed_pkgs,
                 )


 def _build_sdk_packages(config: sysroot.BuildPackagesRunConfig) -> None:
     """The BuildPackages workalike for installing into the staging SDK."""
     _assert_inside_subtools_chroot()
     cros_build_lib.AssertNonRootUser()

     try:
         # sysroot.BuildPackages can't (yet?) be used here, because it _only_
         # supports cross-compilation. SDK package management is currently all
         # handled by src/scripts/sdk_lib/make_chroot.sh (b/191307774).

         emerge = [constants.CHROMITE_BIN_DIR / "parallel_emerge"]
         extra_env = config.GetExtraEnv()
         emerge_flags = config.GetEmergeFlags()
         exclude_pkgs = " ".join(EXCLUDE_PACKAGES)
         emerge_flags.extend(
             [
                 f"--useoldpkg-atoms={exclude_pkgs}",
                 f"--rebuild-exclude={exclude_pkgs}",
             ]
         )
         cmd = emerge + emerge_flags + config.GetPackages()
         _run_system_emerge(
             cmd,
             extra_env,
             config.use_goma,
             config.use_remoteexec,
             reason="subtools builder SDK packages",
         )

     except sysroot_lib.PackageInstallError as e:
         cros_build_lib.Die(e)


 def main(argv: Optional[List[str]] = None) -> Optional[int]:
     opts = parse_args(argv)
     return build_sdk_subtools(opts, argv if argv else [])


 def build_sdk_subtools(opts: Options, argv: List[str]) -> int:
     """Executes SDK subtools builder steps according to `opts`."""
     # BuildTarget needs a str, but opts.output_dir is osutils.ExpandPath.
     custom_output_dir = str(opts.output_dir) if opts.output_dir else None
     build_target = build_target_lib.BuildTarget(
         name=SUBTOOLS_OUTPUT_DIR, build_root=custom_output_dir
     )

     # If the process is in the subtools chroot, we must assume it's already set
     # up (we are in it). So start building.
     if _is_inside_subtools_chroot() and not opts.relaunch_for_setup:
         if opts.update_packages:
             _build_sdk_packages(opts.build_run_config)
         return 0

     # Otherwise, we have the option to set it up. Then restart inside it. The
     # setup runs `cros_sdk` to get a base SDK, creates an SDK subprocess to set
     # it up as a subtools SDK, then restarts inside the subtools SDK.
     if cros_build_lib.IsInsideChroot():
         if opts.relaunch_for_setup:
             # This is the subprocess of the not-in-chroot path used to convert
             # the base SDK to a subtools SDK (within the chroot).
             _setup_base_sdk(build_target, opts.setup_chroot)
             return 0
         else:
             cros_build_lib.Die(
                 "build_sdk_subtools must be run outside the chroot."
             )

     logging.info("Initializing subtools builder in %s", build_target.root)
     subtools_chroot = os.path.join(
         constants.DEFAULT_CHROOT_PATH, build_target.root.lstrip("/")
     )
     chroot_args = ["--chroot", subtools_chroot]

     if opts.setup_chroot:
         # Get an SDK. TODO(b/277992359):
         #   - Fetch an SDK version pinned by pupr rather than the default.
         #   - Should this use cros_sdk_lib directly?

         # Pass "--skip-chroot-upgrade": the SDK should initially be used
         # "as-is", but later steps may upgrade packages in the subtools deptree.
         cros_sdk_args = ["--create", "--skip-chroot-upgrade"]
         cros_sdk_args += ["--delete"] if opts.clean else []
         cros_sdk = cros_build_lib.run(
             ["cros_sdk"] + chroot_args + cros_sdk_args,
             check=False,
             cwd=constants.SOURCE_ROOT,
         )
         if cros_sdk.returncode != 0:
             return cros_sdk.returncode

         # Invoke `_setup_base_sdk()` inside the SDK.
         setup_base_sdk = cros_build_lib.sudo_run(
             ["build_sdk_subtools"] + argv + [_RELAUNCH_FOR_SETUP_FLAG],
             check=False,
             enter_chroot=True,
             chroot_args=chroot_args,
             cwd=constants.SOURCE_ROOT,
         )
         if setup_base_sdk.returncode != 0:
             return setup_base_sdk.returncode

     raise commandline.ChrootRequiredError(
         ["build_sdk_subtools"] + argv, chroot_args=chroot_args
     )
	# Copyright 2023 The ChromiumOS Authors
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.

	"""build_sdk_subtools rebuilds binary packages exported by the subtools builder.

	The build_sdk_subtools process takes (copies) an amd64-host base SDK, compiles
	and installs additional packages needed by subtools, then creates relocatable
	binary subtool bundles that can be consumed by other build hosts and developer
	machines.

	If build_sdk_subtools has already been invoked for the provided chroot, all
	non-toolchain packages in the subtools deptree that have updated revisions or
	changed USE flags will be rebuilt, along with reverse dependencies.

	Packages (e.g. an ebuild) provide manifests that describes how files, once
	installed, are to be bundled and exported.

	If packages are specified in the command line, only consider the deptree from
	those specific packages rather than all of virtual/target-sdk-subtools.
	"""

	import argparse
	import logging
	import os
	from pathlib import Path
	import shutil
	import sys
	from typing import Dict, List, Optional, Protocol, Union

	from chromite.lib import build_target_lib
	from chromite.lib import commandline
	from chromite.lib import constants
	from chromite.lib import cros_build_lib
	from chromite.lib import cros_sdk_lib
	from chromite.lib import osutils
	from chromite.lib import portage_util
	from chromite.lib import sysroot_lib
	from chromite.service import sysroot


	assert sys.version_info >= (3, 8), "build_sdk_subtools uses Python 3.8 features"


	# Affects where building occurs (e.g. /build/amd64-subtools-host) if not
	# overridden by --output-dir. Note this will be a chroot.
	SUBTOOLS_OUTPUT_DIR = "amd64-subtools-host"

	# Version file that identifies a chroot setup as a subtools chroot.
	SUBTOOLS_CHROOT_VERSION_FILE = Path("/etc/cros_subtools_chroot_version")

	# Packages that the subtools builder should never rebuild. This is a superset of
	# sysroot._CRITICAL_SDK_PACKAGES.
	EXCLUDE_PACKAGES = (
	"dev-embedded/hps-sdk",
	"dev-lang/rust",
	"dev-lang/go",
	"sys-libs/glibc",
	"sys-devel/gcc",
	"sys-devel/binutils",
	)

	# Flag passed to subprocesses in chroots that might not yet be set up as a
	# subtools chroot.
	_RELAUNCH_FOR_SETUP_FLAG = "--relaunch-for-setup"


	class Options(Protocol):
	"""Protocol to formalize commandline arguments."""

	build_run_config: sysroot.BuildPackagesRunConfig
	clean: bool
	setup_chroot: bool
	update_packages: bool
	relaunch_for_setup: bool
	output_dir: Path
	packages: List[str]
	jobs: int

	def Freeze(self) -> None:
	pass


	def get_parser() -> commandline.ArgumentParser:
	"""Returns the cmdline argparser, populates the options and descriptions."""
	parser = commandline.ArgumentParser(description=__doc__)

	def add_conforming_bool(flag: str, default: bool, desc: str, no_desc: str):
	"""Adds a boolean argument conforming to chromite recommendations.

	See go/chromite-git/+/HEAD/docs/cli-guidelines.md#Boolean-Options.
	"""
	desc += " (DEFAULT)" if default else ""
	no_desc += " (DEFAULT)" if not default else ""
	dest = flag.replace("-", "_")
	parser.add_argument(
	f"--{flag}", action="store_true", default=default, help=desc
	)
	parser.add_argument(
	f"--no-{flag}", action="store_false", dest=dest, help=no_desc
	)

	add_conforming_bool(
	"clean",
	False,
	"Remove the subtools chroot and re-extract the SDK.",
	"Re-use an existing subtools chroot.",
	)

	add_conforming_bool(
	"setup-chroot",
	True,
	"Look for a newer base SDK and set it up as a subtools SDK.",
	"Don't look for a newer base SDK and assume the chroot is setup.",
	)

	add_conforming_bool(
	"update-packages",
	True,
	"Update and install packages before looking for things to export.",
	"Only export packages already installed in the subtools SDK.",
	)

	parser.add_argument(
	"--output-dir",
	type=osutils.ExpandPath,
	metavar="PATH",
	help=f"Extract SDK and build in chroot (e.g. {SUBTOOLS_OUTPUT_DIR}).",
	)

	parser.add_argument(
	"packages",
	nargs="*",
	default=["virtual/target-sdk-subtools"],
	help="Packages to build before looking for export candidates.",
	)

	parser.add_argument(
	"--jobs",
	"-j",
	type=int,
	default=os.cpu_count(),
	help="Number of packages to build in parallel. (Default: %(default)s)",
	)

	parser.add_argument(
	_RELAUNCH_FOR_SETUP_FLAG,
	action="store_true",
	default=False,
	help=argparse.SUPPRESS,
	)

	# TODO(b/277992359): Consider possibly relevant flags from build_packages:
	# * --rebuild_revdeps=no: don't rebuild reverse dependencies.
	# * --skip-toolchain-update? Likely no - the SDK is our toolchain.
	# * --withdebugsymbols
	# * --rebuild=no "Automatically rebuild dependencies"
	# * --backtrack
	# * --bazel "Use Bazel to build packages"

	return parser


	def parse_args(argv: Optional[List[str]]) -> Options:
	"""Parse and validate CLI arguments."""

	parser = get_parser()
	opts: Options = parser.parse_args(argv)

	# Although `BuildPackages` is not used, sharing a config allows better
	# sharing of subcommands and concepts.
	opts.build_run_config = sysroot.BuildPackagesRunConfig(
	packages=opts.packages,
	jobs=opts.jobs,
	usepkg=False,
	clean_build=False,
	eclean=False,
	rebuild_dep=False,
	)
	opts.Freeze()
	return opts


	def _is_inside_subtools_chroot() -> bool:
	"""Returns True if we are inside subtools chroot."""
	return SUBTOOLS_CHROOT_VERSION_FILE.exists()


	def _assert_inside_subtools_chroot() -> None:
	"""Die if not _is_inside_subtools_chroot()."""
	if not _is_inside_subtools_chroot():
	cros_build_lib.Die("Not in subtools SDK")


	def _setup_base_sdk(
	build_target: build_target_lib.BuildTarget,
	setup_chroot: bool,
	) -> None:
	"""SetupBoard workalike that converts a regular SDK into a subtools chroot.

	Runs inside the /build/amd64-subtools-host subtools SDK chroot.
	"""
	cros_build_lib.AssertInsideChroot()
	cros_build_lib.AssertRootUser()

	# "Convert" the SDK into a subtools SDK.
	if not _is_inside_subtools_chroot():
	# Copy the sentinel file that chromite uses to indicate the chroot's
	# duality. The file is copied (not moved) so that other chromite tooling
	# continues to work.
	shutil.copy(
	cros_sdk_lib.CHROOT_VERSION_FILE, SUBTOOLS_CHROOT_VERSION_FILE
	)

	if setup_chroot:
	# TODO(b/277992359): Additional setup here, e.g., packages, base layout.
	logging.info("Setting up subtools SDK in %s.", build_target.root)


	def _run_system_emerge(
	emerge_cmd: List[Union[str, Path]],
	extra_env: Dict[str, str],
	use_goma: bool,
	use_remoteexec: bool,
	reason: str,
	) -> None:
	"""Runs an emerge command, updating the live system."""
	extra_env = extra_env.copy()
	with osutils.TempDir() as tempdir:
	extra_env[constants.CROS_METRICS_DIR_ENVVAR] = tempdir
	with sysroot.RemoteExecution(use_goma, use_remoteexec):
	logging.info("Merging %s now.", reason)
	try:
	# TODO(b/277992359): Bazel.
	cros_build_lib.sudo_run(
	emerge_cmd,
	preserve_env=True,
	extra_env=extra_env,
	)
	logging.info("Merging %s complete.", reason)
	except cros_build_lib.RunCommandError as e:
	failed_pkgs = portage_util.ParseDieHookStatusFile(tempdir)
	logging.error("Merging %s failed on %s", reason, failed_pkgs)
	raise sysroot_lib.PackageInstallError(
	f"Merging {reason} failed",
	e.result,
	exception=e,
	packages=failed_pkgs,
	)


	def _build_sdk_packages(config: sysroot.BuildPackagesRunConfig) -> None:
	"""The BuildPackages workalike for installing into the staging SDK."""
	_assert_inside_subtools_chroot()
	cros_build_lib.AssertNonRootUser()

	try:
	# sysroot.BuildPackages can't (yet?) be used here, because it _only_
	# supports cross-compilation. SDK package management is currently all
	# handled by src/scripts/sdk_lib/make_chroot.sh (b/191307774).

	emerge = [constants.CHROMITE_BIN_DIR / "parallel_emerge"]
	extra_env = config.GetExtraEnv()
	emerge_flags = config.GetEmergeFlags()
	exclude_pkgs = " ".join(EXCLUDE_PACKAGES)
	emerge_flags.extend(
	[
	f"--useoldpkg-atoms={exclude_pkgs}",
	f"--rebuild-exclude={exclude_pkgs}",
	]
	)
	cmd = emerge + emerge_flags + config.GetPackages()
	_run_system_emerge(
	cmd,
	extra_env,
	config.use_goma,
	config.use_remoteexec,
	reason="subtools builder SDK packages",
	)

	except sysroot_lib.PackageInstallError as e:
	cros_build_lib.Die(e)


	def main(argv: Optional[List[str]] = None) -> Optional[int]:
	opts = parse_args(argv)
	return build_sdk_subtools(opts, argv if argv else [])


	def build_sdk_subtools(opts: Options, argv: List[str]) -> int:
	"""Executes SDK subtools builder steps according to `opts`."""
	# BuildTarget needs a str, but opts.output_dir is osutils.ExpandPath.
	custom_output_dir = str(opts.output_dir) if opts.output_dir else None
	build_target = build_target_lib.BuildTarget(
	name=SUBTOOLS_OUTPUT_DIR, build_root=custom_output_dir
	)

	# If the process is in the subtools chroot, we must assume it's already set
	# up (we are in it). So start building.
	if _is_inside_subtools_chroot() and not opts.relaunch_for_setup:
	if opts.update_packages:
	_build_sdk_packages(opts.build_run_config)
	return 0

	# Otherwise, we have the option to set it up. Then restart inside it. The
	# setup runs `cros_sdk` to get a base SDK, creates an SDK subprocess to set
	# it up as a subtools SDK, then restarts inside the subtools SDK.
	if cros_build_lib.IsInsideChroot():
	if opts.relaunch_for_setup:
	# This is the subprocess of the not-in-chroot path used to convert
	# the base SDK to a subtools SDK (within the chroot).
	_setup_base_sdk(build_target, opts.setup_chroot)
	return 0
	else:
	cros_build_lib.Die(
	"build_sdk_subtools must be run outside the chroot."
	)

	logging.info("Initializing subtools builder in %s", build_target.root)
	subtools_chroot = os.path.join(
	constants.DEFAULT_CHROOT_PATH, build_target.root.lstrip("/")
	)
	chroot_args = ["--chroot", subtools_chroot]

	if opts.setup_chroot:
	# Get an SDK. TODO(b/277992359):
	# - Fetch an SDK version pinned by pupr rather than the default.
	# - Should this use cros_sdk_lib directly?

	# Pass "--skip-chroot-upgrade": the SDK should initially be used
	# "as-is", but later steps may upgrade packages in the subtools deptree.
	cros_sdk_args = ["--create", "--skip-chroot-upgrade"]
	cros_sdk_args += ["--delete"] if opts.clean else []
	cros_sdk = cros_build_lib.run(
	["cros_sdk"] + chroot_args + cros_sdk_args,
	check=False,
	cwd=constants.SOURCE_ROOT,
	)
	if cros_sdk.returncode != 0:
	return cros_sdk.returncode

	# Invoke `_setup_base_sdk()` inside the SDK.
	setup_base_sdk = cros_build_lib.sudo_run(
	["build_sdk_subtools"] + argv + [_RELAUNCH_FOR_SETUP_FLAG],
	check=False,
	enter_chroot=True,
	chroot_args=chroot_args,
	cwd=constants.SOURCE_ROOT,
	)
	if setup_base_sdk.returncode != 0:
	return setup_base_sdk.returncode

	raise commandline.ChrootRequiredError(
	["build_sdk_subtools"] + argv, chroot_args=chroot_args
	)