scripts/package_has_missing_deps.py

# Copyright 2023 The ChromiumOS Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Check whether a package links libraries not in RDEPEND.

If no argument is provided it will check all installed packages. It takes the
BOARD environment variable into account.

Example:
    package_hash_missing_deps.py --board=amd64-generic --match \
        chromeos-base/cryptohome
"""

import argparse
import collections
import logging
import os
from pathlib import Path
import pprint
import sys
from typing import Iterable, List, Optional, Set, Union

from chromite.lib import build_target_lib
from chromite.lib import chroot_lib
from chromite.lib import commandline
from chromite.lib import cros_build_lib
from chromite.lib import parallel
from chromite.lib import portage_util
from chromite.lib.parser import package_info


VIRTUALS = {
    "virtual/acl": ("sys-apps/acl", "media-libs/img-ddk-bin"),
    "virtual/arc-opengles": (
        "media-libs/arc-img-ddk",
        "media-libs/arc-mesa-img",
        "media-libs/arc-mali-drivers",
        "media-libs/arc-mali-drivers-bifrost",
        "media-libs/arc-mali-drivers-bifrost-bin",
        "media-libs/arc-mali-drivers-valhall",
        "media-libs/arc-mali-drivers-valhall-bin",
        "media-libs/arc-mesa",
        "media-libs/arc-mesa-amd",
        "media-libs/arc-mesa-freedreno",
        "media-libs/arc-mesa-iris",
        "media-libs/arc-mesa-virgl",
        "x11-drivers/opengles-headers",
    ),
    "virtual/cros-camera-hal": (
        "media-libs/cros-camera-hal-intel-ipu3",
        "media-libs/cros-camera-hal-intel-ipu6",
        "media-libs/cros-camera-hal-mtk",
        "media-libs/cros-camera-hal-qti",
        "media-libs/cros-camera-hal-rockchip-isp1",
        "media-libs/cros-camera-hal-usb",
        "media-libs/qti-7c-camera-tuning",
    ),
    "virtual/img-ddk": ("media-libs/img-ddk", "media-libs/img-ddk-bin"),
    "virtual/jpeg": ("media-libs/libjpeg-turbo", "media-libs/jpeg"),
    "virtual/krb5": ("app-crypt/mit-krb5", "app-crypt/heimdal"),
    "virtual/libcrypt": ("sys-libs/libxcrypt",),
    "virtual/libelf": ("dev-libs/elfutils", "sys-freebsd/freebsd-lib"),
    "virtual/libiconv": ("dev-libs/libiconv",),
    "virtual/libintl": ("dev-libs/libintl",),
    "virtual/libudev": (
        "sys-apps/systemd-utils",
        "sys-fs/udev",
        "sys-fs/eudev",
        "sys-apps/systemd",
    ),
    "virtual/libusb": ("dev-libs/libusb", "sys-freebsd/freebsd-lib"),
    "virtual/opengles": (
        "media-libs/img-ddk",
        "media-libs/img-ddk-bin",
        "media-libs/libglvnd",
        "media-libs/mali-drivers-bin",
        "media-libs/mali-drivers-bifrost",
        "media-libs/mali-drivers-bifrost-bin",
        "media-libs/mali-drivers-valhall",
        "media-libs/mali-drivers-valhall-bin",
        "media-libs/mesa",
        "media-libs/mesa-amd",
        "media-libs/mesa-freedreno",
        "media-libs/mesa-iris",
        "media-libs/mesa-llvmpipe",
        "media-libs/mesa-panfrost",
        "media-libs/mesa-reven",
        "x11-drivers/opengles-headers",
    ),
    "virtual/vulkan-icd": (
        "media-libs/img-ddk",
        "media-libs/img-ddk-bin",
        "media-libs/mali-drivers-bifrost",
        "media-libs/mali-drivers-bifrost-bin",
        "media-libs/mali-drivers-valhall",
        "media-libs/mali-drivers-valhall-bin",
        "media-libs/mesa",
        "media-libs/mesa-freedreno",
        "media-libs/mesa-iris",
        "media-libs/mesa-llvmpipe",
        "media-libs/mesa-radv",
        "media-libs/vulkan-loader",
    ),
}


def env_to_libs(var: str) -> List[str]:
    """Converts value of REQUIRES to a list of .so files.

    For example:
      "arm_32: libRSSupport.so libblasV8.so libc.so ..."
    Becomes:
      ["libRSSupport.so", "libblasV8.so", "libc.so", ...]
    """
    return [x for x in var.split() if not x.endswith(":")]


class DotSoResolver:
    """Provides shared library related dependency operations."""

    def __init__(
        self,
        board: Optional[str] = None,
        root: Union[os.PathLike, str] = "/",
        chroot: Optional[chroot_lib.Chroot] = None,
    ):
        self.board = board
        self.chroot = chroot if chroot else chroot_lib.Chroot()

        self.sdk_db = portage_util.PortageDB()
        self._sdk_db_packges = None
        self.db = self.sdk_db if root == "/" else portage_util.PortageDB(root)
        self._db_packges = None
        self.provided_libs_cache = {}

        # Lazy initialize since it might not be needed.
        self.lib_to_package_map = None

    @property
    def sdk_db_packages(self):
        """Cache sdk_db.InstalledPackages().

        We won't be modifying it, so it's safe for us to reuse the results.
        """
        if self._sdk_db_packges is None:
            self._sdk_db_packges = self.sdk_db.InstalledPackages()
        return self._sdk_db_packges

    @property
    def db_packages(self):
        """Cache db.InstalledPackages().

        We won't be modifying it, so it's safe for us to reuse the results.
        """
        if self._db_packges is None:
            self._db_packges = self.db.InstalledPackages()
        return self._db_packges

    def get_packages(
        self, query: str, from_sdk=False
    ) -> Iterable[portage_util.InstalledPackage]:
        """Find matching InstalledPackage(s) for the |query|."""
        packages = self.sdk_db_packages if from_sdk else self.db_packages
        info = package_info.parse(query)
        for package in packages:
            if info.package != package.package:
                continue
            if info.category != package.category:
                continue
            dep_info = package.package_info
            if info.revision and info.revision != dep_info.revision:
                continue
            if info.pv and info.pv != dep_info.pv:
                continue
            logging.debug("query: %s: matched %s", query, dep_info.cpvr)
            yield package

    def get_required_libs(self, package) -> Set[str]:
        """Return a set of required .so files."""
        requires = package.requires
        if requires is not None:
            return set(env_to_libs(package.requires))
        # Fallback to needed if requires is not available.
        aggregate = set()
        needed = package.needed
        if needed is not None:
            for libs in needed.values():
                aggregate.update(libs)
        return aggregate

    def get_deps(self, package) -> List[portage_util.InstalledPackage]:
        """Return a list of dependencies.

        This expands the virtuals listed below.
        """
        cpvr = f"{package.category}/{package.pf}"
        expanded = []
        deps = []
        for dep in portage_util.GetFlattenedDepsForPackage(
            cpvr, board=self.board, depth=1
        ):
            logging.debug("%s: found package dependency: %s", cpvr, dep)
            info = package_info.parse(dep)
            if not info:
                continue

            cp = info.cp
            if cp in VIRTUALS:
                expanded += VIRTUALS[cp]
                continue

            pkg = self.db.GetInstalledPackage(info.category, info.pvr)
            if pkg:
                deps.append(pkg)
            else:
                logging.warning("%s: could not find installed %s", cpvr, dep)

        for dep in expanded:
            deps.extend(self.get_packages(dep))

        return deps

    def get_implicit_libs(self):
        """Return a set of .so files that are provided by the system."""
        # libstdc++ comes from the toolchain so always ignore it.
        implicit_libs = {"libstdc++.so", "libstdc++.so.6"}
        for dep, from_sdk in (
            ("cross-aarch64-cros-linux-gnu/glibc", True),
            ("cross-armv7a-cros-linux-gnueabihf/glibc", True),
            ("cross-i686-cros-linux-gnu/glibc", True),
            ("cross-x86_64-cros-linux-gnu/glibc", True),
            ("sys-libs/glibc", False),
            ("sys-libs/libcxx", False),
            ("sys-libs/llvm-libunwind", False),
        ):
            for pkg in self.get_packages(dep, from_sdk):
                implicit_libs.update(self.provided_libs(pkg))
        return implicit_libs

    def provided_libs(self, package: portage_util.InstalledPackage) -> Set[str]:
        """Return a set of .so files provided by |package|."""
        cpvr = f"{package.category}/{package.pf}"
        if cpvr in self.provided_libs_cache:
            return self.provided_libs_cache[cpvr]

        libs = set()
        contents = package.ListContents()
        # Keep only the .so files
        for typ, path in contents:
            if typ == package.DIR:
                continue
            filename = os.path.basename(path)
            if filename.endswith(".so") or (
                ".so." in filename and not filename.endswith(".debug")
            ):
                libs.add(filename)
        self.provided_libs_cache[cpvr] = libs
        return libs

    def cache_libs_from_build(
        self, package: portage_util.InstalledPackage, image_dir: Path
    ):
        """Populate the provided_libs_cache for the package from the image dir.

        When using build-info, CONTENTS might not be available yet. so provide
        alternative using the destination directory of the ebuild.
        """

        cpvr = f"{package.category}/{package.pf}"
        libs = set()
        for _, _, files in os.walk(image_dir):
            for file in files:
                if file.endswith(".so") or (
                    ".so." in file and not file.endswith(".debug")
                ):
                    libs.add(os.path.basename(file))
        self.provided_libs_cache[cpvr] = libs

    def get_provided_from_all_deps(
        self, package: portage_util.InstalledPackage
    ) -> Set[str]:
        """Return a set of .so files provided by the immediate dependencies."""
        provided_libs = set()
        # |package| may not actually be installed yet so manually add it to the
        # since a package can depend on its own libs.
        provided_libs.update(self.provided_libs(package))
        for pkg in self.get_deps(package):
            logging.debug(
                "%s: loading libs from dependency %s",
                package.package_info.cpvr,
                pkg.package_info.cpvr,
            )
            provided_libs.update(self.provided_libs(pkg))
        return provided_libs

    def lib_to_package(self, lib_filename: str = None) -> Set[str]:
        """Return a set of packages that contain the library."""
        if self.lib_to_package_map is None:
            lookup = collections.defaultdict(set)
            for pkg in self.db.InstalledPackages():
                cpvr = f"{pkg.category}/{pkg.pf}"
                # Packages with bundled libs for internal use and/or standaline
                # binary packages.
                if f"{pkg.category}/{pkg.package}" in (
                    "app-emulation/qemu",
                    "chromeos-base/aosp-frameworks-ml-nn-vts",
                    "chromeos-base/factory",
                    "chromeos-base/signingtools-bin",
                    "sys-devel/gcc-bin",
                ):
                    continue
                for lib in set(self.provided_libs(pkg)):
                    lookup[lib].add(cpvr)
            self.lib_to_package_map = lookup
        else:
            lookup = self.lib_to_package_map
        if not lib_filename:
            return set()
        try:
            return lookup[lib_filename]
        except KeyError:
            return set()


def get_parser() -> commandline.ArgumentParser:
    """Build the argument parser."""
    parser = commandline.ArgumentParser(description=__doc__)

    parser.add_argument("package", nargs="*", help="package atom")

    parser.add_argument(
        "-b",
        "--board",
        "--build-target",
        default=cros_build_lib.GetDefaultBoard(),
        help="ChromeOS board (Uses the SDK if not specified)",
    )

    parser.add_argument(
        "--no-default-board",
        dest="board",
        const=None,
        action="store_const",
        help="Ignore the default board",
    )

    parser.add_argument(
        "-i",
        "--build-info",
        default=None,
        type=Path,
        help="Path to build-info folder post src_install",
    )

    parser.add_argument(
        "-x",
        "--image",
        default=None,
        type=Path,
        help="Path to image folder post src_install (${D} if unspecified)",
    )

    parser.add_argument(
        "--match",
        default=False,
        action="store_true",
        help="Try to match missing libraries",
    )

    parser.add_argument(
        "-j",
        "--jobs",
        default=None,
        type=int,
        help="Number of parallel processes",
    )

    return parser


def parse_arguments(argv: List[str]) -> argparse.Namespace:
    """Parse and validate arguments."""
    parser = get_parser()
    opts = parser.parse_args(argv)
    if opts.build_info and opts.package:
        parser.error("Do not specify a package when setting --board-info")
    if opts.image and not opts.build_info:
        parser.error("--image requires --board-info")
    if opts.build_info or len(opts.package) == 1:
        opts.jobs = 1
    return opts


def check_package(
    package: portage_util.InstalledPackage,
    implicit: Set[str],
    resolver: DotSoResolver,
    match: bool,
    debug: bool,
) -> bool:
    """Returns false if the package has missing dependencies"""
    if not package:
        print("missing package")
        return False

    provided = resolver.get_provided_from_all_deps(package)
    if debug:
        print("provided")
        pprint.pprint(sorted(provided))

    available = provided.union(implicit)
    required = resolver.get_required_libs(package)
    if debug:
        print("required")
        pprint.pprint(sorted(required))
    unsatisfied = required - available
    if unsatisfied:
        cpvr = package.package_info.cpvr
        print(f"'{cpvr}' missing deps for: ", end="")
        pprint.pprint(sorted(unsatisfied))
        if match:
            missing = set()
            for lib in unsatisfied:
                missing.update(resolver.lib_to_package(lib))
            if missing:
                print(f"'{cpvr}' needs: ", end="")
                pprint.pprint(sorted(missing))
        return False
    return True


def main(argv: Optional[List[str]]):
    """Main."""
    opts = parse_arguments(argv)
    opts.Freeze()

    board = opts.board
    root = build_target_lib.get_default_sysroot_path(board)
    if board:
        os.environ["PORTAGE_CONFIGROOT"] = root
        os.environ["SYSROOT"] = root
        os.environ["ROOT"] = root

    failed = False
    resolver = DotSoResolver(board, root)

    if not opts.package:
        if opts.build_info:
            pkg = portage_util.InstalledPackage(resolver.db, opts.build_info)
            image_path = opts.image or os.environ.get("D")
            if image_path:
                resolver.cache_libs_from_build(pkg, Path(image_path))
            packages = [pkg]
        else:
            packages = resolver.db.InstalledPackages()
    else:
        packages = []
        for pkg in opts.package:
            packages.extend(resolver.get_packages(pkg))

    implicit = resolver.get_implicit_libs()
    if opts.debug:
        print("implicit")
        pprint.pprint(implicit)

    if opts.jobs == 1:
        for package in packages:
            if not check_package(
                package,
                implicit,
                resolver,
                opts.match,
                opts.debug,
            ):
                failed = True
    else:
        if opts.match:
            # Pre initialize the map before starting jobs.
            resolver.lib_to_package()
        for ret in parallel.RunTasksInProcessPool(
            lambda p: check_package(
                p, implicit, resolver, opts.match, opts.debug
            ),
            [[p] for p in packages],
            opts.jobs,
        ):
            if not ret:
                failed = True

    if failed:
        sys.exit(1)


if __name__ == "__main__":
    main(sys.argv[1:])