scripts/cros_uprev.py

# Copyright 2019 The ChromiumOS Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Simplified cros_mark_as_stable script."""

import logging
import os
from typing import Dict, List, Optional

from chromite.lib import chromeos_version
from chromite.lib import commandline
from chromite.lib import constants
from chromite.lib import cros_build_lib
from chromite.lib import git
from chromite.lib import osutils
from chromite.lib import parallel
from chromite.lib import portage_util


# Commit message subject for uprevving Portage packages.
GIT_COMMIT_SUBJECT = "Marking set of ebuilds as stable"

# Commit message for uprevving Portage packages.
_GIT_COMMIT_MESSAGE = "Marking 9999 ebuild for %s as stable."


def GetParser() -> commandline.ArgumentParser:
    """Build the argument parser."""
    parser = commandline.ArgumentParser(description=__doc__)

    parser.add_argument(
        "--all", action="store_true", help="Mark all packages as stable."
    )
    parser.add_argument("--boards", help="Colon-separated list of boards.")
    parser.add_argument("--chroot", type="path", help="The chroot path.")
    parser.add_argument(
        "--force",
        action="store_true",
        help="Force the stabilization of manually uprevved "
        "packages. (only compatible with -p)",
    )
    parser.add_argument(
        "--force-version",
        type=str,
        help="Force the version of manually uprevved packages.",
    )
    parser.add_argument(
        "--overlay-type",
        required=True,
        choices=["public", "private", "both"],
        help='Populates --overlays based on "public", "private", ' 'or "both".',
    )
    parser.add_argument(
        "--packages", help="Colon separated list of packages to rev."
    )
    parser.add_argument(
        "--dump-files",
        action="store_true",
        help="Dump the revved packages, new files list, and "
        "removed files list files. This is mostly for"
        "debugging right now.",
    )

    return parser


def _ParseArguments(argv: List[str]) -> commandline.ArgumentNamespace:
    """Parse and validate arguments."""
    parser = GetParser()
    options = parser.parse_args(argv)

    # Parse, cleanup, and populate options.
    if options.packages:
        options.packages = options.packages.split(":")
    if options.boards:
        options.boards = options.boards.split(":")
    options.overlays = portage_util.FindOverlays(options.overlay_type)

    # Verify options.
    if not options.packages and not options.all:
        parser.error("Please specify at least one package (--packages)")
    if options.force and options.all:
        parser.error(
            "Cannot use --force with --all. You must specify a list of "
            "packages you want to force uprev."
        )

    options.Freeze()
    return options


def main(argv: List[str]) -> None:
    options = _ParseArguments(argv)

    if not options.boards:
        overlays = portage_util.FindOverlays(options.overlay_type)
    else:
        overlays = set()
        for board in options.boards:
            board_overlays = portage_util.FindOverlays(
                options.overlay_type, board=board
            )
            overlays = overlays.union(board_overlays)

        overlays = list(overlays)

    manifest = git.ManifestCheckout.Cached(constants.SOURCE_ROOT)

    _WorkOnCommit(options, overlays, manifest, options.packages or None)


def CleanStalePackages(
    boards: List[str], package_atoms: List[str], chroot: str
) -> None:
    """Cleans up stale package info from a previous build.

    Args:
      boards: Boards to clean the packages from.
      package_atoms: A list of package atoms to unmerge.
      chroot: The chroot path.
    """
    if package_atoms:
        logging.info("Cleaning up stale packages %s.", package_atoms)

    # First unmerge all the packages for a board, then eclean it.
    # We need these two steps to run in order (unmerge/eclean),
    # but we can let all the boards run in parallel.
    def _CleanStalePackages(board: str):
        if board:
            suffix = "-" + board
            runcmd = cros_build_lib.run
        else:
            suffix = ""
            runcmd = cros_build_lib.sudo_run

        chroot_args = ["--chroot", chroot] if chroot else None
        emerge, eclean = "emerge" + suffix, "eclean" + suffix
        if not osutils.FindMissingBinaries([emerge, eclean]):
            if package_atoms:
                # If nothing was found to be unmerged, emerge will exit(1).
                result = runcmd(
                    [emerge, "-q", "--unmerge"] + list(package_atoms),
                    enter_chroot=True,
                    chroot_args=chroot_args,
                    extra_env={"CLEAN_DELAY": "0"},
                    check=False,
                    cwd=constants.SOURCE_ROOT,
                )
                if result.returncode not in (0, 1):
                    raise cros_build_lib.RunCommandError(
                        "unexpected error", result
                    )
            runcmd(
                [eclean, "-d", "packages"],
                cwd=constants.SOURCE_ROOT,
                enter_chroot=True,
                chroot_args=chroot_args,
                stdout=True,
                stderr=True,
            )

    tasks = []
    for board in boards:
        tasks.append([board])
    tasks.append([None])

    parallel.RunTasksInProcessPool(_CleanStalePackages, tasks)


def _WorkOnCommit(
    options: commandline.ArgumentNamespace,
    overlays: List[str],
    manifest: git.ManifestCheckout,
    package_list: Optional[List[str]],
) -> None:
    """Commit uprevs of overlays belonging to different git projects in parallel.

    Args:
      options: The options object returned by the argument parser.
      overlays: A list of overlays to work on.
      manifest: The manifest of the given source root.
      package_list: A list of packages passed from commandline to work on.
    """
    overlay_ebuilds = _GetOverlayToEbuildsMap(
        overlays, package_list, options.force
    )

    with parallel.Manager() as manager:
        # Contains the array of packages we actually revved.
        revved_packages = manager.list()
        new_package_atoms = manager.list()
        new_ebuild_files = manager.list()
        removed_ebuild_files = manager.list()

        inputs = [
            [
                manifest,
                [overlay],
                overlay_ebuilds,
                revved_packages,
                new_package_atoms,
                new_ebuild_files,
                removed_ebuild_files,
                options,
            ]
            for overlay in overlays
        ]
        parallel.RunTasksInProcessPool(_UprevOverlays, inputs)

        if options.chroot and os.path.exists(options.chroot):
            CleanStalePackages(
                options.boards or [], new_package_atoms, options.chroot
            )

        if options.dump_files:
            osutils.WriteFile(
                "/tmp/revved_packages", "\n".join(revved_packages)
            )
            osutils.WriteFile(
                "/tmp/new_ebuild_files", "\n".join(new_ebuild_files)
            )
            osutils.WriteFile(
                "/tmp/removed_ebuild_files", "\n".join(removed_ebuild_files)
            )


def _GetOverlayToEbuildsMap(
    overlays: List[str], package_list: Optional[List[str]], force: bool
) -> Dict:
    """Get ebuilds for overlays.

    Args:
      overlays: A list of overlays to work on.
      package_list: A list of packages passed from commandline to work on.
      force: Whether to use packages even if in manual uprev list.

    Returns:
      A dict mapping each overlay to a list of ebuilds belonging to it.
    """
    root_version = chromeos_version.VersionInfo.from_repo(constants.SOURCE_ROOT)
    subdir_removal = chromeos_version.VersionInfo("10363.0.0")
    require_subdir_support = root_version < subdir_removal
    use_all = not package_list

    overlay_ebuilds = {}
    inputs = [
        [overlay, use_all, package_list, force, require_subdir_support]
        for overlay in overlays
    ]
    result = parallel.RunTasksInProcessPool(
        portage_util.GetOverlayEBuilds, inputs
    )
    for idx, ebuilds in enumerate(result):
        overlay_ebuilds[overlays[idx]] = ebuilds

    return overlay_ebuilds


def _UprevOverlays(
    manifest: git.ManifestCheckout,
    overlays: List[str],
    overlay_ebuilds: Dict,
    revved_packages: List[str],
    new_package_atoms: List[str],
    new_ebuild_files: List[str],
    removed_ebuild_files: List[str],
    options: commandline.ArgumentNamespace,
) -> None:
    """Execute uprevs for overlays in sequence.

    Args:
      manifest: The manifest of the given source root.
      overlays: A list over overlays to commit.
      overlay_ebuilds: A dict mapping overlays to their ebuilds.
      revved_packages: A shared list of revved packages.
      new_package_atoms: A shared list of new package atoms.
      new_ebuild_files: New stable ebuild paths.
      removed_ebuild_files: Old ebuild paths that were removed.
      options: The options object returned by the argument parser.
    """
    for overlay in overlays:
        if not os.path.isdir(overlay):
            logging.warning("Skipping %s, which is not a directory.", overlay)
            continue

        ebuilds = overlay_ebuilds.get(overlay, [])
        if not ebuilds:
            continue

        with parallel.Manager() as manager:
            # Contains the array of packages we actually revved.
            messages = manager.list()

            inputs = [
                [
                    overlay,
                    ebuild,
                    manifest,
                    new_ebuild_files,
                    removed_ebuild_files,
                    messages,
                    revved_packages,
                    new_package_atoms,
                    options,
                ]
                for ebuild in ebuilds
            ]
            parallel.RunTasksInProcessPool(_WorkOnEbuild, inputs)


def _WorkOnEbuild(
    overlay: str,
    ebuild: portage_util.EBuild,
    manifest: git.ManifestCheckout,
    new_ebuild_files: List[str],
    removed_ebuild_files: List[str],
    messages: List[str],
    revved_packages: List[str],
    new_package_atoms: List[str],
    options: commandline.ArgumentNamespace,
) -> None:
    """Work on a single ebuild.

    Args:
      overlay: The overlay where the ebuild belongs to.
      ebuild: The ebuild to work on.
      manifest: The manifest of the given source root.
      new_ebuild_files: New stable ebuild paths that were created.
      removed_ebuild_files: Old ebuild paths that were removed.
      messages: A share list of commit messages.
      revved_packages: A shared list of revved packages.
      new_package_atoms: A shared list of new package atoms.
      options: The options object returned by the argument parser.
    """
    logging.debug(
        "Working on %s, info %s", ebuild.package, ebuild.cros_workon_vars
    )
    try:
        result = ebuild.RevWorkOnEBuild(
            os.path.join(constants.SOURCE_ROOT, "src"),
            manifest,
            new_version=options.force_version,
        )
    except portage_util.InvalidUprevSourceError as e:
        logging.error(
            "An error occurred while uprevving %s: %s", ebuild.package, e
        )
        raise
    except OSError:
        logging.warning(
            "Cannot rev %s\n"
            "Note you will have to go into %s "
            "and reset the git repo yourself.",
            ebuild.package,
            overlay,
        )
        raise

    if result:
        new_package, ebuild_path_to_add, ebuild_path_to_remove = result

        if ebuild_path_to_add:
            new_ebuild_files.append(ebuild_path_to_add)
        if ebuild_path_to_remove:
            osutils.SafeUnlink(ebuild_path_to_remove)
            removed_ebuild_files.append(ebuild_path_to_remove)

        messages.append(_GIT_COMMIT_MESSAGE % ebuild.package)
        revved_packages.append(ebuild.package)
        new_package_atoms.append("=%s" % new_package)