projects/populate-workspace.py

#!/usr/bin/env python3
# Copyright 2022 The ChromiumOS Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Populates the top-level Cargo.toml with all necessary workspace entries.

Also updates cargo-vet's policy information with all of the crates discovered
in the tree.
"""

import argparse
import logging
import os
import subprocess
import sys
import textwrap
from pathlib import Path
from typing import List

import toml

WORKSPACE_FILE_HEADER = """\
# Copyright 2022 The ChromiumOS Authors.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# !! Autogenerated by `populate-workspace.py`; please don't edit. !!

[workspace.metadata.vet]
store = { path = '../cargo-vet' }

"""

# Tools which only run on the host. These are subject to less stringent audit
# requirements.
HOST_TOOLS = frozenset(
    (
        "bindgen-deps",
        "chromeos-dbus-bindings-deps",
        "cxxbridge-cmd-deps",
        "dbus-codegen-deps",
        "factory_installer-deps",
        "flashrom_tester-deps",
        "prjoxide-deps",
        "pyprjoxide-deps",
    )
)

# Dependencies that should not be treated as crates.io dependencies.
NON_CRATES_IO_DEPS = (
    "deqp-runner",
    "dlib",
    "wayland-commons",
    "wayland-scanner",
    "wayland-server",
    "wayland-sys",
)


def update_cargo_vet_info(
    projects_dir: Path, projects: List[str], use_relaxed_target_criteria: bool
):
    """Updates cargo-vet's config.toml with requirements for all `projects`."""
    cargo_vet_config = projects_dir.parent / "cargo-vet" / "config.toml"
    config = cargo_vet_config.read_text(encoding="utf-8")

    cargo_vet_policy = toml.loads(config).get("policy", ())
    project_names = []
    unseen_io_deps = set(
        x for x in NON_CRATES_IO_DEPS if x not in cargo_vet_policy
    )
    for project in projects:
        cargo_toml = projects_dir / project / "Cargo.toml"
        with cargo_toml.open(encoding="utf-8") as f:
            project_name = toml.load(f)["package"]["name"]
        project_names.append((project, project_name))
        unseen_io_deps.discard(project_name)

    add_projects = [x for x in project_names if x[1] not in cargo_vet_policy]
    add_projects += ((None, x) for x in unseen_io_deps)
    if not add_projects:
        logging.info("No cargo-vet-related updates to make.")
        return

    host_safety_criteria = "safe-to-run"
    target_safety_criteria = "rule-of-two-safe-to-deploy"

    # As ugly as it is to hand-update a toml file, our toml implementation has
    # known bugs that lead to invalid toml in quite a few cases (b/242668603).
    # Stick stuff at the end and have `cargo-vet` handle making it pretty.
    with cargo_vet_config.open("a", encoding="utf-8") as f:
        for project_path, project_name in add_projects:
            if project_path:
                logging.info(
                    "Adding entry %s from %s into cargo-vet",
                    project_path,
                    project_name,
                )
            else:
                logging.info(
                    "Synthesizing entry %s for cargo-vet",
                    project_name,
                )

            if project_name in HOST_TOOLS:
                safety_criteria = host_safety_criteria
                note = "Only safe-to-run is required, as this is a host tool."
            elif use_relaxed_target_criteria:
                safety_criteria = host_safety_criteria
                note = "Using relaxed target criteria to help with migration."
            else:
                safety_criteria = target_safety_criteria
                note = None

            f.write("\n")
            f.write(
                textwrap.dedent(
                    f"""\
                    [policy."{project_name}"]
                    criteria = ["{safety_criteria}", "crypto-safe"]
                    dev-criteria = ["{host_safety_criteria}", "crypto-safe"]
                    """
                )
            )
            if note:
                # Only handle these if we have to.
                assert "\\" not in note and '"' not in note, note
                f.write(f'notes = "{note}"\n')

            if project_name in NON_CRATES_IO_DEPS:
                f.write(f"audit-as-crates-io = false\n")

    subprocess.check_call(
        ["scripts/cargo-vet.py", "fmt"], cwd=projects_dir.parent
    )
    logging.info("Cargo vet info updated.")


def find_projects(projects_dir: Path) -> List[str]:
    """Returns a list of projects under `projects_dir`.

    This creates src/ directories for any project that does not currently have
    one, for convenience.
    """
    projects = []
    for dir_path, subdirs, files in os.walk(projects_dir):
        if "Cargo.toml" not in files:
            continue

        dir_path = Path(dir_path)
        if dir_path == projects_dir:
            continue

        projects.append(dir_path.relative_to(projects_dir))

        # It's a waste to descend into src directories.
        if "src" in subdirs:
            del subdirs[subdirs.index("src")]
        else:
            # ...Though if src/ doesn't exist, Cargo will get confused.
            # Synthesize one with a nop lib.rs.
            src = dir_path / "src"
            src.mkdir()
            (src / "lib.rs").write_bytes(b"")
            logging.info("Synthesized src/lib.rs for %s", dir_path)

    projects.sort()
    return projects


def get_parser() -> argparse.ArgumentParser:
    """Gets the arg parser for this script."""
    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    parser.add_argument(
        "--use-strict-target-criteria",
        action="store_true",
        help="Require rule-of-two-safe-to-deploy for target crates.",
    )
    return parser


def main(argv: List[str]):
    """Main function."""
    opts = get_parser().parse_args(argv)
    logging.basicConfig(
        format=">> %(asctime)s: %(levelname)s: %(filename)s:%(lineno)d: "
        "%(message)s",
        level=logging.INFO,
    )

    projects_dir = Path(__file__).resolve().parent
    projects = find_projects(projects_dir)
    assert projects, f"No projects found under {projects_dir}"
    logging.info("Identified %d projects", len(projects))

    workspace_toml_file = projects_dir / "Cargo.toml"
    with workspace_toml_file.open("w", encoding="utf-8") as f:
        f.write(WORKSPACE_FILE_HEADER)
        # The `toml` crate writes this as a massive line, which is hard to
        # read. Since this is simple to write, write it directly.
        # TODO(b/242668603): find a toml crate with prettier formatting
        f.write("[workspace]\nmembers = [\n")
        for project in projects:
            project = str(project)
            assert '"' not in project and "\\" not in project, project
            f.write(f'    "{project}",\n')
        f.write("]")

    logging.info("Workspace Cargo.toml successfully written.")
    update_cargo_vet_info(
        projects_dir,
        projects,
        use_relaxed_target_criteria=not opts.use_strict_target_criteria,
    )


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))