establish scripts/ directory; factor out cargo-audit installation logic
`cargo-vet` needs similar installation (and PATH adding) logic; sharing
is caring.
Also move this to a scripts/ dir, since having 4+ py scripts at the root
of this (one or two of which users will invoke) is starting to seem a
tad excessive.
BUG=b:250919469
TEST=./cargo-audit.py
Change-Id: Ia50d235d79b319401568713a6198bdf54f0d69ac
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/rust_crates/+/4199979
Reviewed-by: Allen Webb <allenwebb@google.com>
Tested-by: George Burgess <gbiv@chromium.org>
diff --git a/.gitignore b/.gitignore
index 8ca40cf..6654af2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
+__pycache__/
vendor/**/OWNERS
vendor/**/OWNERS.*
diff --git a/scripts/README.md b/scripts/README.md
new file mode 100644
index 0000000..0959552
--- /dev/null
+++ b/scripts/README.md
@@ -0,0 +1,8 @@
+Various scripts and helpers used in the gardening of this repository.
+
+- `./run-cargo-vet.py` is used to run `cargo-vet` with special exclusion criteria.
+ This will download and install a hermetic version of `cargo-vet` if necessary.
+- `./cargo-audit.py` runs cargo-audit, and is run regularly by automation w/
+ reporting on findings.
+- `./incremental-cargo-update` is a tool used to help keep this repository
+ up-to-date. It is run regularly by automation.
diff --git a/cargo-audit.py b/scripts/cargo-audit.py
similarity index 72%
rename from cargo-audit.py
rename to scripts/cargo-audit.py
index cd2971e..b1e567a 100755
--- a/cargo-audit.py
+++ b/scripts/cargo-audit.py
@@ -30,6 +30,8 @@
from pathlib import Path
from typing import Any, List, NamedTuple, Set, Union
+import cargo
+
# The CPU arches that we care about.
SUPPORTED_ARCHES = (
"aarch64",
@@ -206,99 +208,33 @@
return empty
-def ensure_cargo_bin_is_in_path():
- """Ensures that .cargo/bin is in $PATH for this process."""
- cargo_bin = str(Path.home() / ".cargo" / "bin")
- path = os.getenv("PATH", "")
- path_has_cargo_bin = path.endswith(cargo_bin) or cargo_bin + ":" in path
- if not path_has_cargo_bin:
- os.environ["PATH"] = cargo_bin + ":" + path
-
-
+# Instructions on how to generate a `cargo audit` tarball:
+# 1. `git clone` the rustsec repo here:
+# https://github.com/rustsec/rustsec
+# 2. `checkout` the tag you're interested in, e.g.,
+# `git checkout cargo-audit/v0.17.4`
+# 3. `rm -rf .git` in the repo.
+# 4. tweak the version number in rustsec/cargo-audit/Cargo.toml to
+# include `+cros`, so we always autosync to the hermetic ChromeOS
+# version.
+# 5. `cargo vendor` in rustsec/cargo-audit, and follow the instructions
+# that it prints out RE "To use vendored sources, ...".
+# 6. `cargo build --offline --locked && rm -rf ../target` in
+# rustsec/cargo-audit, to ensure it builds.
+# 7. `tar cf rustsec-${version}.tar.bz2 rustsec \
+# --use-compress-program="bzip2 -9"`
+# in the parent of your `rustsec` directory.
+# 8. Upload to gs://; don't forget the `-a public-read`.
def ensure_cargo_audit_is_installed():
- """Ensures the proper version of cargo-audit is installed and usable."""
+ """Ensures that `cargo-audit` is installed."""
want_version = "0.17.4+cros"
-
- # Unfortunately, `cargo audit --version` simply prints `cargo-audit-audit`.
- # Call the cargo-audit binary directly to get the version.
- version = subprocess.run(
- ["cargo", "install", "--list"],
- check=True,
- stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT,
- encoding="utf-8",
+ cargo.ensure_cargo_utility_is_installed(
+ utility_name="cargo-audit",
+ want_version=want_version,
+ gs_path=f"gs://chromeos-localmirror/distfiles/rustsec-{want_version}.tar.bz2",
+ sha256="dd9137486b850d30febc84340d9f6aa3964c06a6e786434ca99477d147bd68ae",
+ build_subdir=Path("rustsec") / "cargo-audit",
)
- # Since we do local installations, cargo-install will list this as
- # `cargo-audit v{want_version} ({tempdir_it_was_installed_in}):`.
- want_version_string = f"cargo-audit v{want_version} "
- has_version = any(
- x.startswith(want_version_string) for x in version.stdout.splitlines()
- )
- if has_version:
- return
-
- # Instructions on how to generate a `cargo audit` tarball:
- # 1. `git clone` the rustsec repo here:
- # https://github.com/rustsec/rustsec
- # 2. `checkout` the tag you're interested in, e.g.,
- # `git checkout cargo-audit/v0.17.4`
- # 3. `rm -rf .git` in the repo.
- # 4. tweak the version number in rustsec/cargo-audit/Cargo.toml to
- # include `+cros`, so we always autosync to the hermetic ChromeOS
- # version.
- # 5. `cargo vendor` in rustsec/cargo-audit, and follow the instructions
- # that it prints out RE "To use vendored sources, ...".
- # 6. `cargo build --offline --locked && rm -rf ../target` in
- # rustsec/cargo-audit, to ensure it builds.
- # 7. `tar cf rustsec-${version}.tar.bz2 rustsec \
- # --use-compress-program="bzip2 -9"`
- # in the parent of your `rustsec` directory.
- # 8. Upload to gs://; don't forget the `-a public-read`.
- logging.info("Auto-installing cargo-audit version %s", want_version)
- gs_path = (
- "gs://chromeos-localmirror/distfiles/" f"rustsec-{want_version}.tar.bz2"
- )
- sha256 = "dd9137486b850d30febc84340d9f6aa3964c06a6e786434ca99477d147bd68ae"
-
- tempdir = Path(tempfile.mkdtemp(prefix="cargo-audit-install"))
- logging.info(
- "Using %s as a tempdir. This will not be cleaned up on failures.",
- tempdir,
- )
- logging.info("Downloading cargo-audit...")
- tbz2_name = "cargo-audit.tar.bz2"
- subprocess.run(
- ["gsutil.py", "cp", gs_path, tbz2_name],
- check=True,
- cwd=tempdir,
- )
-
- logging.info("Verifying SHA...")
- with (tempdir / tbz2_name).open("rb") as f:
- got_sha256 = hashlib.sha256()
- for block in iter(lambda: f.read(32 * 1024), b""):
- got_sha256.update(block)
- got_sha256 = got_sha256.hexdigest()
- if got_sha256 != sha256:
- raise ValueError(
- f"SHA256 mismatch for {gs_path}. Got {got_sha256}, want "
- f"{sha256}"
- )
-
- logging.info("Unpacking...")
- subprocess.run(
- ["tar", "xaf", tbz2_name],
- check=True,
- cwd=tempdir,
- )
- logging.info("Installing...")
- subprocess.run(
- ["cargo", "install", "--locked", "--offline", "--path=."],
- check=True,
- cwd=tempdir / "rustsec" / "cargo-audit",
- )
- logging.info("`cargo-audit` installed successfully.")
- shutil.rmtree(tempdir)
def main(argv: List[str]):
@@ -315,7 +251,7 @@
"--rust-crates",
type=Path,
help="Path to rust_crates.",
- default=Path(__file__).resolve().parent,
+ default=Path(__file__).resolve().parent.parent,
)
parser.add_argument(
"--skip-install",
@@ -330,7 +266,7 @@
level=logging.DEBUG if opts.debug else logging.INFO,
)
- ensure_cargo_bin_is_in_path()
+ cargo.ensure_cargo_bin_is_in_path()
if not opts.skip_install:
ensure_cargo_audit_is_installed()
diff --git a/scripts/cargo.py b/scripts/cargo.py
new file mode 100644
index 0000000..ac705ef
--- /dev/null
+++ b/scripts/cargo.py
@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+# Copyright 2023 The ChromiumOS Authors
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Utilities to help interact with cargo."""
+
+import argparse
+import hashlib
+import logging
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+
+def ensure_cargo_bin_is_in_path():
+ """Ensures that .cargo/bin is in $PATH for this process."""
+ cargo_bin = str(Path.home() / ".cargo" / "bin")
+ path = os.getenv("PATH", "")
+ path_has_cargo_bin = path.endswith(cargo_bin) or cargo_bin + ":" in path
+ if not path_has_cargo_bin:
+ os.environ["PATH"] = cargo_bin + ":" + path
+
+
+def ensure_cargo_utility_is_installed(
+ utility_name: str,
+ want_version: str,
+ gs_path: str,
+ sha256: str,
+ build_subdir: Path,
+):
+ """Ensures that the given cargo utility is installed.
+
+ Args:
+ utility_name: the name of the given utility, e.g., cargo-audit.
+ want_version: the version string that should be installed.
+ gs_path: the gs:// path to download vendored sources for this from.
+ sha256: the SHA to expect for the gs_path tarball.
+ build_subdir: the subdirectory from which we should run `cargo
+ install`. This is relative to the directory from which the source
+ tarball is unpacked.
+ """
+ # Unfortunately, `cargo ${tool} --version` does not always print the
+ # version for the values of ${tool} we care about. Query `cargo` instead.
+ version = subprocess.run(
+ ["cargo", "install", "--list"],
+ check=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ encoding="utf-8",
+ )
+ # Since we do local installations, cargo-install will list this as e.g.,
+ # `{utility_name} v{want_version} ({tempdir_it_was_installed_in}):`.
+ want_version_string = f"{utility_name} v{want_version} "
+ has_version = any(
+ x.startswith(want_version_string) for x in version.stdout.splitlines()
+ )
+ if has_version:
+ return
+
+ logging.info("Auto-installing %s version %s", utility_name, want_version)
+
+ tempdir = Path(tempfile.mkdtemp(prefix="cargo-util-install"))
+ logging.info(
+ "Using %s as a tempdir. This will not be cleaned up on failures.",
+ tempdir,
+ )
+ logging.info("Downloading %s...", gs_path)
+ tbz2_name = "cargo-utility.tar.bz2"
+ subprocess.run(
+ ["gsutil.py", "cp", gs_path, tbz2_name],
+ check=True,
+ cwd=tempdir,
+ )
+
+ logging.info("Verifying SHA...")
+ with (tempdir / tbz2_name).open("rb") as f:
+ got_sha256 = hashlib.sha256()
+ for block in iter(lambda: f.read(32 * 1024), b""):
+ got_sha256.update(block)
+ got_sha256 = got_sha256.hexdigest()
+ if got_sha256 != sha256:
+ raise ValueError(
+ f"SHA256 mismatch for {gs_path}. Got {got_sha256}, want "
+ f"{sha256}"
+ )
+
+ logging.info("Unpacking...")
+ subprocess.run(
+ ["tar", "xaf", tbz2_name],
+ check=True,
+ cwd=tempdir,
+ )
+ logging.info("Installing...")
+ subprocess.run(
+ ["cargo", "install", "--locked", "--offline", "--path=."],
+ check=True,
+ cwd=tempdir / build_subdir,
+ )
+ logging.info("`%s` installed successfully.", utility_name)
+
+ # Only clean this up on successful installs. It's useful for debugging, and
+ # lands in /tmp anyway.
+ shutil.rmtree(tempdir)
diff --git a/incremental-cargo-update/.gitignore b/scripts/incremental-cargo-update/.gitignore
similarity index 100%
rename from incremental-cargo-update/.gitignore
rename to scripts/incremental-cargo-update/.gitignore
diff --git a/incremental-cargo-update/Cargo.lock b/scripts/incremental-cargo-update/Cargo.lock
similarity index 100%
rename from incremental-cargo-update/Cargo.lock
rename to scripts/incremental-cargo-update/Cargo.lock
diff --git a/incremental-cargo-update/Cargo.toml b/scripts/incremental-cargo-update/Cargo.toml
similarity index 100%
rename from incremental-cargo-update/Cargo.toml
rename to scripts/incremental-cargo-update/Cargo.toml
diff --git a/incremental-cargo-update/README.md b/scripts/incremental-cargo-update/README.md
similarity index 100%
rename from incremental-cargo-update/README.md
rename to scripts/incremental-cargo-update/README.md
diff --git a/incremental-cargo-update/src/main.rs b/scripts/incremental-cargo-update/src/main.rs
similarity index 100%
rename from incremental-cargo-update/src/main.rs
rename to scripts/incremental-cargo-update/src/main.rs