George Burgess IV | 85e6933 | 2020-01-16 17:33:16 -0800 | [diff] [blame] | 1 | # Copyright 2020 The Chromium OS Authors. All rights reserved. |
| 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | |
George Burgess IV | 87c4dbf | 2022-03-21 12:05:11 -0700 | [diff] [blame] | 5 | """Determines all Chrome OS packages that are marked as `hot`. |
George Burgess IV | 85e6933 | 2020-01-16 17:33:16 -0800 | [diff] [blame] | 6 | |
| 7 | Dumps results as a list of package names to a JSON file. Hotness is |
| 8 | determined by statically analyzing an ebuild. |
| 9 | |
| 10 | Primarily intended for use by the Chrome OS toolchain team. |
| 11 | """ |
| 12 | |
George Burgess IV | 85e6933 | 2020-01-16 17:33:16 -0800 | [diff] [blame] | 13 | import json |
Chris McDonald | 59650c3 | 2021-07-20 15:29:28 -0600 | [diff] [blame] | 14 | import logging |
George Burgess IV | 87c4dbf | 2022-03-21 12:05:11 -0700 | [diff] [blame] | 15 | from pathlib import Path |
| 16 | from typing import Iterable, List, Tuple |
George Burgess IV | 85e6933 | 2020-01-16 17:33:16 -0800 | [diff] [blame] | 17 | |
| 18 | from chromite.lib import commandline |
George Burgess IV | 85e6933 | 2020-01-16 17:33:16 -0800 | [diff] [blame] | 19 | from chromite.lib import portage_util |
| 20 | |
| 21 | |
George Burgess IV | 87c4dbf | 2022-03-21 12:05:11 -0700 | [diff] [blame] | 22 | def contains_ebuild_hot_marker(file_path: Path): |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 23 | """Returns whether a file seems to be marked as hot.""" |
| 24 | ebuild_contents = file_path.read_text(encoding="utf-8") |
| 25 | return "cros_optimize_package_for_speed" in ebuild_contents |
George Burgess IV | 85e6933 | 2020-01-16 17:33:16 -0800 | [diff] [blame] | 26 | |
| 27 | |
George Burgess IV | 87c4dbf | 2022-03-21 12:05:11 -0700 | [diff] [blame] | 28 | def locate_all_package_ebuilds( |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 29 | overlays: Iterable[Path], |
| 30 | ) -> Iterable[Tuple[Path, str, List[Path]]]: |
| 31 | """Determines package -> ebuild mappings for all packages. |
George Burgess IV | 85e6933 | 2020-01-16 17:33:16 -0800 | [diff] [blame] | 32 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 33 | Yields a series of (package_path, package_name, [path_to_ebuilds]). This may |
| 34 | yield the same package name multiple times if it's available in multiple |
| 35 | overlays. |
| 36 | """ |
| 37 | for overlay in overlays: |
| 38 | # Note that portage_util.GetOverlayEBuilds can't be used here, since that |
| 39 | # specifically only searches for cros_workon candidates. We care about |
| 40 | # everything we can possibly build. |
| 41 | for package_dir in overlay.glob("*/*"): |
| 42 | ebuilds = list(package_dir.glob("*.ebuild")) |
| 43 | if ebuilds: |
| 44 | package_name = f"{package_dir.parent.name}/{package_dir.name}" |
| 45 | yield package_dir, package_name, ebuilds |
George Burgess IV | 85e6933 | 2020-01-16 17:33:16 -0800 | [diff] [blame] | 46 | |
| 47 | |
George Burgess IV | 87c4dbf | 2022-03-21 12:05:11 -0700 | [diff] [blame] | 48 | def locate_all_hot_env_packages(overlays: Iterable[Path]) -> Iterable[str]: |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 49 | """Yields packages marked hot by files in chromeos/config/env.""" |
| 50 | for overlay in overlays: |
| 51 | env_dir = overlay / "chromeos" / "config" / "env" |
| 52 | for f in env_dir.glob("*/*"): |
| 53 | if f.is_file() and contains_ebuild_hot_marker(f): |
| 54 | yield f"{f.parent.name}/{f.name}" |
George Burgess IV | 87c4dbf | 2022-03-21 12:05:11 -0700 | [diff] [blame] | 55 | |
| 56 | |
| 57 | def main(argv: List[str]): |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 58 | parser = commandline.ArgumentParser(description=__doc__) |
| 59 | parser.add_argument("--output", required=True, type=Path) |
| 60 | opts = parser.parse_args(argv) |
George Burgess IV | 85e6933 | 2020-01-16 17:33:16 -0800 | [diff] [blame] | 61 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 62 | ebuilds_found = 0 |
| 63 | packages_found = 0 |
| 64 | merged_packages = 0 |
George Burgess IV | 87c4dbf | 2022-03-21 12:05:11 -0700 | [diff] [blame] | 65 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 66 | overlays = portage_util.FindOverlays(overlay_type="both") |
| 67 | logging.debug("Found overlays %s", overlays) |
| 68 | overlays = [Path(x) for x in overlays] |
George Burgess IV | 87c4dbf | 2022-03-21 12:05:11 -0700 | [diff] [blame] | 69 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 70 | mappings = {x: True for x in locate_all_hot_env_packages(overlays)} |
| 71 | logging.debug("Found hot packages %r from env search", sorted(mappings)) |
| 72 | for package_dir, package, ebuilds in locate_all_package_ebuilds(overlays): |
| 73 | packages_found += 1 |
| 74 | ebuilds_found += len(ebuilds) |
| 75 | logging.debug( |
| 76 | "Found package %r in %r with ebuilds %r", |
| 77 | package, |
| 78 | package_dir, |
| 79 | ebuilds, |
| 80 | ) |
George Burgess IV | 85e6933 | 2020-01-16 17:33:16 -0800 | [diff] [blame] | 81 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 82 | is_marked_hot = any(contains_ebuild_hot_marker(x) for x in ebuilds) |
| 83 | if is_marked_hot: |
| 84 | logging.debug("Package is marked as hot") |
| 85 | else: |
| 86 | logging.debug("Package is not marked as hot") |
George Burgess IV | 85e6933 | 2020-01-16 17:33:16 -0800 | [diff] [blame] | 87 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 88 | if package in mappings: |
| 89 | logging.warning( |
| 90 | "Multiple entries found for package %r; merging", package |
| 91 | ) |
| 92 | merged_packages += 1 |
| 93 | mappings[package] = is_marked_hot or mappings[package] |
| 94 | else: |
| 95 | mappings[package] = is_marked_hot |
George Burgess IV | 85e6933 | 2020-01-16 17:33:16 -0800 | [diff] [blame] | 96 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 97 | hot_packages = sorted( |
| 98 | package for package, is_hot in mappings.items() if is_hot |
| 99 | ) |
George Burgess IV | 85e6933 | 2020-01-16 17:33:16 -0800 | [diff] [blame] | 100 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 101 | logging.info("%d ebuilds found", ebuilds_found) |
| 102 | logging.info("%d packages found", packages_found) |
| 103 | logging.info("%d packages merged", merged_packages) |
| 104 | logging.info("%d hot packages found, total", len(hot_packages)) |
George Burgess IV | 85e6933 | 2020-01-16 17:33:16 -0800 | [diff] [blame] | 105 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 106 | opts.output.write_text(json.dumps(hot_packages), encoding="utf-8") |