Abhishek Pandit-Subedi | b75bd56 | 2021-02-25 15:32:22 -0800 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
| 2 | # -*- coding: utf-8 -*- |
| 3 | # Copyright 2021 The Chromium OS Authors. All rights reserved. |
| 4 | # Use of this source code is governed by a BSD-style license that can be |
| 5 | # found in the LICENSE file. |
| 6 | """ This script cleans up the vendor directory. |
| 7 | """ |
Abhishek Pandit-Subedi | e393cb7 | 2021-08-22 10:41:13 -0700 | [diff] [blame] | 8 | import argparse |
George Burgess IV | 635f726 | 2022-08-09 21:32:20 -0700 | [diff] [blame] | 9 | import collections |
Abhishek Pandit-Subedi | 5065a0f | 2021-06-13 20:38:55 +0000 | [diff] [blame] | 10 | import hashlib |
Abhishek Pandit-Subedi | b75bd56 | 2021-02-25 15:32:22 -0800 | [diff] [blame] | 11 | import json |
| 12 | import os |
| 13 | import pathlib |
Abhishek Pandit-Subedi | e393cb7 | 2021-08-22 10:41:13 -0700 | [diff] [blame] | 14 | import re |
Abhishek Pandit-Subedi | f0eb6e0 | 2021-09-24 16:36:12 -0700 | [diff] [blame] | 15 | import shutil |
Abhishek Pandit-Subedi | 5065a0f | 2021-06-13 20:38:55 +0000 | [diff] [blame] | 16 | import subprocess |
George Burgess IV | 0483370 | 2022-08-09 22:00:38 -0700 | [diff] [blame] | 17 | import textwrap |
Abhishek Pandit-Subedi | ce0f5b2 | 2021-09-10 15:50:08 -0700 | [diff] [blame] | 18 | import toml |
Abhishek Pandit-Subedi | 5065a0f | 2021-06-13 20:38:55 +0000 | [diff] [blame] | 19 | |
Abhishek Pandit-Subedi | e393cb7 | 2021-08-22 10:41:13 -0700 | [diff] [blame] | 20 | # We only care about crates we're actually going to use and that's usually |
| 21 | # limited to ones with cfg(linux). For running `cargo metadata`, limit results |
| 22 | # to only this platform |
| 23 | DEFAULT_PLATFORM_FILTER = "x86_64-unknown-linux-gnu" |
| 24 | |
Abhishek Pandit-Subedi | 5065a0f | 2021-06-13 20:38:55 +0000 | [diff] [blame] | 25 | |
| 26 | def _rerun_checksums(package_path): |
| 27 | """Re-run checksums for given package. |
| 28 | |
| 29 | Writes resulting checksums to $package_path/.cargo-checksum.json. |
| 30 | """ |
Abhishek Pandit-Subedi | e393cb7 | 2021-08-22 10:41:13 -0700 | [diff] [blame] | 31 | hashes = dict() |
Abhishek Pandit-Subedi | 5065a0f | 2021-06-13 20:38:55 +0000 | [diff] [blame] | 32 | checksum_path = os.path.join(package_path, '.cargo-checksum.json') |
| 33 | if not pathlib.Path(checksum_path).is_file(): |
| 34 | return False |
| 35 | |
| 36 | with open(checksum_path, 'r') as fread: |
| 37 | contents = json.load(fread) |
| 38 | |
| 39 | for root, _, files in os.walk(package_path, topdown=True): |
| 40 | for f in files: |
| 41 | # Don't checksum an existing checksum file |
| 42 | if f == ".cargo-checksum.json": |
| 43 | continue |
| 44 | |
| 45 | file_path = os.path.join(root, f) |
| 46 | with open(file_path, 'rb') as frb: |
| 47 | m = hashlib.sha256() |
| 48 | m.update(frb.read()) |
| 49 | d = m.hexdigest() |
| 50 | |
| 51 | # Key is relative to the package path so strip from beginning |
| 52 | key = os.path.relpath(file_path, package_path) |
| 53 | hashes[key] = d |
| 54 | |
| 55 | if hashes: |
Abhishek Pandit-Subedi | e393cb7 | 2021-08-22 10:41:13 -0700 | [diff] [blame] | 56 | print("{} regenerated {} hashes".format(package_path, |
| 57 | len(hashes.keys()))) |
Abhishek Pandit-Subedi | 5065a0f | 2021-06-13 20:38:55 +0000 | [diff] [blame] | 58 | contents['files'] = hashes |
Abhishek Pandit-Subedi | 5065a0f | 2021-06-13 20:38:55 +0000 | [diff] [blame] | 59 | with open(checksum_path, 'w') as fwrite: |
Abhishek Pandit-Subedi | e393cb7 | 2021-08-22 10:41:13 -0700 | [diff] [blame] | 60 | json.dump(contents, fwrite, sort_keys=True) |
Abhishek Pandit-Subedi | 5065a0f | 2021-06-13 20:38:55 +0000 | [diff] [blame] | 61 | |
| 62 | return True |
Abhishek Pandit-Subedi | b75bd56 | 2021-02-25 15:32:22 -0800 | [diff] [blame] | 63 | |
| 64 | |
| 65 | def _remove_OWNERS_checksum(root): |
| 66 | """ Delete all OWNERS files from the checksum file. |
| 67 | |
Abhishek Pandit-Subedi | 5065a0f | 2021-06-13 20:38:55 +0000 | [diff] [blame] | 68 | Args: |
| 69 | root: Root directory for the vendored crate. |
Abhishek Pandit-Subedi | b75bd56 | 2021-02-25 15:32:22 -0800 | [diff] [blame] | 70 | |
Abhishek Pandit-Subedi | 5065a0f | 2021-06-13 20:38:55 +0000 | [diff] [blame] | 71 | Returns: |
| 72 | True if OWNERS was found and cleaned up. Otherwise False. |
| 73 | """ |
Abhishek Pandit-Subedi | b75bd56 | 2021-02-25 15:32:22 -0800 | [diff] [blame] | 74 | checksum_path = os.path.join(root, '.cargo-checksum.json') |
| 75 | if not pathlib.Path(checksum_path).is_file(): |
| 76 | return False |
| 77 | |
| 78 | with open(checksum_path, 'r') as fread: |
| 79 | contents = json.load(fread) |
| 80 | |
| 81 | del_keys = [] |
| 82 | for cfile in contents['files']: |
| 83 | if 'OWNERS' in cfile: |
| 84 | del_keys.append(cfile) |
| 85 | |
| 86 | for key in del_keys: |
| 87 | del contents['files'][key] |
| 88 | |
| 89 | if del_keys: |
| 90 | print('{} deleted: {}'.format(root, del_keys)) |
| 91 | with open(checksum_path, 'w') as fwrite: |
Abhishek Pandit-Subedi | e393cb7 | 2021-08-22 10:41:13 -0700 | [diff] [blame] | 92 | json.dump(contents, fwrite, sort_keys=True) |
Abhishek Pandit-Subedi | b75bd56 | 2021-02-25 15:32:22 -0800 | [diff] [blame] | 93 | |
| 94 | return bool(del_keys) |
| 95 | |
| 96 | |
| 97 | def cleanup_owners(vendor_path): |
| 98 | """ Remove owners checksums from the vendor directory. |
| 99 | |
Abhishek Pandit-Subedi | 5065a0f | 2021-06-13 20:38:55 +0000 | [diff] [blame] | 100 | We currently do not check in the OWNERS files from vendored crates because |
| 101 | they interfere with the find-owners functionality in gerrit. This cleanup |
| 102 | simply finds all instances of "OWNERS" in the checksum files within and |
| 103 | removes them. |
Abhishek Pandit-Subedi | b75bd56 | 2021-02-25 15:32:22 -0800 | [diff] [blame] | 104 | |
Abhishek Pandit-Subedi | 5065a0f | 2021-06-13 20:38:55 +0000 | [diff] [blame] | 105 | Args: |
| 106 | vendor_path: Absolute path to vendor directory. |
| 107 | """ |
Abhishek Pandit-Subedi | b75bd56 | 2021-02-25 15:32:22 -0800 | [diff] [blame] | 108 | deps_cleaned = [] |
| 109 | for root, dirs, _ in os.walk(vendor_path): |
| 110 | for d in dirs: |
| 111 | removed = _remove_OWNERS_checksum(os.path.join(root, d)) |
| 112 | if removed: |
| 113 | deps_cleaned.append(d) |
| 114 | |
| 115 | if deps_cleaned: |
| 116 | print('Cleanup owners:\n {}'.format("\n".join(deps_cleaned))) |
| 117 | |
| 118 | |
Abhishek Pandit-Subedi | 5065a0f | 2021-06-13 20:38:55 +0000 | [diff] [blame] | 119 | def apply_single_patch(patch, workdir): |
| 120 | """Apply a single patch and return whether it was successful. |
| 121 | |
| 122 | Returns: |
| 123 | True if successful. False otherwise. |
| 124 | """ |
Abhishek Pandit-Subedi | 5065a0f | 2021-06-13 20:38:55 +0000 | [diff] [blame] | 125 | proc = subprocess.run(["patch", "-p1", "-i", patch], cwd=workdir) |
| 126 | return proc.returncode == 0 |
| 127 | |
| 128 | |
George Burgess IV | 30c5c36 | 2022-08-19 17:05:02 -0700 | [diff] [blame] | 129 | def apply_patch_script(script, workdir): |
| 130 | """Run the given patch script, returning whether it exited cleanly. |
| 131 | |
| 132 | Returns: |
| 133 | True if successful. False otherwise. |
| 134 | """ |
| 135 | return subprocess.run([script], cwd=workdir).returncode == 0 |
| 136 | |
| 137 | |
George Burgess IV | 635f726 | 2022-08-09 21:32:20 -0700 | [diff] [blame] | 138 | def determine_vendor_crates(vendor_path): |
| 139 | """Returns a map of {crate_name: [directory]} at the given vendor_path.""" |
| 140 | result = collections.defaultdict(list) |
| 141 | for crate_name_plus_ver in os.listdir(vendor_path): |
George Burgess IV | 40cc91c | 2022-08-15 13:07:40 -0700 | [diff] [blame] | 142 | name, _ = crate_name_plus_ver.rsplit('-', 1) |
| 143 | result[name].append(crate_name_plus_ver) |
George Burgess IV | 635f726 | 2022-08-09 21:32:20 -0700 | [diff] [blame] | 144 | |
| 145 | for crate_list in result.values(): |
George Burgess IV | 40cc91c | 2022-08-15 13:07:40 -0700 | [diff] [blame] | 146 | crate_list.sort() |
George Burgess IV | 635f726 | 2022-08-09 21:32:20 -0700 | [diff] [blame] | 147 | return result |
| 148 | |
| 149 | |
Abhishek Pandit-Subedi | 5065a0f | 2021-06-13 20:38:55 +0000 | [diff] [blame] | 150 | def apply_patches(patches_path, vendor_path): |
| 151 | """Finds patches and applies them to sub-folders in the vendored crates. |
| 152 | |
| 153 | Args: |
| 154 | patches_path: Path to folder with patches. Expect all patches to be one |
| 155 | level down (matching the crate name). |
| 156 | vendor_path: Root path to vendored crates directory. |
| 157 | """ |
| 158 | checksums_for = {} |
| 159 | |
| 160 | # Don't bother running if patches directory is empty |
| 161 | if not pathlib.Path(patches_path).is_dir(): |
Abhishek Pandit-Subedi | e393cb7 | 2021-08-22 10:41:13 -0700 | [diff] [blame] | 162 | return |
Abhishek Pandit-Subedi | 5065a0f | 2021-06-13 20:38:55 +0000 | [diff] [blame] | 163 | |
George Burgess IV | 30c5c36 | 2022-08-19 17:05:02 -0700 | [diff] [blame] | 164 | patches_failed = False |
George Burgess IV | 635f726 | 2022-08-09 21:32:20 -0700 | [diff] [blame] | 165 | vendor_crate_map = determine_vendor_crates(vendor_path) |
Abhishek Pandit-Subedi | 5065a0f | 2021-06-13 20:38:55 +0000 | [diff] [blame] | 166 | # Look for all patches and apply them |
| 167 | for d in os.listdir(patches_path): |
| 168 | dir_path = os.path.join(patches_path, d) |
| 169 | |
| 170 | # We don't process patches in root dir |
| 171 | if not os.path.isdir(dir_path): |
| 172 | continue |
| 173 | |
George Burgess IV | 30c5c36 | 2022-08-19 17:05:02 -0700 | [diff] [blame] | 174 | # We accept one of two forms here: |
| 175 | # - direct targets (these name # `${crate_name}-${version}`) |
| 176 | # - simply the crate name (which applies to all versions of the |
| 177 | # crate) |
| 178 | direct_target = os.path.join(vendor_path, d) |
| 179 | if os.path.isdir(direct_target): |
| 180 | patch_targets = [d] |
| 181 | elif d in vendor_crate_map: |
| 182 | patch_targets = vendor_crate_map[d] |
| 183 | else: |
| 184 | raise RuntimeError(f'Unknown crate in {vendor_path}: {d}') |
| 185 | |
George Burgess IV | 635f726 | 2022-08-09 21:32:20 -0700 | [diff] [blame] | 186 | for patch in os.listdir(dir_path): |
Abhishek Pandit-Subedi | 5065a0f | 2021-06-13 20:38:55 +0000 | [diff] [blame] | 187 | file_path = os.path.join(dir_path, patch) |
| 188 | |
| 189 | # Skip if not a patch file |
George Burgess IV | 30c5c36 | 2022-08-19 17:05:02 -0700 | [diff] [blame] | 190 | if not os.path.isfile(file_path): |
Abhishek Pandit-Subedi | 5065a0f | 2021-06-13 20:38:55 +0000 | [diff] [blame] | 191 | continue |
| 192 | |
George Burgess IV | 30c5c36 | 2022-08-19 17:05:02 -0700 | [diff] [blame] | 193 | if patch.endswith(".patch"): |
| 194 | apply = apply_single_patch |
| 195 | elif os.access(file_path, os.X_OK): |
| 196 | apply = apply_patch_script |
George Burgess IV | 635f726 | 2022-08-09 21:32:20 -0700 | [diff] [blame] | 197 | else: |
George Burgess IV | 30c5c36 | 2022-08-19 17:05:02 -0700 | [diff] [blame] | 198 | # Unrecognized. Skip it. |
| 199 | continue |
| 200 | |
| 201 | for target_name in patch_targets: |
| 202 | checksums_for[target_name] = True |
| 203 | target = os.path.join(vendor_path, target_name) |
| 204 | print(f"-- Applying {file_path} to {target}") |
| 205 | if not apply(file_path, target): |
| 206 | print(f"Failed to apply {file_path} to {target}") |
| 207 | patches_failed = True |
| 208 | |
| 209 | # Do this late, so we can report all of the failing patches in one |
| 210 | # invocation. |
| 211 | if patches_failed: |
| 212 | raise ValueError('Patches failed; please see above logs') |
Abhishek Pandit-Subedi | 5065a0f | 2021-06-13 20:38:55 +0000 | [diff] [blame] | 213 | |
Abhishek Pandit-Subedi | 5065a0f | 2021-06-13 20:38:55 +0000 | [diff] [blame] | 214 | # Re-run checksums for all modified packages since we applied patches. |
| 215 | for key in checksums_for.keys(): |
| 216 | _rerun_checksums(os.path.join(vendor_path, key)) |
| 217 | |
Abhishek Pandit-Subedi | e393cb7 | 2021-08-22 10:41:13 -0700 | [diff] [blame] | 218 | |
George Burgess IV | 40cc91c | 2022-08-15 13:07:40 -0700 | [diff] [blame] | 219 | def fetch_project_cargo_toml_files(working_dir): |
| 220 | """Returns all Cargo.toml files under working_dir.""" |
| 221 | projects = working_dir / 'projects' |
| 222 | return sorted(projects.glob('**/Cargo.toml')) |
| 223 | |
| 224 | |
Abhishek Pandit-Subedi | fa90238 | 2021-08-20 11:04:33 -0700 | [diff] [blame] | 225 | def run_cargo_vendor(working_dir): |
| 226 | """Runs cargo vendor. |
| 227 | |
| 228 | Args: |
| 229 | working_dir: Directory to run inside. This should be the directory where |
Abhishek Pandit-Subedi | ce0f5b2 | 2021-09-10 15:50:08 -0700 | [diff] [blame] | 230 | Cargo.toml is kept. |
Abhishek Pandit-Subedi | fa90238 | 2021-08-20 11:04:33 -0700 | [diff] [blame] | 231 | """ |
George Burgess IV | 635f726 | 2022-08-09 21:32:20 -0700 | [diff] [blame] | 232 | # Cargo will refuse to revendor into versioned directories, which leads to |
| 233 | # repeated `./vendor.py` invocations trying to apply patches to |
| 234 | # already-patched sources. Remove the existing vendor directory to avoid |
| 235 | # this. |
| 236 | vendor_dir = working_dir / 'vendor' |
| 237 | if vendor_dir.exists(): |
George Burgess IV | 40cc91c | 2022-08-15 13:07:40 -0700 | [diff] [blame] | 238 | shutil.rmtree(vendor_dir) |
| 239 | |
| 240 | cargo_cmdline = ['cargo', 'vendor', '--versioned-dirs', '-v'] |
| 241 | for i, cargo_toml in enumerate(fetch_project_cargo_toml_files(working_dir)): |
| 242 | # `cargo vendor` requires a 'root' manifest; select an arbitrary one, |
| 243 | # then tack other manifests on to it. Order doesn't really matter. |
| 244 | if i == 0: |
| 245 | cargo_cmdline.append('--manifest-path') |
| 246 | else: |
| 247 | cargo_cmdline.append('-s') |
| 248 | cargo_cmdline.append(str(cargo_toml)) |
| 249 | |
| 250 | # Autocreate src/lib.rs if necessary. |
| 251 | lib_rs = cargo_toml.parent / 'src' / 'lib.rs' |
| 252 | if not lib_rs.exists(): |
| 253 | lib_rs.parent.mkdir(exist_ok=True) |
| 254 | lib_rs.write_bytes(b'') |
| 255 | |
| 256 | # Always place vendor/ at the top-level directory. |
| 257 | cargo_cmdline += ('--', 'vendor') |
| 258 | subprocess.check_call(cargo_cmdline, cwd=working_dir) |
Abhishek Pandit-Subedi | 5065a0f | 2021-06-13 20:38:55 +0000 | [diff] [blame] | 259 | |
Abhishek Pandit-Subedi | ce0f5b2 | 2021-09-10 15:50:08 -0700 | [diff] [blame] | 260 | |
Abhishek Pandit-Subedi | e393cb7 | 2021-08-22 10:41:13 -0700 | [diff] [blame] | 261 | def load_metadata(working_dir, filter_platform=DEFAULT_PLATFORM_FILTER): |
George Burgess IV | 40cc91c | 2022-08-15 13:07:40 -0700 | [diff] [blame] | 262 | """Load metadata for all projects under a given directory. |
Abhishek Pandit-Subedi | e393cb7 | 2021-08-22 10:41:13 -0700 | [diff] [blame] | 263 | |
| 264 | Args: |
George Burgess IV | 40cc91c | 2022-08-15 13:07:40 -0700 | [diff] [blame] | 265 | working_dir: Base directory to run from. |
Abhishek Pandit-Subedi | e393cb7 | 2021-08-22 10:41:13 -0700 | [diff] [blame] | 266 | filter_platform: Filter packages to ones configured for this platform. |
| 267 | """ |
George Burgess IV | 40cc91c | 2022-08-15 13:07:40 -0700 | [diff] [blame] | 268 | metadata_objects = [] |
| 269 | for manifest_path in fetch_project_cargo_toml_files(working_dir): |
| 270 | cmd = [ |
| 271 | 'cargo', 'metadata', '--format-version', '1', '--manifest-path', |
| 272 | manifest_path |
| 273 | ] |
| 274 | # Conditionally add platform filter |
| 275 | if filter_platform: |
| 276 | cmd += ("--filter-platform", filter_platform) |
| 277 | output = subprocess.check_output(cmd, cwd=working_dir) |
| 278 | metadata_objects.append(json.loads(output)) |
| 279 | return metadata_objects |
Abhishek Pandit-Subedi | f0eb6e0 | 2021-09-24 16:36:12 -0700 | [diff] [blame] | 280 | |
Abhishek Pandit-Subedi | f0eb6e0 | 2021-09-24 16:36:12 -0700 | [diff] [blame] | 281 | |
George Burgess IV | 40cc91c | 2022-08-15 13:07:40 -0700 | [diff] [blame] | 282 | def load_all_metadata_packages(working_dir, |
| 283 | filter_platform=DEFAULT_PLATFORM_FILTER, |
| 284 | unique=False): |
| 285 | """Returns a list of all packages returned by load_metadata.""" |
| 286 | results = [] |
| 287 | for metadata in load_metadata(working_dir, filter_platform): |
| 288 | results += metadata['packages'] |
Abhishek Pandit-Subedi | e393cb7 | 2021-08-22 10:41:13 -0700 | [diff] [blame] | 289 | |
George Burgess IV | 40cc91c | 2022-08-15 13:07:40 -0700 | [diff] [blame] | 290 | if not unique: |
| 291 | return results |
Abhishek Pandit-Subedi | e393cb7 | 2021-08-22 10:41:13 -0700 | [diff] [blame] | 292 | |
George Burgess IV | 40cc91c | 2022-08-15 13:07:40 -0700 | [diff] [blame] | 293 | new_results = [] |
| 294 | seen_keys = set() |
| 295 | for item in results: |
| 296 | key = item['id'] |
| 297 | if key in seen_keys: |
| 298 | continue |
| 299 | seen_keys.add(key) |
| 300 | new_results.append(item) |
| 301 | return new_results |
Abhishek Pandit-Subedi | e393cb7 | 2021-08-22 10:41:13 -0700 | [diff] [blame] | 302 | |
| 303 | class LicenseManager: |
| 304 | """ Manage consolidating licenses for all packages.""" |
| 305 | |
| 306 | # These are all the licenses we support. Keys are what is seen in metadata and |
| 307 | # values are what is expected by the ebuild. |
| 308 | SUPPORTED_LICENSES = { |
| 309 | 'Apache-2.0': 'Apache-2.0', |
| 310 | 'MIT': 'MIT', |
| 311 | 'BSD-3-Clause': 'BSD-3', |
George Burgess IV | 4ae4206 | 2022-08-15 18:54:51 -0700 | [diff] [blame] | 312 | 'ISC': 'ISC', |
| 313 | 'unicode': 'unicode', |
Abhishek Pandit-Subedi | e393cb7 | 2021-08-22 10:41:13 -0700 | [diff] [blame] | 314 | } |
| 315 | |
| 316 | # Prefer to take attribution licenses in this order. All these require that |
| 317 | # we actually use the license file found in the package so they MUST have |
| 318 | # a license file set. |
| 319 | PREFERRED_ATTRIB_LICENSE_ORDER = ['MIT', 'BSD-3', 'ISC'] |
| 320 | |
| 321 | # If Apache license is found, always prefer it (simplifies attribution) |
| 322 | APACHE_LICENSE = 'Apache-2.0' |
| 323 | |
| 324 | # Regex for license files found in the vendored directories. Search for |
| 325 | # these files with re.IGNORECASE. |
| 326 | # |
| 327 | # These will be searched in order with the earlier entries being preferred. |
| 328 | LICENSE_NAMES_REGEX = [ |
| 329 | r'^license-mit$', |
| 330 | r'^copyright$', |
| 331 | r'^licen[cs]e.*$', |
| 332 | ] |
| 333 | |
| 334 | # Some crates have their license file in other crates. This usually occurs |
| 335 | # because multiple crates are published from the same git repository and the |
| 336 | # license isn't updated in each sub-crate. In these cases, we can just |
| 337 | # ignore these packages. |
| 338 | MAP_LICENSE_TO_OTHER = { |
| 339 | 'failure_derive': 'failure', |
| 340 | 'grpcio-compiler': 'grpcio', |
| 341 | 'grpcio-sys': 'grpcio', |
| 342 | 'rustyline-derive': 'rustyline', |
| 343 | } |
| 344 | |
| 345 | # Map a package to a specific license and license file. Only use this if |
| 346 | # a package doesn't have an easily discoverable license or exports its |
| 347 | # license in a weird way. Prefer to patch the project with a license and |
| 348 | # upstream the patch instead. |
| 349 | STATIC_LICENSE_MAP = { |
| 350 | # "package name": ( "license name", "license file relative location") |
| 351 | } |
| 352 | |
| 353 | def __init__(self, working_dir, vendor_dir): |
| 354 | self.working_dir = working_dir |
| 355 | self.vendor_dir = vendor_dir |
| 356 | |
| 357 | def _find_license_in_dir(self, search_dir): |
| 358 | for p in os.listdir(search_dir): |
| 359 | # Ignore anything that's not a file |
| 360 | if not os.path.isfile(os.path.join(search_dir, p)): |
| 361 | continue |
| 362 | |
| 363 | # Now check if the name matches any of the regexes |
| 364 | # We'll return the first matching file. |
| 365 | for regex in self.LICENSE_NAMES_REGEX: |
| 366 | if re.search(regex, p, re.IGNORECASE): |
| 367 | yield os.path.join(search_dir, p) |
| 368 | break |
| 369 | |
| 370 | def _guess_license_type(self, license_file): |
| 371 | if '-MIT' in license_file: |
| 372 | return 'MIT' |
| 373 | elif '-APACHE' in license_file: |
| 374 | return 'APACHE' |
| 375 | elif '-BSD' in license_file: |
| 376 | return 'BSD-3' |
| 377 | |
| 378 | with open(license_file, 'r') as f: |
| 379 | lines = f.read() |
| 380 | if 'MIT' in lines: |
| 381 | return 'MIT' |
| 382 | elif 'Apache' in lines: |
| 383 | return 'APACHE' |
| 384 | elif 'BSD 3-Clause' in lines: |
| 385 | return 'BSD-3' |
| 386 | |
| 387 | return '' |
| 388 | |
George Burgess IV | 0483370 | 2022-08-09 22:00:38 -0700 | [diff] [blame] | 389 | def generate_license(self, skip_license_check, print_map_to_file, |
| 390 | license_shorthand_file): |
Abhishek Pandit-Subedi | e393cb7 | 2021-08-22 10:41:13 -0700 | [diff] [blame] | 391 | """Generate single massive license file from metadata.""" |
George Burgess IV | 40cc91c | 2022-08-15 13:07:40 -0700 | [diff] [blame] | 392 | all_packages = load_all_metadata_packages(self.working_dir, unique=True) |
Abhishek Pandit-Subedi | e393cb7 | 2021-08-22 10:41:13 -0700 | [diff] [blame] | 393 | |
| 394 | has_license_types = set() |
| 395 | bad_licenses = {} |
| 396 | |
| 397 | # Keep license map ordered so it generates a consistent license map |
| 398 | license_map = {} |
| 399 | |
| 400 | skip_license_check = skip_license_check or [] |
George Burgess IV | 4ae4206 | 2022-08-15 18:54:51 -0700 | [diff] [blame] | 401 | has_unicode_license = False |
Abhishek Pandit-Subedi | e393cb7 | 2021-08-22 10:41:13 -0700 | [diff] [blame] | 402 | |
George Burgess IV | 40cc91c | 2022-08-15 13:07:40 -0700 | [diff] [blame] | 403 | for package in all_packages: |
| 404 | # Skip the synthesized Cargo.toml packages that exist solely to |
| 405 | # list dependencies. |
| 406 | if 'path+file:///' in package['id']: |
Abhishek Pandit-Subedi | e393cb7 | 2021-08-22 10:41:13 -0700 | [diff] [blame] | 407 | continue |
| 408 | |
George Burgess IV | 40cc91c | 2022-08-15 13:07:40 -0700 | [diff] [blame] | 409 | pkg_name = package['name'] |
Abhishek Pandit-Subedi | e393cb7 | 2021-08-22 10:41:13 -0700 | [diff] [blame] | 410 | if pkg_name in skip_license_check: |
| 411 | print( |
| 412 | "Skipped license check on {}. Reason: Skipped from command line" |
| 413 | .format(pkg_name)) |
| 414 | continue |
| 415 | |
| 416 | if pkg_name in self.MAP_LICENSE_TO_OTHER: |
| 417 | print( |
| 418 | 'Skipped license check on {}. Reason: License already in {}' |
| 419 | .format(pkg_name, self.MAP_LICENSE_TO_OTHER[pkg_name])) |
| 420 | continue |
| 421 | |
| 422 | # Check if we have a static license map for this package. Use the |
| 423 | # static values if we have it already set. |
| 424 | if pkg_name in self.STATIC_LICENSE_MAP: |
| 425 | (license, license_file) = self.STATIC_LICENSE_MAP[pkg_name] |
| 426 | license_map[pkg_name] = { |
| 427 | "license": license, |
| 428 | "license_file": license_file, |
| 429 | } |
| 430 | continue |
| 431 | |
| 432 | license_files = [] |
George Burgess IV | 93ba473 | 2022-08-13 14:10:10 -0700 | [diff] [blame] | 433 | # use `or ''` instead of get's default, since `package` may have a |
| 434 | # None value for 'license'. |
| 435 | license = package.get('license') or '' |
Abhishek Pandit-Subedi | e393cb7 | 2021-08-22 10:41:13 -0700 | [diff] [blame] | 436 | |
| 437 | # We ignore the metadata for license file because most crates don't |
| 438 | # have it set. Just scan the source for licenses. |
George Burgess IV | 635f726 | 2022-08-09 21:32:20 -0700 | [diff] [blame] | 439 | pkg_version = package['version'] |
George Burgess IV | 40cc91c | 2022-08-15 13:07:40 -0700 | [diff] [blame] | 440 | license_files = list(self._find_license_in_dir( |
| 441 | os.path.join(self.vendor_dir, f'{pkg_name}-{pkg_version}'))) |
Abhishek Pandit-Subedi | e393cb7 | 2021-08-22 10:41:13 -0700 | [diff] [blame] | 442 | |
George Burgess IV | 4ae4206 | 2022-08-15 18:54:51 -0700 | [diff] [blame] | 443 | # FIXME(b/240953811): The code later in this loop is only |
| 444 | # structured to handle ORs, not ANDs. Fortunately, this license in |
| 445 | # particular is `AND`ed between a super common license (Apache) and |
| 446 | # a more obscure one (unicode). This hack is specifically intended |
| 447 | # for the `unicode-ident` crate, though no crate name check is |
| 448 | # made, since it's OK other crates happen to have this license. |
| 449 | if license == '(MIT OR Apache-2.0) AND Unicode-DFS-2016': |
| 450 | has_unicode_license = True |
| 451 | # We'll check later to be sure MIT or Apache-2.0 is represented |
| 452 | # properly. |
| 453 | for x in license_files: |
| 454 | if os.path.basename(x) == 'LICENSE-UNICODE': |
| 455 | license_file = x |
| 456 | break |
| 457 | else: |
| 458 | raise ValueError('No LICENSE-UNICODE found in ' |
| 459 | f'{license_files}') |
| 460 | license_map[pkg_name] = { |
| 461 | "license": license, |
| 462 | "license_file": license_file, |
| 463 | } |
| 464 | has_license_types.add('unicode') |
| 465 | continue |
| 466 | |
Abhishek Pandit-Subedi | e393cb7 | 2021-08-22 10:41:13 -0700 | [diff] [blame] | 467 | # If there are multiple licenses, they are delimited with "OR" or "/" |
| 468 | delim = ' OR ' if ' OR ' in license else '/' |
George Burgess IV | 40cc91c | 2022-08-15 13:07:40 -0700 | [diff] [blame] | 469 | found = [x.strip() for x in license.split(delim)] |
Abhishek Pandit-Subedi | e393cb7 | 2021-08-22 10:41:13 -0700 | [diff] [blame] | 470 | |
| 471 | # Filter licenses to ones we support |
| 472 | licenses_or = [ |
| 473 | self.SUPPORTED_LICENSES[f] for f in found |
| 474 | if f in self.SUPPORTED_LICENSES |
| 475 | ] |
| 476 | |
| 477 | # If apache license is found, always prefer it because it simplifies |
| 478 | # license attribution (we can use existing Apache notice) |
| 479 | if self.APACHE_LICENSE in licenses_or: |
| 480 | has_license_types.add(self.APACHE_LICENSE) |
| 481 | license_map[pkg_name] = {'license': self.APACHE_LICENSE} |
| 482 | |
| 483 | # Handle single license that has at least one license file |
| 484 | # We pick the first license file and the license |
| 485 | elif len(licenses_or) == 1: |
| 486 | if license_files: |
| 487 | l = licenses_or[0] |
| 488 | lf = license_files[0] |
| 489 | |
| 490 | has_license_types.add(l) |
| 491 | license_map[pkg_name] = { |
| 492 | 'license': l, |
| 493 | 'license_file': os.path.relpath(lf, self.working_dir), |
| 494 | } |
| 495 | else: |
| 496 | bad_licenses[pkg_name] = "{} missing license file".format( |
| 497 | licenses_or[0]) |
| 498 | # Handle multiple licenses |
| 499 | elif len(licenses_or) > 1: |
| 500 | # Check preferred licenses in order |
| 501 | license_found = False |
| 502 | for l in self.PREFERRED_ATTRIB_LICENSE_ORDER: |
| 503 | if not l in licenses_or: |
| 504 | continue |
| 505 | |
| 506 | for f in license_files: |
| 507 | if self._guess_license_type(f) == l: |
| 508 | license_found = True |
| 509 | has_license_types.add(l) |
| 510 | license_map[pkg_name] = { |
| 511 | 'license': |
| 512 | l, |
| 513 | 'license_file': |
| 514 | os.path.relpath(f, self.working_dir), |
| 515 | } |
| 516 | break |
| 517 | |
| 518 | # Break out of loop if license is found |
| 519 | if license_found: |
| 520 | break |
| 521 | else: |
| 522 | bad_licenses[pkg_name] = license |
| 523 | |
| 524 | # If we had any bad licenses, we need to abort |
| 525 | if bad_licenses: |
| 526 | for k in bad_licenses.keys(): |
| 527 | print("{} had no acceptable licenses: {}".format( |
| 528 | k, bad_licenses[k])) |
| 529 | raise Exception("Bad licenses in vendored packages.") |
| 530 | |
| 531 | # Write license map to file |
| 532 | if print_map_to_file: |
| 533 | with open(os.path.join(self.working_dir, print_map_to_file), |
| 534 | 'w') as lfile: |
| 535 | json.dump(license_map, lfile, sort_keys=True) |
| 536 | |
| 537 | # Raise missing licenses unless we have a valid reason to ignore them |
| 538 | raise_missing_license = False |
| 539 | for name, v in license_map.items(): |
| 540 | if 'license_file' not in v and v.get('license', |
| 541 | '') != self.APACHE_LICENSE: |
| 542 | raise_missing_license = True |
| 543 | print(" {}: Missing license file. Fix or add to ignorelist.". |
| 544 | format(name)) |
| 545 | |
| 546 | if raise_missing_license: |
| 547 | raise Exception( |
| 548 | "Unhandled missing license file. " |
| 549 | "Make sure all are accounted for before continuing.") |
| 550 | |
George Burgess IV | 4ae4206 | 2022-08-15 18:54:51 -0700 | [diff] [blame] | 551 | if has_unicode_license: |
| 552 | if self.APACHE_LICENSE not in has_license_types: |
| 553 | raise ValueError('Need the apache license; currently have: ' |
| 554 | f'{sorted(has_license_types)}') |
| 555 | |
George Burgess IV | 0483370 | 2022-08-09 22:00:38 -0700 | [diff] [blame] | 556 | sorted_licenses = sorted(has_license_types) |
| 557 | print("Add the following licenses to the ebuild:\n", |
| 558 | sorted_licenses) |
| 559 | header = textwrap.dedent("""\ |
| 560 | # File to describe the licenses used by this registry. |
| 561 | # Used to it's easy to automatically verify ebuilds are updated. |
| 562 | # Each line is a license. Lines starting with # are comments. |
| 563 | """) |
| 564 | with open(license_shorthand_file, 'w', encoding='utf-8') as f: |
| 565 | f.write(header) |
| 566 | f.write('\n'.join(sorted_licenses)) |
Abhishek Pandit-Subedi | e393cb7 | 2021-08-22 10:41:13 -0700 | [diff] [blame] | 567 | |
| 568 | |
Abhishek Pandit-Subedi | ce0f5b2 | 2021-09-10 15:50:08 -0700 | [diff] [blame] | 569 | # TODO(abps) - This needs to be replaced with datalog later. We should compile |
| 570 | # all crab files into datalog and query it with our requirements |
| 571 | # instead. |
| 572 | class CrabManager: |
| 573 | """Manage audit files.""" |
| 574 | def __init__(self, working_dir, crab_dir): |
| 575 | self.working_dir = working_dir |
| 576 | self.crab_dir = crab_dir |
| 577 | |
| 578 | def _check_bad_traits(self, crabdata): |
| 579 | """Checks that a package's crab audit meets our requirements. |
| 580 | |
| 581 | Args: |
| 582 | crabdata: Dict with crab keys in standard templated format. |
| 583 | """ |
| 584 | common = crabdata['common'] |
| 585 | # TODO(b/200578411) - Figure out what conditions we should enforce as |
| 586 | # part of the audit. |
| 587 | conditions = [ |
| 588 | common.get('deny', None), |
| 589 | ] |
| 590 | |
| 591 | # If any conditions are true, this crate is not acceptable. |
| 592 | return any(conditions) |
| 593 | |
| 594 | def verify_traits(self): |
| 595 | """ Verify that all required CRAB traits for this repository are met. |
| 596 | """ |
George Burgess IV | 40cc91c | 2022-08-15 13:07:40 -0700 | [diff] [blame] | 597 | all_packages = load_all_metadata_packages(self.working_dir, unique=True) |
Abhishek Pandit-Subedi | ce0f5b2 | 2021-09-10 15:50:08 -0700 | [diff] [blame] | 598 | |
| 599 | failing_crates = {} |
| 600 | |
| 601 | # Verify all packages have a CRAB file associated with it and they meet |
| 602 | # all our required traits |
George Burgess IV | 40cc91c | 2022-08-15 13:07:40 -0700 | [diff] [blame] | 603 | for package in all_packages: |
| 604 | # Skip the synthesized Cargo.toml packages that exist solely to |
| 605 | # list dependencies. |
| 606 | if 'path+file:///' in package['id']: |
Abhishek Pandit-Subedi | ce0f5b2 | 2021-09-10 15:50:08 -0700 | [diff] [blame] | 607 | continue |
| 608 | |
| 609 | crabname = "{}-{}".format(package['name'], package['version']) |
| 610 | filename = os.path.join(self.crab_dir, "{}.toml".format(crabname)) |
| 611 | |
| 612 | # If crab file doesn't exist, the crate fails |
| 613 | if not os.path.isfile(filename): |
| 614 | failing_crates[crabname] = "No crab file".format(filename) |
| 615 | continue |
| 616 | |
| 617 | with open(filename, 'r') as f: |
| 618 | crabdata = toml.loads(f.read()) |
| 619 | |
| 620 | # If crab file's crate_name and version keys don't match this |
| 621 | # package, it also fails. This is just housekeeping... |
| 622 | if package['name'] != crabdata['crate_name'] or package[ |
| 623 | 'version'] != crabdata['version']: |
| 624 | failing_crates[crabname] = "Crate name or version don't match" |
| 625 | continue |
| 626 | |
| 627 | if self._check_bad_traits(crabdata): |
| 628 | failing_crates[crabname] = "Failed bad traits check" |
| 629 | |
| 630 | # If we had any failing crates, list them now |
| 631 | if failing_crates: |
| 632 | print('Failed CRAB audit:') |
| 633 | for k, v in failing_crates.items(): |
| 634 | print(' {}: {}'.format(k, v)) |
| 635 | |
| 636 | |
George Burgess IV | d4ff050 | 2022-08-14 23:27:57 -0700 | [diff] [blame] | 637 | def clean_features_in_place(cargo_toml): |
| 638 | """Removes all side-effects of features in `cargo_toml`.""" |
| 639 | features = cargo_toml.get('features') |
| 640 | if not features: |
| 641 | return |
| 642 | |
| 643 | for name, value in features.items(): |
| 644 | if name != 'default': |
| 645 | features[name] = [] |
| 646 | |
| 647 | |
George Burgess IV | 0313d78 | 2022-08-15 23:45:44 -0700 | [diff] [blame] | 648 | def remove_all_target_dependencies_in_place(cargo_toml): |
George Burgess IV | d4ff050 | 2022-08-14 23:27:57 -0700 | [diff] [blame] | 649 | """Removes all `target.*.dependencies` from `cargo_toml`.""" |
| 650 | target = cargo_toml.get('target') |
| 651 | if not target: |
| 652 | return |
George Burgess IV | 0313d78 | 2022-08-15 23:45:44 -0700 | [diff] [blame] | 653 | |
George Burgess IV | d4ff050 | 2022-08-14 23:27:57 -0700 | [diff] [blame] | 654 | empty_keys = [] |
| 655 | for key, values in target.items(): |
| 656 | values.pop('dependencies', None) |
| 657 | values.pop('dev-dependencies', None) |
| 658 | if not values: |
| 659 | empty_keys.append(key) |
George Burgess IV | 0313d78 | 2022-08-15 23:45:44 -0700 | [diff] [blame] | 660 | |
George Burgess IV | d4ff050 | 2022-08-14 23:27:57 -0700 | [diff] [blame] | 661 | if len(empty_keys) == len(target): |
| 662 | del cargo_toml['target'] |
| 663 | else: |
| 664 | for key in empty_keys: |
| 665 | del target[key] |
George Burgess IV | 0313d78 | 2022-08-15 23:45:44 -0700 | [diff] [blame] | 666 | |
| 667 | |
Abhishek Pandit-Subedi | f0eb6e0 | 2021-09-24 16:36:12 -0700 | [diff] [blame] | 668 | class CrateDestroyer(): |
| 669 | LIB_RS_BODY = """compile_error!("This crate cannot be built for this configuration.");\n""" |
| 670 | |
| 671 | def __init__(self, working_dir, vendor_dir): |
| 672 | self.working_dir = working_dir |
| 673 | self.vendor_dir = vendor_dir |
| 674 | |
| 675 | def _modify_cargo_toml(self, pkg_path): |
George Burgess IV | d4ff050 | 2022-08-14 23:27:57 -0700 | [diff] [blame] | 676 | with open(os.path.join(pkg_path, 'Cargo.toml'), 'r') as cargo: |
Abhishek Pandit-Subedi | f0eb6e0 | 2021-09-24 16:36:12 -0700 | [diff] [blame] | 677 | contents = toml.load(cargo) |
| 678 | |
George Burgess IV | d4ff050 | 2022-08-14 23:27:57 -0700 | [diff] [blame] | 679 | package = contents['package'] |
| 680 | |
Abhishek Pandit-Subedi | f0eb6e0 | 2021-09-24 16:36:12 -0700 | [diff] [blame] | 681 | # Change description, license and delete license key |
George Burgess IV | d4ff050 | 2022-08-14 23:27:57 -0700 | [diff] [blame] | 682 | package['description'] = 'Empty crate that should not build.' |
| 683 | package['license'] = 'Apache-2.0' |
| 684 | |
| 685 | package.pop('license_file', None) |
| 686 | # If there's no build.rs but we specify `links = "foo"`, Cargo gets |
| 687 | # upset. |
| 688 | package.pop('links', None) |
Abhishek Pandit-Subedi | f0eb6e0 | 2021-09-24 16:36:12 -0700 | [diff] [blame] | 689 | |
George Burgess IV | 0313d78 | 2022-08-15 23:45:44 -0700 | [diff] [blame] | 690 | # Some packages have cfg-specific dependencies. Remove them here; we |
| 691 | # don't care about the dependencies of an empty package. |
| 692 | # |
| 693 | # This is a load-bearing optimization: `dev-python/toml` doesn't |
| 694 | # always round-trip dumps(loads(x)) correctly when `x` has keys with |
| 695 | # strings (b/242589711#comment3). The place this has bitten us so far |
| 696 | # is target dependencies, which can be harmlessly removed for now. |
George Burgess IV | d4ff050 | 2022-08-14 23:27:57 -0700 | [diff] [blame] | 697 | # |
| 698 | # Cleaning features in-place is also necessary, since we're removing |
| 699 | # dependencies, and a feature can enable features in dependencies. |
| 700 | # Cargo errors out on `[features] foo = "bar/baz"` if `bar` isn't a |
| 701 | # dependency. |
| 702 | clean_features_in_place(contents) |
George Burgess IV | 0313d78 | 2022-08-15 23:45:44 -0700 | [diff] [blame] | 703 | remove_all_target_dependencies_in_place(contents) |
| 704 | |
Abhishek Pandit-Subedi | f0eb6e0 | 2021-09-24 16:36:12 -0700 | [diff] [blame] | 705 | with open(os.path.join(pkg_path, "Cargo.toml"), "w") as cargo: |
| 706 | toml.dump(contents, cargo) |
| 707 | |
| 708 | def _replace_source_contents(self, package_path): |
| 709 | # First load the checksum file before starting |
| 710 | checksum_file = os.path.join(package_path, ".cargo-checksum.json") |
| 711 | with open(checksum_file, 'r') as csum: |
| 712 | checksum_contents = json.load(csum) |
| 713 | |
| 714 | # Also load the cargo.toml file which we need to write back |
| 715 | cargo_file = os.path.join(package_path, "Cargo.toml") |
George Burgess IV | 3e344e4 | 2022-08-09 21:07:04 -0700 | [diff] [blame] | 716 | with open(cargo_file, 'rb') as cfile: |
| 717 | cargo_contents = cfile.read() |
Abhishek Pandit-Subedi | f0eb6e0 | 2021-09-24 16:36:12 -0700 | [diff] [blame] | 718 | |
| 719 | shutil.rmtree(package_path) |
| 720 | |
| 721 | # Make package and src dirs and replace lib.rs |
| 722 | os.makedirs(os.path.join(package_path, "src"), exist_ok=True) |
| 723 | with open(os.path.join(package_path, "src", "lib.rs"), "w") as librs: |
| 724 | librs.write(self.LIB_RS_BODY) |
| 725 | |
| 726 | # Restore cargo.toml |
George Burgess IV | 3e344e4 | 2022-08-09 21:07:04 -0700 | [diff] [blame] | 727 | with open(cargo_file, 'wb') as cfile: |
| 728 | cfile.write(cargo_contents) |
Abhishek Pandit-Subedi | f0eb6e0 | 2021-09-24 16:36:12 -0700 | [diff] [blame] | 729 | |
| 730 | # Restore checksum |
| 731 | with open(checksum_file, 'w') as csum: |
| 732 | json.dump(checksum_contents, csum) |
| 733 | |
| 734 | def destroy_unused_crates(self): |
George Burgess IV | 40cc91c | 2022-08-15 13:07:40 -0700 | [diff] [blame] | 735 | all_packages = load_all_metadata_packages(self.working_dir, |
| 736 | filter_platform=None, |
| 737 | unique=True) |
| 738 | used_packages = {p["name"] |
| 739 | for p in load_all_metadata_packages(self.working_dir)} |
Abhishek Pandit-Subedi | f0eb6e0 | 2021-09-24 16:36:12 -0700 | [diff] [blame] | 740 | |
| 741 | cleaned_packages = [] |
George Burgess IV | 40cc91c | 2022-08-15 13:07:40 -0700 | [diff] [blame] | 742 | # Since we're asking for _all_ metadata packages, we may see |
| 743 | # duplication. |
| 744 | for package in all_packages: |
Abhishek Pandit-Subedi | f0eb6e0 | 2021-09-24 16:36:12 -0700 | [diff] [blame] | 745 | # Skip used packages |
| 746 | if package["name"] in used_packages: |
| 747 | continue |
| 748 | |
| 749 | # Detect the correct package path to destroy |
| 750 | pkg_path = os.path.join(self.vendor_dir, "{}-{}".format(package["name"], package["version"])) |
| 751 | if not os.path.isdir(pkg_path): |
George Burgess IV | 635f726 | 2022-08-09 21:32:20 -0700 | [diff] [blame] | 752 | print(f'Crate {package["name"]} not found at {pkg_path}') |
| 753 | continue |
Abhishek Pandit-Subedi | f0eb6e0 | 2021-09-24 16:36:12 -0700 | [diff] [blame] | 754 | |
| 755 | self._replace_source_contents(pkg_path) |
| 756 | self._modify_cargo_toml(pkg_path) |
| 757 | _rerun_checksums(pkg_path) |
| 758 | cleaned_packages.append(package["name"]) |
| 759 | |
| 760 | for pkg in cleaned_packages: |
George Burgess IV | 635f726 | 2022-08-09 21:32:20 -0700 | [diff] [blame] | 761 | print("Removed unused crate", pkg) |
Abhishek Pandit-Subedi | f0eb6e0 | 2021-09-24 16:36:12 -0700 | [diff] [blame] | 762 | |
Abhishek Pandit-Subedi | e393cb7 | 2021-08-22 10:41:13 -0700 | [diff] [blame] | 763 | def main(args): |
Abhishek Pandit-Subedi | b75bd56 | 2021-02-25 15:32:22 -0800 | [diff] [blame] | 764 | current_path = pathlib.Path(__file__).parent.absolute() |
Abhishek Pandit-Subedi | 5065a0f | 2021-06-13 20:38:55 +0000 | [diff] [blame] | 765 | patches = os.path.join(current_path, "patches") |
| 766 | vendor = os.path.join(current_path, "vendor") |
Abhishek Pandit-Subedi | ce0f5b2 | 2021-09-10 15:50:08 -0700 | [diff] [blame] | 767 | crab_dir = os.path.join(current_path, "crab", "crates") |
George Burgess IV | 0483370 | 2022-08-09 22:00:38 -0700 | [diff] [blame] | 768 | license_shorthand_file = os.path.join(current_path, "licenses_used.txt") |
Abhishek Pandit-Subedi | b75bd56 | 2021-02-25 15:32:22 -0800 | [diff] [blame] | 769 | |
Abhishek Pandit-Subedi | fa90238 | 2021-08-20 11:04:33 -0700 | [diff] [blame] | 770 | # First, actually run cargo vendor |
| 771 | run_cargo_vendor(current_path) |
| 772 | |
Abhishek Pandit-Subedi | 5065a0f | 2021-06-13 20:38:55 +0000 | [diff] [blame] | 773 | # Order matters here: |
| 774 | # - Apply patches (also re-calculates checksums) |
| 775 | # - Cleanup any owners files (otherwise, git check-in or checksums are |
| 776 | # unhappy) |
Abhishek Pandit-Subedi | f0eb6e0 | 2021-09-24 16:36:12 -0700 | [diff] [blame] | 777 | # - Destroy unused crates |
Abhishek Pandit-Subedi | 5065a0f | 2021-06-13 20:38:55 +0000 | [diff] [blame] | 778 | apply_patches(patches, vendor) |
| 779 | cleanup_owners(vendor) |
Abhishek Pandit-Subedi | f0eb6e0 | 2021-09-24 16:36:12 -0700 | [diff] [blame] | 780 | destroyer = CrateDestroyer(current_path, vendor) |
| 781 | destroyer.destroy_unused_crates() |
Abhishek Pandit-Subedi | b75bd56 | 2021-02-25 15:32:22 -0800 | [diff] [blame] | 782 | |
Abhishek Pandit-Subedi | e393cb7 | 2021-08-22 10:41:13 -0700 | [diff] [blame] | 783 | # Combine license file and check for any bad licenses |
| 784 | lm = LicenseManager(current_path, vendor) |
George Burgess IV | 0483370 | 2022-08-09 22:00:38 -0700 | [diff] [blame] | 785 | lm.generate_license(args.skip_license_check, args.license_map, |
| 786 | license_shorthand_file) |
Abhishek Pandit-Subedi | e393cb7 | 2021-08-22 10:41:13 -0700 | [diff] [blame] | 787 | |
Abhishek Pandit-Subedi | ce0f5b2 | 2021-09-10 15:50:08 -0700 | [diff] [blame] | 788 | # Run crab audit on all packages |
| 789 | crab = CrabManager(current_path, crab_dir) |
| 790 | crab.verify_traits() |
| 791 | |
Abhishek Pandit-Subedi | b75bd56 | 2021-02-25 15:32:22 -0800 | [diff] [blame] | 792 | |
| 793 | if __name__ == '__main__': |
Abhishek Pandit-Subedi | e393cb7 | 2021-08-22 10:41:13 -0700 | [diff] [blame] | 794 | parser = argparse.ArgumentParser(description='Vendor packages properly') |
| 795 | parser.add_argument('--skip-license-check', |
| 796 | '-s', |
| 797 | help='Skip the license check on a specific package', |
| 798 | action='append') |
| 799 | parser.add_argument('--license-map', help='Write license map to this file') |
| 800 | args = parser.parse_args() |
| 801 | |
| 802 | main(args) |