blob: dc7c72adc21d026fd82379631dba69b604972326 [file] [log] [blame]
Abhishek Pandit-Subedib75bd562021-02-25 15:32:22 -08001#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3# Copyright 2021 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6""" This script cleans up the vendor directory.
7"""
Abhishek Pandit-Subedie393cb72021-08-22 10:41:13 -07008import argparse
George Burgess IV635f7262022-08-09 21:32:20 -07009import collections
Abhishek Pandit-Subedi5065a0f2021-06-13 20:38:55 +000010import hashlib
Abhishek Pandit-Subedib75bd562021-02-25 15:32:22 -080011import json
12import os
13import pathlib
Abhishek Pandit-Subedie393cb72021-08-22 10:41:13 -070014import re
Abhishek Pandit-Subedif0eb6e02021-09-24 16:36:12 -070015import shutil
Abhishek Pandit-Subedi5065a0f2021-06-13 20:38:55 +000016import subprocess
George Burgess IV04833702022-08-09 22:00:38 -070017import textwrap
Abhishek Pandit-Subedice0f5b22021-09-10 15:50:08 -070018import toml
Abhishek Pandit-Subedi5065a0f2021-06-13 20:38:55 +000019
Abhishek Pandit-Subedie393cb72021-08-22 10:41:13 -070020# We only care about crates we're actually going to use and that's usually
21# limited to ones with cfg(linux). For running `cargo metadata`, limit results
22# to only this platform
23DEFAULT_PLATFORM_FILTER = "x86_64-unknown-linux-gnu"
24
Abhishek Pandit-Subedi5065a0f2021-06-13 20:38:55 +000025
26def _rerun_checksums(package_path):
27 """Re-run checksums for given package.
28
29 Writes resulting checksums to $package_path/.cargo-checksum.json.
30 """
Abhishek Pandit-Subedie393cb72021-08-22 10:41:13 -070031 hashes = dict()
Abhishek Pandit-Subedi5065a0f2021-06-13 20:38:55 +000032 checksum_path = os.path.join(package_path, '.cargo-checksum.json')
33 if not pathlib.Path(checksum_path).is_file():
34 return False
35
36 with open(checksum_path, 'r') as fread:
37 contents = json.load(fread)
38
39 for root, _, files in os.walk(package_path, topdown=True):
40 for f in files:
41 # Don't checksum an existing checksum file
42 if f == ".cargo-checksum.json":
43 continue
44
45 file_path = os.path.join(root, f)
46 with open(file_path, 'rb') as frb:
47 m = hashlib.sha256()
48 m.update(frb.read())
49 d = m.hexdigest()
50
51 # Key is relative to the package path so strip from beginning
52 key = os.path.relpath(file_path, package_path)
53 hashes[key] = d
54
55 if hashes:
Abhishek Pandit-Subedie393cb72021-08-22 10:41:13 -070056 print("{} regenerated {} hashes".format(package_path,
57 len(hashes.keys())))
Abhishek Pandit-Subedi5065a0f2021-06-13 20:38:55 +000058 contents['files'] = hashes
Abhishek Pandit-Subedi5065a0f2021-06-13 20:38:55 +000059 with open(checksum_path, 'w') as fwrite:
Abhishek Pandit-Subedie393cb72021-08-22 10:41:13 -070060 json.dump(contents, fwrite, sort_keys=True)
Abhishek Pandit-Subedi5065a0f2021-06-13 20:38:55 +000061
62 return True
Abhishek Pandit-Subedib75bd562021-02-25 15:32:22 -080063
64
65def _remove_OWNERS_checksum(root):
66 """ Delete all OWNERS files from the checksum file.
67
Abhishek Pandit-Subedi5065a0f2021-06-13 20:38:55 +000068 Args:
69 root: Root directory for the vendored crate.
Abhishek Pandit-Subedib75bd562021-02-25 15:32:22 -080070
Abhishek Pandit-Subedi5065a0f2021-06-13 20:38:55 +000071 Returns:
72 True if OWNERS was found and cleaned up. Otherwise False.
73 """
Abhishek Pandit-Subedib75bd562021-02-25 15:32:22 -080074 checksum_path = os.path.join(root, '.cargo-checksum.json')
75 if not pathlib.Path(checksum_path).is_file():
76 return False
77
78 with open(checksum_path, 'r') as fread:
79 contents = json.load(fread)
80
81 del_keys = []
82 for cfile in contents['files']:
83 if 'OWNERS' in cfile:
84 del_keys.append(cfile)
85
86 for key in del_keys:
87 del contents['files'][key]
88
89 if del_keys:
90 print('{} deleted: {}'.format(root, del_keys))
91 with open(checksum_path, 'w') as fwrite:
Abhishek Pandit-Subedie393cb72021-08-22 10:41:13 -070092 json.dump(contents, fwrite, sort_keys=True)
Abhishek Pandit-Subedib75bd562021-02-25 15:32:22 -080093
94 return bool(del_keys)
95
96
97def cleanup_owners(vendor_path):
98 """ Remove owners checksums from the vendor directory.
99
Abhishek Pandit-Subedi5065a0f2021-06-13 20:38:55 +0000100 We currently do not check in the OWNERS files from vendored crates because
101 they interfere with the find-owners functionality in gerrit. This cleanup
102 simply finds all instances of "OWNERS" in the checksum files within and
103 removes them.
Abhishek Pandit-Subedib75bd562021-02-25 15:32:22 -0800104
Abhishek Pandit-Subedi5065a0f2021-06-13 20:38:55 +0000105 Args:
106 vendor_path: Absolute path to vendor directory.
107 """
Abhishek Pandit-Subedib75bd562021-02-25 15:32:22 -0800108 deps_cleaned = []
109 for root, dirs, _ in os.walk(vendor_path):
110 for d in dirs:
111 removed = _remove_OWNERS_checksum(os.path.join(root, d))
112 if removed:
113 deps_cleaned.append(d)
114
115 if deps_cleaned:
116 print('Cleanup owners:\n {}'.format("\n".join(deps_cleaned)))
117
118
Abhishek Pandit-Subedi5065a0f2021-06-13 20:38:55 +0000119def apply_single_patch(patch, workdir):
120 """Apply a single patch and return whether it was successful.
121
122 Returns:
123 True if successful. False otherwise.
124 """
Abhishek Pandit-Subedi5065a0f2021-06-13 20:38:55 +0000125 proc = subprocess.run(["patch", "-p1", "-i", patch], cwd=workdir)
126 return proc.returncode == 0
127
128
George Burgess IV30c5c362022-08-19 17:05:02 -0700129def apply_patch_script(script, workdir):
130 """Run the given patch script, returning whether it exited cleanly.
131
132 Returns:
133 True if successful. False otherwise.
134 """
135 return subprocess.run([script], cwd=workdir).returncode == 0
136
137
George Burgess IV635f7262022-08-09 21:32:20 -0700138def determine_vendor_crates(vendor_path):
139 """Returns a map of {crate_name: [directory]} at the given vendor_path."""
140 result = collections.defaultdict(list)
141 for crate_name_plus_ver in os.listdir(vendor_path):
George Burgess IV40cc91c2022-08-15 13:07:40 -0700142 name, _ = crate_name_plus_ver.rsplit('-', 1)
143 result[name].append(crate_name_plus_ver)
George Burgess IV635f7262022-08-09 21:32:20 -0700144
145 for crate_list in result.values():
George Burgess IV40cc91c2022-08-15 13:07:40 -0700146 crate_list.sort()
George Burgess IV635f7262022-08-09 21:32:20 -0700147 return result
148
149
Abhishek Pandit-Subedi5065a0f2021-06-13 20:38:55 +0000150def apply_patches(patches_path, vendor_path):
151 """Finds patches and applies them to sub-folders in the vendored crates.
152
153 Args:
154 patches_path: Path to folder with patches. Expect all patches to be one
155 level down (matching the crate name).
156 vendor_path: Root path to vendored crates directory.
157 """
158 checksums_for = {}
159
160 # Don't bother running if patches directory is empty
161 if not pathlib.Path(patches_path).is_dir():
Abhishek Pandit-Subedie393cb72021-08-22 10:41:13 -0700162 return
Abhishek Pandit-Subedi5065a0f2021-06-13 20:38:55 +0000163
George Burgess IV30c5c362022-08-19 17:05:02 -0700164 patches_failed = False
George Burgess IV635f7262022-08-09 21:32:20 -0700165 vendor_crate_map = determine_vendor_crates(vendor_path)
Abhishek Pandit-Subedi5065a0f2021-06-13 20:38:55 +0000166 # Look for all patches and apply them
167 for d in os.listdir(patches_path):
168 dir_path = os.path.join(patches_path, d)
169
170 # We don't process patches in root dir
171 if not os.path.isdir(dir_path):
172 continue
173
George Burgess IV30c5c362022-08-19 17:05:02 -0700174 # We accept one of two forms here:
175 # - direct targets (these name # `${crate_name}-${version}`)
176 # - simply the crate name (which applies to all versions of the
177 # crate)
178 direct_target = os.path.join(vendor_path, d)
179 if os.path.isdir(direct_target):
180 patch_targets = [d]
181 elif d in vendor_crate_map:
182 patch_targets = vendor_crate_map[d]
183 else:
184 raise RuntimeError(f'Unknown crate in {vendor_path}: {d}')
185
George Burgess IV635f7262022-08-09 21:32:20 -0700186 for patch in os.listdir(dir_path):
Abhishek Pandit-Subedi5065a0f2021-06-13 20:38:55 +0000187 file_path = os.path.join(dir_path, patch)
188
189 # Skip if not a patch file
George Burgess IV30c5c362022-08-19 17:05:02 -0700190 if not os.path.isfile(file_path):
Abhishek Pandit-Subedi5065a0f2021-06-13 20:38:55 +0000191 continue
192
George Burgess IV30c5c362022-08-19 17:05:02 -0700193 if patch.endswith(".patch"):
194 apply = apply_single_patch
195 elif os.access(file_path, os.X_OK):
196 apply = apply_patch_script
George Burgess IV635f7262022-08-09 21:32:20 -0700197 else:
George Burgess IV30c5c362022-08-19 17:05:02 -0700198 # Unrecognized. Skip it.
199 continue
200
201 for target_name in patch_targets:
202 checksums_for[target_name] = True
203 target = os.path.join(vendor_path, target_name)
204 print(f"-- Applying {file_path} to {target}")
205 if not apply(file_path, target):
206 print(f"Failed to apply {file_path} to {target}")
207 patches_failed = True
208
209 # Do this late, so we can report all of the failing patches in one
210 # invocation.
211 if patches_failed:
212 raise ValueError('Patches failed; please see above logs')
Abhishek Pandit-Subedi5065a0f2021-06-13 20:38:55 +0000213
Abhishek Pandit-Subedi5065a0f2021-06-13 20:38:55 +0000214 # Re-run checksums for all modified packages since we applied patches.
215 for key in checksums_for.keys():
216 _rerun_checksums(os.path.join(vendor_path, key))
217
Abhishek Pandit-Subedie393cb72021-08-22 10:41:13 -0700218
George Burgess IV40cc91c2022-08-15 13:07:40 -0700219def fetch_project_cargo_toml_files(working_dir):
220 """Returns all Cargo.toml files under working_dir."""
221 projects = working_dir / 'projects'
222 return sorted(projects.glob('**/Cargo.toml'))
223
224
Abhishek Pandit-Subedifa902382021-08-20 11:04:33 -0700225def run_cargo_vendor(working_dir):
226 """Runs cargo vendor.
227
228 Args:
229 working_dir: Directory to run inside. This should be the directory where
Abhishek Pandit-Subedice0f5b22021-09-10 15:50:08 -0700230 Cargo.toml is kept.
Abhishek Pandit-Subedifa902382021-08-20 11:04:33 -0700231 """
George Burgess IV635f7262022-08-09 21:32:20 -0700232 # Cargo will refuse to revendor into versioned directories, which leads to
233 # repeated `./vendor.py` invocations trying to apply patches to
234 # already-patched sources. Remove the existing vendor directory to avoid
235 # this.
236 vendor_dir = working_dir / 'vendor'
237 if vendor_dir.exists():
George Burgess IV40cc91c2022-08-15 13:07:40 -0700238 shutil.rmtree(vendor_dir)
239
240 cargo_cmdline = ['cargo', 'vendor', '--versioned-dirs', '-v']
241 for i, cargo_toml in enumerate(fetch_project_cargo_toml_files(working_dir)):
242 # `cargo vendor` requires a 'root' manifest; select an arbitrary one,
243 # then tack other manifests on to it. Order doesn't really matter.
244 if i == 0:
245 cargo_cmdline.append('--manifest-path')
246 else:
247 cargo_cmdline.append('-s')
248 cargo_cmdline.append(str(cargo_toml))
249
250 # Autocreate src/lib.rs if necessary.
251 lib_rs = cargo_toml.parent / 'src' / 'lib.rs'
252 if not lib_rs.exists():
253 lib_rs.parent.mkdir(exist_ok=True)
254 lib_rs.write_bytes(b'')
255
256 # Always place vendor/ at the top-level directory.
257 cargo_cmdline += ('--', 'vendor')
258 subprocess.check_call(cargo_cmdline, cwd=working_dir)
Abhishek Pandit-Subedi5065a0f2021-06-13 20:38:55 +0000259
Abhishek Pandit-Subedice0f5b22021-09-10 15:50:08 -0700260
Abhishek Pandit-Subedie393cb72021-08-22 10:41:13 -0700261def load_metadata(working_dir, filter_platform=DEFAULT_PLATFORM_FILTER):
George Burgess IV40cc91c2022-08-15 13:07:40 -0700262 """Load metadata for all projects under a given directory.
Abhishek Pandit-Subedie393cb72021-08-22 10:41:13 -0700263
264 Args:
George Burgess IV40cc91c2022-08-15 13:07:40 -0700265 working_dir: Base directory to run from.
Abhishek Pandit-Subedie393cb72021-08-22 10:41:13 -0700266 filter_platform: Filter packages to ones configured for this platform.
267 """
George Burgess IV40cc91c2022-08-15 13:07:40 -0700268 metadata_objects = []
269 for manifest_path in fetch_project_cargo_toml_files(working_dir):
270 cmd = [
271 'cargo', 'metadata', '--format-version', '1', '--manifest-path',
272 manifest_path
273 ]
274 # Conditionally add platform filter
275 if filter_platform:
276 cmd += ("--filter-platform", filter_platform)
277 output = subprocess.check_output(cmd, cwd=working_dir)
278 metadata_objects.append(json.loads(output))
279 return metadata_objects
Abhishek Pandit-Subedif0eb6e02021-09-24 16:36:12 -0700280
Abhishek Pandit-Subedif0eb6e02021-09-24 16:36:12 -0700281
George Burgess IV40cc91c2022-08-15 13:07:40 -0700282def load_all_metadata_packages(working_dir,
283 filter_platform=DEFAULT_PLATFORM_FILTER,
284 unique=False):
285 """Returns a list of all packages returned by load_metadata."""
286 results = []
287 for metadata in load_metadata(working_dir, filter_platform):
288 results += metadata['packages']
Abhishek Pandit-Subedie393cb72021-08-22 10:41:13 -0700289
George Burgess IV40cc91c2022-08-15 13:07:40 -0700290 if not unique:
291 return results
Abhishek Pandit-Subedie393cb72021-08-22 10:41:13 -0700292
George Burgess IV40cc91c2022-08-15 13:07:40 -0700293 new_results = []
294 seen_keys = set()
295 for item in results:
296 key = item['id']
297 if key in seen_keys:
298 continue
299 seen_keys.add(key)
300 new_results.append(item)
301 return new_results
Abhishek Pandit-Subedie393cb72021-08-22 10:41:13 -0700302
303class LicenseManager:
304 """ Manage consolidating licenses for all packages."""
305
306 # These are all the licenses we support. Keys are what is seen in metadata and
307 # values are what is expected by the ebuild.
308 SUPPORTED_LICENSES = {
309 'Apache-2.0': 'Apache-2.0',
310 'MIT': 'MIT',
311 'BSD-3-Clause': 'BSD-3',
George Burgess IV4ae42062022-08-15 18:54:51 -0700312 'ISC': 'ISC',
313 'unicode': 'unicode',
Abhishek Pandit-Subedie393cb72021-08-22 10:41:13 -0700314 }
315
316 # Prefer to take attribution licenses in this order. All these require that
317 # we actually use the license file found in the package so they MUST have
318 # a license file set.
319 PREFERRED_ATTRIB_LICENSE_ORDER = ['MIT', 'BSD-3', 'ISC']
320
321 # If Apache license is found, always prefer it (simplifies attribution)
322 APACHE_LICENSE = 'Apache-2.0'
323
324 # Regex for license files found in the vendored directories. Search for
325 # these files with re.IGNORECASE.
326 #
327 # These will be searched in order with the earlier entries being preferred.
328 LICENSE_NAMES_REGEX = [
329 r'^license-mit$',
330 r'^copyright$',
331 r'^licen[cs]e.*$',
332 ]
333
334 # Some crates have their license file in other crates. This usually occurs
335 # because multiple crates are published from the same git repository and the
336 # license isn't updated in each sub-crate. In these cases, we can just
337 # ignore these packages.
338 MAP_LICENSE_TO_OTHER = {
339 'failure_derive': 'failure',
340 'grpcio-compiler': 'grpcio',
341 'grpcio-sys': 'grpcio',
342 'rustyline-derive': 'rustyline',
343 }
344
345 # Map a package to a specific license and license file. Only use this if
346 # a package doesn't have an easily discoverable license or exports its
347 # license in a weird way. Prefer to patch the project with a license and
348 # upstream the patch instead.
349 STATIC_LICENSE_MAP = {
350 # "package name": ( "license name", "license file relative location")
351 }
352
353 def __init__(self, working_dir, vendor_dir):
354 self.working_dir = working_dir
355 self.vendor_dir = vendor_dir
356
357 def _find_license_in_dir(self, search_dir):
358 for p in os.listdir(search_dir):
359 # Ignore anything that's not a file
360 if not os.path.isfile(os.path.join(search_dir, p)):
361 continue
362
363 # Now check if the name matches any of the regexes
364 # We'll return the first matching file.
365 for regex in self.LICENSE_NAMES_REGEX:
366 if re.search(regex, p, re.IGNORECASE):
367 yield os.path.join(search_dir, p)
368 break
369
370 def _guess_license_type(self, license_file):
371 if '-MIT' in license_file:
372 return 'MIT'
373 elif '-APACHE' in license_file:
374 return 'APACHE'
375 elif '-BSD' in license_file:
376 return 'BSD-3'
377
378 with open(license_file, 'r') as f:
379 lines = f.read()
380 if 'MIT' in lines:
381 return 'MIT'
382 elif 'Apache' in lines:
383 return 'APACHE'
384 elif 'BSD 3-Clause' in lines:
385 return 'BSD-3'
386
387 return ''
388
George Burgess IV04833702022-08-09 22:00:38 -0700389 def generate_license(self, skip_license_check, print_map_to_file,
390 license_shorthand_file):
Abhishek Pandit-Subedie393cb72021-08-22 10:41:13 -0700391 """Generate single massive license file from metadata."""
George Burgess IV40cc91c2022-08-15 13:07:40 -0700392 all_packages = load_all_metadata_packages(self.working_dir, unique=True)
Abhishek Pandit-Subedie393cb72021-08-22 10:41:13 -0700393
394 has_license_types = set()
395 bad_licenses = {}
396
397 # Keep license map ordered so it generates a consistent license map
398 license_map = {}
399
400 skip_license_check = skip_license_check or []
George Burgess IV4ae42062022-08-15 18:54:51 -0700401 has_unicode_license = False
Abhishek Pandit-Subedie393cb72021-08-22 10:41:13 -0700402
George Burgess IV40cc91c2022-08-15 13:07:40 -0700403 for package in all_packages:
404 # Skip the synthesized Cargo.toml packages that exist solely to
405 # list dependencies.
406 if 'path+file:///' in package['id']:
Abhishek Pandit-Subedie393cb72021-08-22 10:41:13 -0700407 continue
408
George Burgess IV40cc91c2022-08-15 13:07:40 -0700409 pkg_name = package['name']
Abhishek Pandit-Subedie393cb72021-08-22 10:41:13 -0700410 if pkg_name in skip_license_check:
411 print(
412 "Skipped license check on {}. Reason: Skipped from command line"
413 .format(pkg_name))
414 continue
415
416 if pkg_name in self.MAP_LICENSE_TO_OTHER:
417 print(
418 'Skipped license check on {}. Reason: License already in {}'
419 .format(pkg_name, self.MAP_LICENSE_TO_OTHER[pkg_name]))
420 continue
421
422 # Check if we have a static license map for this package. Use the
423 # static values if we have it already set.
424 if pkg_name in self.STATIC_LICENSE_MAP:
425 (license, license_file) = self.STATIC_LICENSE_MAP[pkg_name]
426 license_map[pkg_name] = {
427 "license": license,
428 "license_file": license_file,
429 }
430 continue
431
432 license_files = []
George Burgess IV93ba4732022-08-13 14:10:10 -0700433 # use `or ''` instead of get's default, since `package` may have a
434 # None value for 'license'.
435 license = package.get('license') or ''
Abhishek Pandit-Subedie393cb72021-08-22 10:41:13 -0700436
437 # We ignore the metadata for license file because most crates don't
438 # have it set. Just scan the source for licenses.
George Burgess IV635f7262022-08-09 21:32:20 -0700439 pkg_version = package['version']
George Burgess IV40cc91c2022-08-15 13:07:40 -0700440 license_files = list(self._find_license_in_dir(
441 os.path.join(self.vendor_dir, f'{pkg_name}-{pkg_version}')))
Abhishek Pandit-Subedie393cb72021-08-22 10:41:13 -0700442
George Burgess IV4ae42062022-08-15 18:54:51 -0700443 # FIXME(b/240953811): The code later in this loop is only
444 # structured to handle ORs, not ANDs. Fortunately, this license in
445 # particular is `AND`ed between a super common license (Apache) and
446 # a more obscure one (unicode). This hack is specifically intended
447 # for the `unicode-ident` crate, though no crate name check is
448 # made, since it's OK other crates happen to have this license.
449 if license == '(MIT OR Apache-2.0) AND Unicode-DFS-2016':
450 has_unicode_license = True
451 # We'll check later to be sure MIT or Apache-2.0 is represented
452 # properly.
453 for x in license_files:
454 if os.path.basename(x) == 'LICENSE-UNICODE':
455 license_file = x
456 break
457 else:
458 raise ValueError('No LICENSE-UNICODE found in '
459 f'{license_files}')
460 license_map[pkg_name] = {
461 "license": license,
462 "license_file": license_file,
463 }
464 has_license_types.add('unicode')
465 continue
466
Abhishek Pandit-Subedie393cb72021-08-22 10:41:13 -0700467 # If there are multiple licenses, they are delimited with "OR" or "/"
468 delim = ' OR ' if ' OR ' in license else '/'
George Burgess IV40cc91c2022-08-15 13:07:40 -0700469 found = [x.strip() for x in license.split(delim)]
Abhishek Pandit-Subedie393cb72021-08-22 10:41:13 -0700470
471 # Filter licenses to ones we support
472 licenses_or = [
473 self.SUPPORTED_LICENSES[f] for f in found
474 if f in self.SUPPORTED_LICENSES
475 ]
476
477 # If apache license is found, always prefer it because it simplifies
478 # license attribution (we can use existing Apache notice)
479 if self.APACHE_LICENSE in licenses_or:
480 has_license_types.add(self.APACHE_LICENSE)
481 license_map[pkg_name] = {'license': self.APACHE_LICENSE}
482
483 # Handle single license that has at least one license file
484 # We pick the first license file and the license
485 elif len(licenses_or) == 1:
486 if license_files:
487 l = licenses_or[0]
488 lf = license_files[0]
489
490 has_license_types.add(l)
491 license_map[pkg_name] = {
492 'license': l,
493 'license_file': os.path.relpath(lf, self.working_dir),
494 }
495 else:
496 bad_licenses[pkg_name] = "{} missing license file".format(
497 licenses_or[0])
498 # Handle multiple licenses
499 elif len(licenses_or) > 1:
500 # Check preferred licenses in order
501 license_found = False
502 for l in self.PREFERRED_ATTRIB_LICENSE_ORDER:
503 if not l in licenses_or:
504 continue
505
506 for f in license_files:
507 if self._guess_license_type(f) == l:
508 license_found = True
509 has_license_types.add(l)
510 license_map[pkg_name] = {
511 'license':
512 l,
513 'license_file':
514 os.path.relpath(f, self.working_dir),
515 }
516 break
517
518 # Break out of loop if license is found
519 if license_found:
520 break
521 else:
522 bad_licenses[pkg_name] = license
523
524 # If we had any bad licenses, we need to abort
525 if bad_licenses:
526 for k in bad_licenses.keys():
527 print("{} had no acceptable licenses: {}".format(
528 k, bad_licenses[k]))
529 raise Exception("Bad licenses in vendored packages.")
530
531 # Write license map to file
532 if print_map_to_file:
533 with open(os.path.join(self.working_dir, print_map_to_file),
534 'w') as lfile:
535 json.dump(license_map, lfile, sort_keys=True)
536
537 # Raise missing licenses unless we have a valid reason to ignore them
538 raise_missing_license = False
539 for name, v in license_map.items():
540 if 'license_file' not in v and v.get('license',
541 '') != self.APACHE_LICENSE:
542 raise_missing_license = True
543 print(" {}: Missing license file. Fix or add to ignorelist.".
544 format(name))
545
546 if raise_missing_license:
547 raise Exception(
548 "Unhandled missing license file. "
549 "Make sure all are accounted for before continuing.")
550
George Burgess IV4ae42062022-08-15 18:54:51 -0700551 if has_unicode_license:
552 if self.APACHE_LICENSE not in has_license_types:
553 raise ValueError('Need the apache license; currently have: '
554 f'{sorted(has_license_types)}')
555
George Burgess IV04833702022-08-09 22:00:38 -0700556 sorted_licenses = sorted(has_license_types)
557 print("Add the following licenses to the ebuild:\n",
558 sorted_licenses)
559 header = textwrap.dedent("""\
560 # File to describe the licenses used by this registry.
561 # Used to it's easy to automatically verify ebuilds are updated.
562 # Each line is a license. Lines starting with # are comments.
563 """)
564 with open(license_shorthand_file, 'w', encoding='utf-8') as f:
565 f.write(header)
566 f.write('\n'.join(sorted_licenses))
Abhishek Pandit-Subedie393cb72021-08-22 10:41:13 -0700567
568
Abhishek Pandit-Subedice0f5b22021-09-10 15:50:08 -0700569# TODO(abps) - This needs to be replaced with datalog later. We should compile
570# all crab files into datalog and query it with our requirements
571# instead.
572class CrabManager:
573 """Manage audit files."""
574 def __init__(self, working_dir, crab_dir):
575 self.working_dir = working_dir
576 self.crab_dir = crab_dir
577
578 def _check_bad_traits(self, crabdata):
579 """Checks that a package's crab audit meets our requirements.
580
581 Args:
582 crabdata: Dict with crab keys in standard templated format.
583 """
584 common = crabdata['common']
585 # TODO(b/200578411) - Figure out what conditions we should enforce as
586 # part of the audit.
587 conditions = [
588 common.get('deny', None),
589 ]
590
591 # If any conditions are true, this crate is not acceptable.
592 return any(conditions)
593
594 def verify_traits(self):
595 """ Verify that all required CRAB traits for this repository are met.
596 """
George Burgess IV40cc91c2022-08-15 13:07:40 -0700597 all_packages = load_all_metadata_packages(self.working_dir, unique=True)
Abhishek Pandit-Subedice0f5b22021-09-10 15:50:08 -0700598
599 failing_crates = {}
600
601 # Verify all packages have a CRAB file associated with it and they meet
602 # all our required traits
George Burgess IV40cc91c2022-08-15 13:07:40 -0700603 for package in all_packages:
604 # Skip the synthesized Cargo.toml packages that exist solely to
605 # list dependencies.
606 if 'path+file:///' in package['id']:
Abhishek Pandit-Subedice0f5b22021-09-10 15:50:08 -0700607 continue
608
609 crabname = "{}-{}".format(package['name'], package['version'])
610 filename = os.path.join(self.crab_dir, "{}.toml".format(crabname))
611
612 # If crab file doesn't exist, the crate fails
613 if not os.path.isfile(filename):
614 failing_crates[crabname] = "No crab file".format(filename)
615 continue
616
617 with open(filename, 'r') as f:
618 crabdata = toml.loads(f.read())
619
620 # If crab file's crate_name and version keys don't match this
621 # package, it also fails. This is just housekeeping...
622 if package['name'] != crabdata['crate_name'] or package[
623 'version'] != crabdata['version']:
624 failing_crates[crabname] = "Crate name or version don't match"
625 continue
626
627 if self._check_bad_traits(crabdata):
628 failing_crates[crabname] = "Failed bad traits check"
629
630 # If we had any failing crates, list them now
631 if failing_crates:
632 print('Failed CRAB audit:')
633 for k, v in failing_crates.items():
634 print(' {}: {}'.format(k, v))
635
636
George Burgess IVd4ff0502022-08-14 23:27:57 -0700637def clean_features_in_place(cargo_toml):
638 """Removes all side-effects of features in `cargo_toml`."""
639 features = cargo_toml.get('features')
640 if not features:
641 return
642
643 for name, value in features.items():
644 if name != 'default':
645 features[name] = []
646
647
George Burgess IV0313d782022-08-15 23:45:44 -0700648def remove_all_target_dependencies_in_place(cargo_toml):
George Burgess IVd4ff0502022-08-14 23:27:57 -0700649 """Removes all `target.*.dependencies` from `cargo_toml`."""
650 target = cargo_toml.get('target')
651 if not target:
652 return
George Burgess IV0313d782022-08-15 23:45:44 -0700653
George Burgess IVd4ff0502022-08-14 23:27:57 -0700654 empty_keys = []
655 for key, values in target.items():
656 values.pop('dependencies', None)
657 values.pop('dev-dependencies', None)
658 if not values:
659 empty_keys.append(key)
George Burgess IV0313d782022-08-15 23:45:44 -0700660
George Burgess IVd4ff0502022-08-14 23:27:57 -0700661 if len(empty_keys) == len(target):
662 del cargo_toml['target']
663 else:
664 for key in empty_keys:
665 del target[key]
George Burgess IV0313d782022-08-15 23:45:44 -0700666
667
Abhishek Pandit-Subedif0eb6e02021-09-24 16:36:12 -0700668class CrateDestroyer():
669 LIB_RS_BODY = """compile_error!("This crate cannot be built for this configuration.");\n"""
670
671 def __init__(self, working_dir, vendor_dir):
672 self.working_dir = working_dir
673 self.vendor_dir = vendor_dir
674
675 def _modify_cargo_toml(self, pkg_path):
George Burgess IVd4ff0502022-08-14 23:27:57 -0700676 with open(os.path.join(pkg_path, 'Cargo.toml'), 'r') as cargo:
Abhishek Pandit-Subedif0eb6e02021-09-24 16:36:12 -0700677 contents = toml.load(cargo)
678
George Burgess IVd4ff0502022-08-14 23:27:57 -0700679 package = contents['package']
680
Abhishek Pandit-Subedif0eb6e02021-09-24 16:36:12 -0700681 # Change description, license and delete license key
George Burgess IVd4ff0502022-08-14 23:27:57 -0700682 package['description'] = 'Empty crate that should not build.'
683 package['license'] = 'Apache-2.0'
684
685 package.pop('license_file', None)
686 # If there's no build.rs but we specify `links = "foo"`, Cargo gets
687 # upset.
688 package.pop('links', None)
Abhishek Pandit-Subedif0eb6e02021-09-24 16:36:12 -0700689
George Burgess IV0313d782022-08-15 23:45:44 -0700690 # Some packages have cfg-specific dependencies. Remove them here; we
691 # don't care about the dependencies of an empty package.
692 #
693 # This is a load-bearing optimization: `dev-python/toml` doesn't
694 # always round-trip dumps(loads(x)) correctly when `x` has keys with
695 # strings (b/242589711#comment3). The place this has bitten us so far
696 # is target dependencies, which can be harmlessly removed for now.
George Burgess IVd4ff0502022-08-14 23:27:57 -0700697 #
698 # Cleaning features in-place is also necessary, since we're removing
699 # dependencies, and a feature can enable features in dependencies.
700 # Cargo errors out on `[features] foo = "bar/baz"` if `bar` isn't a
701 # dependency.
702 clean_features_in_place(contents)
George Burgess IV0313d782022-08-15 23:45:44 -0700703 remove_all_target_dependencies_in_place(contents)
704
Abhishek Pandit-Subedif0eb6e02021-09-24 16:36:12 -0700705 with open(os.path.join(pkg_path, "Cargo.toml"), "w") as cargo:
706 toml.dump(contents, cargo)
707
708 def _replace_source_contents(self, package_path):
709 # First load the checksum file before starting
710 checksum_file = os.path.join(package_path, ".cargo-checksum.json")
711 with open(checksum_file, 'r') as csum:
712 checksum_contents = json.load(csum)
713
714 # Also load the cargo.toml file which we need to write back
715 cargo_file = os.path.join(package_path, "Cargo.toml")
George Burgess IV3e344e42022-08-09 21:07:04 -0700716 with open(cargo_file, 'rb') as cfile:
717 cargo_contents = cfile.read()
Abhishek Pandit-Subedif0eb6e02021-09-24 16:36:12 -0700718
719 shutil.rmtree(package_path)
720
721 # Make package and src dirs and replace lib.rs
722 os.makedirs(os.path.join(package_path, "src"), exist_ok=True)
723 with open(os.path.join(package_path, "src", "lib.rs"), "w") as librs:
724 librs.write(self.LIB_RS_BODY)
725
726 # Restore cargo.toml
George Burgess IV3e344e42022-08-09 21:07:04 -0700727 with open(cargo_file, 'wb') as cfile:
728 cfile.write(cargo_contents)
Abhishek Pandit-Subedif0eb6e02021-09-24 16:36:12 -0700729
730 # Restore checksum
731 with open(checksum_file, 'w') as csum:
732 json.dump(checksum_contents, csum)
733
734 def destroy_unused_crates(self):
George Burgess IV40cc91c2022-08-15 13:07:40 -0700735 all_packages = load_all_metadata_packages(self.working_dir,
736 filter_platform=None,
737 unique=True)
738 used_packages = {p["name"]
739 for p in load_all_metadata_packages(self.working_dir)}
Abhishek Pandit-Subedif0eb6e02021-09-24 16:36:12 -0700740
741 cleaned_packages = []
George Burgess IV40cc91c2022-08-15 13:07:40 -0700742 # Since we're asking for _all_ metadata packages, we may see
743 # duplication.
744 for package in all_packages:
Abhishek Pandit-Subedif0eb6e02021-09-24 16:36:12 -0700745 # Skip used packages
746 if package["name"] in used_packages:
747 continue
748
749 # Detect the correct package path to destroy
750 pkg_path = os.path.join(self.vendor_dir, "{}-{}".format(package["name"], package["version"]))
751 if not os.path.isdir(pkg_path):
George Burgess IV635f7262022-08-09 21:32:20 -0700752 print(f'Crate {package["name"]} not found at {pkg_path}')
753 continue
Abhishek Pandit-Subedif0eb6e02021-09-24 16:36:12 -0700754
755 self._replace_source_contents(pkg_path)
756 self._modify_cargo_toml(pkg_path)
757 _rerun_checksums(pkg_path)
758 cleaned_packages.append(package["name"])
759
760 for pkg in cleaned_packages:
George Burgess IV635f7262022-08-09 21:32:20 -0700761 print("Removed unused crate", pkg)
Abhishek Pandit-Subedif0eb6e02021-09-24 16:36:12 -0700762
Abhishek Pandit-Subedie393cb72021-08-22 10:41:13 -0700763def main(args):
Abhishek Pandit-Subedib75bd562021-02-25 15:32:22 -0800764 current_path = pathlib.Path(__file__).parent.absolute()
Abhishek Pandit-Subedi5065a0f2021-06-13 20:38:55 +0000765 patches = os.path.join(current_path, "patches")
766 vendor = os.path.join(current_path, "vendor")
Abhishek Pandit-Subedice0f5b22021-09-10 15:50:08 -0700767 crab_dir = os.path.join(current_path, "crab", "crates")
George Burgess IV04833702022-08-09 22:00:38 -0700768 license_shorthand_file = os.path.join(current_path, "licenses_used.txt")
Abhishek Pandit-Subedib75bd562021-02-25 15:32:22 -0800769
Abhishek Pandit-Subedifa902382021-08-20 11:04:33 -0700770 # First, actually run cargo vendor
771 run_cargo_vendor(current_path)
772
Abhishek Pandit-Subedi5065a0f2021-06-13 20:38:55 +0000773 # Order matters here:
774 # - Apply patches (also re-calculates checksums)
775 # - Cleanup any owners files (otherwise, git check-in or checksums are
776 # unhappy)
Abhishek Pandit-Subedif0eb6e02021-09-24 16:36:12 -0700777 # - Destroy unused crates
Abhishek Pandit-Subedi5065a0f2021-06-13 20:38:55 +0000778 apply_patches(patches, vendor)
779 cleanup_owners(vendor)
Abhishek Pandit-Subedif0eb6e02021-09-24 16:36:12 -0700780 destroyer = CrateDestroyer(current_path, vendor)
781 destroyer.destroy_unused_crates()
Abhishek Pandit-Subedib75bd562021-02-25 15:32:22 -0800782
Abhishek Pandit-Subedie393cb72021-08-22 10:41:13 -0700783 # Combine license file and check for any bad licenses
784 lm = LicenseManager(current_path, vendor)
George Burgess IV04833702022-08-09 22:00:38 -0700785 lm.generate_license(args.skip_license_check, args.license_map,
786 license_shorthand_file)
Abhishek Pandit-Subedie393cb72021-08-22 10:41:13 -0700787
Abhishek Pandit-Subedice0f5b22021-09-10 15:50:08 -0700788 # Run crab audit on all packages
789 crab = CrabManager(current_path, crab_dir)
790 crab.verify_traits()
791
Abhishek Pandit-Subedib75bd562021-02-25 15:32:22 -0800792
793if __name__ == '__main__':
Abhishek Pandit-Subedie393cb72021-08-22 10:41:13 -0700794 parser = argparse.ArgumentParser(description='Vendor packages properly')
795 parser.add_argument('--skip-license-check',
796 '-s',
797 help='Skip the license check on a specific package',
798 action='append')
799 parser.add_argument('--license-map', help='Write license map to this file')
800 args = parser.parse_args()
801
802 main(args)