Jim Hebert | 91c052c | 2011-03-11 11:00:53 -0800 | [diff] [blame] | 1 | # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. |
| 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | |
Don Garrett | 25f309a | 2014-03-19 14:02:12 -0700 | [diff] [blame] | 5 | """Command to extract the dependancy tree for a given package.""" |
| 6 | |
Mike Frysinger | 383367e | 2014-09-16 15:06:17 -0400 | [diff] [blame] | 7 | from __future__ import print_function |
| 8 | |
Jim Hebert | 91c052c | 2011-03-11 11:00:53 -0800 | [diff] [blame] | 9 | import json |
Don Garrett | f8bf784 | 2014-03-20 17:03:42 -0700 | [diff] [blame] | 10 | import portage # pylint: disable=F0401 |
| 11 | |
Jim Hebert | 91c052c | 2011-03-11 11:00:53 -0800 | [diff] [blame] | 12 | from parallel_emerge import DepGraphGenerator |
Don Garrett | f8bf784 | 2014-03-20 17:03:42 -0700 | [diff] [blame] | 13 | |
Jim Hebert | cf870d7 | 2013-06-12 15:33:34 -0700 | [diff] [blame] | 14 | from chromite.lib import commandline |
| 15 | from chromite.lib import cros_build_lib |
Jim Hebert | 91c052c | 2011-03-11 11:00:53 -0800 | [diff] [blame] | 16 | |
Mike Frysinger | cc83883 | 2014-05-24 13:10:30 -0400 | [diff] [blame] | 17 | |
Yu-Ju Hong | 7f01e9a | 2014-10-23 11:01:57 -0700 | [diff] [blame] | 18 | def FlattenDepTree(deptree, pkgtable=None, parentcpv=None, get_cpe=False): |
Don Garrett | 25f309a | 2014-03-19 14:02:12 -0700 | [diff] [blame] | 19 | """Simplify dependency json. |
| 20 | |
| 21 | Turn something like this (the parallel_emerge DepsTree format): |
Jim Hebert | 91c052c | 2011-03-11 11:00:53 -0800 | [diff] [blame] | 22 | { |
| 23 | "app-admin/eselect-1.2.9": { |
| 24 | "action": "merge", |
| 25 | "deps": { |
| 26 | "sys-apps/coreutils-7.5-r1": { |
| 27 | "action": "merge", |
| 28 | "deps": {}, |
| 29 | "deptype": "runtime" |
| 30 | }, |
| 31 | ... |
| 32 | } |
| 33 | } |
| 34 | } |
| 35 | ...into something like this (the cros_extract_deps format): |
| 36 | { |
| 37 | "app-admin/eselect-1.2.9": { |
| 38 | "deps": ["coreutils-7.5-r1"], |
| 39 | "rev_deps": [], |
| 40 | "name": "eselect", |
| 41 | "category": "app-admin", |
| 42 | "version": "1.2.9", |
| 43 | "full_name": "app-admin/eselect-1.2.9", |
| 44 | "action": "merge" |
| 45 | }, |
| 46 | "sys-apps/coreutils-7.5-r1": { |
| 47 | "deps": [], |
| 48 | "rev_deps": ["app-admin/eselect-1.2.9"], |
| 49 | "name": "coreutils", |
| 50 | "category": "sys-apps", |
| 51 | "version": "7.5-r1", |
| 52 | "full_name": "sys-apps/coreutils-7.5-r1", |
| 53 | "action": "merge" |
| 54 | } |
| 55 | } |
Yu-Ju Hong | 7f01e9a | 2014-10-23 11:01:57 -0700 | [diff] [blame] | 56 | |
| 57 | Args: |
| 58 | deptree: The dependency tree. |
| 59 | pkgtable: The package table to update. If None, create a new one. |
| 60 | parentcpv: The parent CPV. |
| 61 | get_cpe: If set True, include CPE in the flattened dependency tree. |
| 62 | |
| 63 | Returns: |
| 64 | A flattened dependency tree. |
Jim Hebert | 91c052c | 2011-03-11 11:00:53 -0800 | [diff] [blame] | 65 | """ |
David James | 1b36358 | 2012-12-17 11:53:11 -0800 | [diff] [blame] | 66 | if pkgtable is None: |
| 67 | pkgtable = {} |
Yu-Ju Hong | 7f01e9a | 2014-10-23 11:01:57 -0700 | [diff] [blame] | 68 | for cpv, record in deptree.iteritems(): |
Jim Hebert | 91c052c | 2011-03-11 11:00:53 -0800 | [diff] [blame] | 69 | if cpv not in pkgtable: |
David James | 1b36358 | 2012-12-17 11:53:11 -0800 | [diff] [blame] | 70 | cat, nam, ver, rev = portage.versions.catpkgsplit(cpv) |
Yu-Ju Hong | 7f01e9a | 2014-10-23 11:01:57 -0700 | [diff] [blame] | 71 | pkgtable[cpv] = {'deps': [], |
| 72 | 'rev_deps': [], |
| 73 | 'name': nam, |
| 74 | 'category': cat, |
| 75 | 'version': '%s-%s' % (ver, rev), |
| 76 | 'full_name': cpv, |
| 77 | 'cpes': [], |
| 78 | 'action': record['action']} |
| 79 | if get_cpe: |
| 80 | pkgtable[cpv]['cpes'].extend(GetCPEFromCPV(cat, nam, ver)) |
| 81 | |
Jim Hebert | 91c052c | 2011-03-11 11:00:53 -0800 | [diff] [blame] | 82 | # If we have a parent, that is a rev_dep for the current package. |
| 83 | if parentcpv: |
Yu-Ju Hong | 7f01e9a | 2014-10-23 11:01:57 -0700 | [diff] [blame] | 84 | pkgtable[cpv]['rev_deps'].append(parentcpv) |
Jim Hebert | 91c052c | 2011-03-11 11:00:53 -0800 | [diff] [blame] | 85 | # If current package has any deps, record those. |
Yu-Ju Hong | 7f01e9a | 2014-10-23 11:01:57 -0700 | [diff] [blame] | 86 | for childcpv in record['deps']: |
| 87 | pkgtable[cpv]['deps'].append(childcpv) |
Jim Hebert | 91c052c | 2011-03-11 11:00:53 -0800 | [diff] [blame] | 88 | # Visit the subtree recursively as well. |
Yu-Ju Hong | 7f01e9a | 2014-10-23 11:01:57 -0700 | [diff] [blame] | 89 | FlattenDepTree(record['deps'], pkgtable=pkgtable, parentcpv=cpv, |
| 90 | get_cpe=get_cpe) |
Jim Hebert | 91c052c | 2011-03-11 11:00:53 -0800 | [diff] [blame] | 91 | return pkgtable |
| 92 | |
| 93 | |
Jim Hebert | cf870d7 | 2013-06-12 15:33:34 -0700 | [diff] [blame] | 94 | def GetCPEFromCPV(category, package, version): |
| 95 | """Look up the CPE for a specified Portage package. |
Jim Hebert | 91c052c | 2011-03-11 11:00:53 -0800 | [diff] [blame] | 96 | |
Jim Hebert | cf870d7 | 2013-06-12 15:33:34 -0700 | [diff] [blame] | 97 | Args: |
| 98 | category: The Portage package's category, e.g. "net-misc" |
| 99 | package: The Portage package's name, e.g. "curl" |
| 100 | version: The Portage version, e.g. "7.30.0" |
| 101 | |
Mike Frysinger | 02e1e07 | 2013-11-10 22:11:34 -0500 | [diff] [blame] | 102 | Returns: |
| 103 | A list of CPE Name strings, e.g. |
| 104 | ["cpe:/a:curl:curl:7.30.0", "cpe:/a:curl:libcurl:7.30.0"] |
Jim Hebert | cf870d7 | 2013-06-12 15:33:34 -0700 | [diff] [blame] | 105 | """ |
Yu-Ju Hong | 7f01e9a | 2014-10-23 11:01:57 -0700 | [diff] [blame] | 106 | equery_cmd = ['equery', 'm', '-U', '%s/%s' % (category, package)] |
Jim Hebert | cf870d7 | 2013-06-12 15:33:34 -0700 | [diff] [blame] | 107 | lines = cros_build_lib.RunCommand(equery_cmd, error_code_ok=True, |
| 108 | print_cmd=False, |
| 109 | redirect_stdout=True).output.splitlines() |
| 110 | # Look for lines like "Remote-ID: cpe:/a:kernel:linux-pam ID: cpe" |
| 111 | # and extract the cpe URI. |
| 112 | cpes = [] |
| 113 | for line in lines: |
Yu-Ju Hong | 7f01e9a | 2014-10-23 11:01:57 -0700 | [diff] [blame] | 114 | if 'ID: cpe' not in line: |
Jim Hebert | cf870d7 | 2013-06-12 15:33:34 -0700 | [diff] [blame] | 115 | continue |
Yu-Ju Hong | 7f01e9a | 2014-10-23 11:01:57 -0700 | [diff] [blame] | 116 | cpes.append('%s:%s' % (line.split()[1], version.replace('_', ''))) |
Jim Hebert | cf870d7 | 2013-06-12 15:33:34 -0700 | [diff] [blame] | 117 | # Note that we're assuming we can combine the root of the CPE, taken |
| 118 | # from metadata.xml, and tack on the version number as used by |
| 119 | # Portage, and come up with a legitimate CPE. This works so long as |
| 120 | # Portage and CPE agree on the precise formatting of the version |
Jim Hebert | 96aff9c | 2013-07-16 15:43:17 -0700 | [diff] [blame] | 121 | # number, which they almost always do. The major exception we've |
| 122 | # identified thus far is that our ebuilds have a pattern of inserting |
| 123 | # underscores prior to patchlevels, that neither upstream nor CPE |
| 124 | # use. For example, our code will decide we have |
Jim Hebert | cf870d7 | 2013-06-12 15:33:34 -0700 | [diff] [blame] | 125 | # cpe:/a:todd_miller:sudo:1.8.6_p7 yet the advisories use a format |
| 126 | # like cpe:/a:todd_miller:sudo:1.8.6p7, without the underscore. (CPE |
| 127 | # is "right" in this example, in that it matches www.sudo.ws.) |
| 128 | # |
Jim Hebert | 96aff9c | 2013-07-16 15:43:17 -0700 | [diff] [blame] | 129 | # Removing underscores seems to improve our chances of correctly |
| 130 | # arriving at the CPE used by NVD. However, at the end of the day, |
| 131 | # ebuild version numbers are rev'd by people who don't have "try to |
| 132 | # match NVD" as one of their goals, and there is always going to be |
| 133 | # some risk of minor formatting disagreements at the version number |
| 134 | # level, if not from stray underscores then from something else. |
| 135 | # |
Jim Hebert | cf870d7 | 2013-06-12 15:33:34 -0700 | [diff] [blame] | 136 | # This is livable so long as you do some fuzzy version number |
| 137 | # comparison in your vulnerability monitoring, between what-we-have |
| 138 | # and what-the-advisory-says-is-affected. |
| 139 | return cpes |
| 140 | |
| 141 | |
| 142 | def ExtractCPEList(deps_list): |
| 143 | cpe_dump = [] |
Yu-Ju Hong | 7f01e9a | 2014-10-23 11:01:57 -0700 | [diff] [blame] | 144 | for cpv, record in deps_list.iteritems(): |
| 145 | if record['cpes']: |
| 146 | name = '%s/%s' % (record['category'], record['name']) |
| 147 | cpe_dump.append({'ComponentName': name, |
| 148 | 'Repository': 'cros', |
| 149 | 'Targets': sorted(record['cpes'])}) |
Jim Hebert | cf870d7 | 2013-06-12 15:33:34 -0700 | [diff] [blame] | 150 | else: |
Yu-Ju Hong | 7f01e9a | 2014-10-23 11:01:57 -0700 | [diff] [blame] | 151 | cros_build_lib.Warning('No CPE entry for %s', cpv) |
| 152 | return sorted(cpe_dump, key=lambda k: k['ComponentName']) |
Jim Hebert | 91c052c | 2011-03-11 11:00:53 -0800 | [diff] [blame] | 153 | |
| 154 | |
Brian Harring | 3067505 | 2012-02-29 12:18:22 -0800 | [diff] [blame] | 155 | def main(argv): |
Jim Hebert | cf870d7 | 2013-06-12 15:33:34 -0700 | [diff] [blame] | 156 | parser = commandline.ArgumentParser(description=""" |
| 157 | This extracts the dependency tree for the specified package, and outputs it |
| 158 | to stdout, in a serialized JSON format.""") |
Yu-Ju Hong | 997108e | 2014-10-27 11:21:47 -0700 | [diff] [blame] | 159 | parser.add_argument('--board', default=None, |
Yu-Ju Hong | 7f01e9a | 2014-10-23 11:01:57 -0700 | [diff] [blame] | 160 | help='The board to use when computing deps.') |
| 161 | parser.add_argument('--format', default='deps', |
| 162 | choices=['deps', 'cpe'], |
Yu-Ju Hong | f48d398 | 2014-10-30 16:12:16 -0700 | [diff] [blame] | 163 | help='Output either traditional deps or CPE-only JSON.') |
| 164 | parser.add_argument('--output-path', default=None, |
| 165 | help='Write output to the given path.') |
Jim Hebert | cf870d7 | 2013-06-12 15:33:34 -0700 | [diff] [blame] | 166 | known_args, unknown_args = parser.parse_known_args(argv) |
Jim Hebert | 91c052c | 2011-03-11 11:00:53 -0800 | [diff] [blame] | 167 | |
Yu-Ju Hong | 997108e | 2014-10-27 11:21:47 -0700 | [diff] [blame] | 168 | lib_argv = [] |
| 169 | if known_args.board: |
| 170 | lib_argv += ['--board=%s' % known_args.board] |
David James | f147b43 | 2014-12-02 13:51:55 -0800 | [diff] [blame] | 171 | lib_argv += ['--quiet', '--pretend', '--emptytree'] |
Jim Hebert | cf870d7 | 2013-06-12 15:33:34 -0700 | [diff] [blame] | 172 | lib_argv.extend(unknown_args) |
Jim Hebert | 91c052c | 2011-03-11 11:00:53 -0800 | [diff] [blame] | 173 | |
| 174 | deps = DepGraphGenerator() |
Jim Hebert | cf870d7 | 2013-06-12 15:33:34 -0700 | [diff] [blame] | 175 | deps.Initialize(lib_argv) |
David James | 1b36358 | 2012-12-17 11:53:11 -0800 | [diff] [blame] | 176 | deps_tree, _deps_info = deps.GenDependencyTree() |
Yu-Ju Hong | 7f01e9a | 2014-10-23 11:01:57 -0700 | [diff] [blame] | 177 | deps_list = FlattenDepTree(deps_tree, get_cpe=(known_args.format == 'cpe')) |
| 178 | if known_args.format == 'cpe': |
Jim Hebert | cf870d7 | 2013-06-12 15:33:34 -0700 | [diff] [blame] | 179 | deps_list = ExtractCPEList(deps_list) |
Yu-Ju Hong | f48d398 | 2014-10-30 16:12:16 -0700 | [diff] [blame] | 180 | |
| 181 | deps_output = json.dumps(deps_list, sort_keys=True, indent=2) |
| 182 | if known_args.output_path: |
| 183 | with open(known_args.output_path, 'w') as f: |
| 184 | f.write(deps_output) |
| 185 | else: |
| 186 | print(deps_output) |