blob: 6f31cc30ab06d6366a520f88dd0202808f7ac83d [file] [log] [blame]
Mike Frysingere58c0e22017-10-04 15:43:30 -04001# -*- coding: utf-8 -*-
Jim Hebert91c052c2011-03-11 11:00:53 -08002# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
Mike Frysinger012371e2019-01-03 03:42:32 -05006"""Command to extract the dependancy tree for a given package.
7
8This produces JSON output for other tools to process.
9"""
Don Garrett25f309a2014-03-19 14:02:12 -070010
Mike Frysinger383367e2014-09-16 15:06:17 -040011from __future__ import print_function
Chris McDonaldd2fa6162019-07-30 15:30:58 -060012from __future__ import absolute_import
Mike Frysinger383367e2014-09-16 15:06:17 -040013
Mike Frysingera942aee2020-03-20 03:53:37 -040014import sys
Don Garrettf8bf7842014-03-20 17:03:42 -070015
Chris McDonalda4fb7fe2019-08-19 15:16:57 -060016from chromite.lib.depgraph import DepGraphGenerator
Don Garrettf8bf7842014-03-20 17:03:42 -070017
Jim Hebertcf870d72013-06-12 15:33:34 -070018from chromite.lib import commandline
19from chromite.lib import cros_build_lib
Ralph Nathan446aee92015-03-23 14:44:56 -070020from chromite.lib import cros_logging as logging
Mike Frysingerd0960812020-06-09 01:53:32 -040021from chromite.lib import pformat
Alex Klein5cdd57f2020-11-23 11:53:28 -070022from chromite.lib import sysroot_lib
Alex Klein18a60af2020-06-11 12:08:47 -060023from chromite.lib.parser import package_info
Mike Frysingercc838832014-05-24 13:10:30 -040024
Chris McDonaldd8a7f112019-11-01 10:35:07 -060025
Mike Frysingera942aee2020-03-20 03:53:37 -040026assert sys.version_info >= (3, 6), 'This module requires Python 3.6+'
27
28
Yu-Ju Hong7f01e9a2014-10-23 11:01:57 -070029def FlattenDepTree(deptree, pkgtable=None, parentcpv=None, get_cpe=False):
Don Garrett25f309a2014-03-19 14:02:12 -070030 """Simplify dependency json.
31
32Turn something like this (the parallel_emerge DepsTree format):
Jim Hebert91c052c2011-03-11 11:00:53 -080033{
34 "app-admin/eselect-1.2.9": {
35 "action": "merge",
36 "deps": {
37 "sys-apps/coreutils-7.5-r1": {
38 "action": "merge",
39 "deps": {},
40 "deptype": "runtime"
41 },
42 ...
43 }
44 }
45}
46 ...into something like this (the cros_extract_deps format):
47{
48 "app-admin/eselect-1.2.9": {
49 "deps": ["coreutils-7.5-r1"],
50 "rev_deps": [],
51 "name": "eselect",
52 "category": "app-admin",
53 "version": "1.2.9",
54 "full_name": "app-admin/eselect-1.2.9",
55 "action": "merge"
56 },
57 "sys-apps/coreutils-7.5-r1": {
58 "deps": [],
59 "rev_deps": ["app-admin/eselect-1.2.9"],
60 "name": "coreutils",
61 "category": "sys-apps",
62 "version": "7.5-r1",
63 "full_name": "sys-apps/coreutils-7.5-r1",
64 "action": "merge"
65 }
66}
Yu-Ju Hong7f01e9a2014-10-23 11:01:57 -070067
68 Args:
69 deptree: The dependency tree.
70 pkgtable: The package table to update. If None, create a new one.
71 parentcpv: The parent CPV.
72 get_cpe: If set True, include CPE in the flattened dependency tree.
73
74 Returns:
75 A flattened dependency tree.
Jim Hebert91c052c2011-03-11 11:00:53 -080076 """
David James1b363582012-12-17 11:53:11 -080077 if pkgtable is None:
78 pkgtable = {}
Mike Frysinger0bdbc102019-06-13 15:27:29 -040079 for cpv, record in deptree.items():
Jim Hebert91c052c2011-03-11 11:00:53 -080080 if cpv not in pkgtable:
Alex Klein18a60af2020-06-11 12:08:47 -060081 split = package_info.SplitCPV(cpv)
Yu-Ju Hong7f01e9a2014-10-23 11:01:57 -070082 pkgtable[cpv] = {'deps': [],
83 'rev_deps': [],
Xuewei Zhang656f9932017-09-15 16:15:05 -070084 'name': split.package,
85 'category': split.category,
86 'version': '%s' % split.version,
Yu-Ju Hong7f01e9a2014-10-23 11:01:57 -070087 'full_name': cpv,
88 'cpes': [],
89 'action': record['action']}
90 if get_cpe:
Xuewei Zhang656f9932017-09-15 16:15:05 -070091 pkgtable[cpv]['cpes'].extend(GetCPEFromCPV(
92 split.category, split.package, split.version_no_rev))
Yu-Ju Hong7f01e9a2014-10-23 11:01:57 -070093
Jim Hebert91c052c2011-03-11 11:00:53 -080094 # If we have a parent, that is a rev_dep for the current package.
95 if parentcpv:
Yu-Ju Hong7f01e9a2014-10-23 11:01:57 -070096 pkgtable[cpv]['rev_deps'].append(parentcpv)
Jim Hebert91c052c2011-03-11 11:00:53 -080097 # If current package has any deps, record those.
Yu-Ju Hong7f01e9a2014-10-23 11:01:57 -070098 for childcpv in record['deps']:
99 pkgtable[cpv]['deps'].append(childcpv)
Jim Hebert91c052c2011-03-11 11:00:53 -0800100 # Visit the subtree recursively as well.
Yu-Ju Hong7f01e9a2014-10-23 11:01:57 -0700101 FlattenDepTree(record['deps'], pkgtable=pkgtable, parentcpv=cpv,
102 get_cpe=get_cpe)
Ned Nguyen3bbd2072019-01-28 19:33:41 -0700103 # Sort 'deps' & 'rev_deps' alphabetically to make them more readable.
104 pkgtable[cpv]['deps'].sort()
105 pkgtable[cpv]['rev_deps'].sort()
Jim Hebert91c052c2011-03-11 11:00:53 -0800106 return pkgtable
107
108
Jim Hebertcf870d72013-06-12 15:33:34 -0700109def GetCPEFromCPV(category, package, version):
110 """Look up the CPE for a specified Portage package.
Jim Hebert91c052c2011-03-11 11:00:53 -0800111
Jim Hebertcf870d72013-06-12 15:33:34 -0700112 Args:
113 category: The Portage package's category, e.g. "net-misc"
114 package: The Portage package's name, e.g. "curl"
115 version: The Portage version, e.g. "7.30.0"
116
Mike Frysinger02e1e072013-11-10 22:11:34 -0500117 Returns:
118 A list of CPE Name strings, e.g.
119 ["cpe:/a:curl:curl:7.30.0", "cpe:/a:curl:libcurl:7.30.0"]
Jim Hebertcf870d72013-06-12 15:33:34 -0700120 """
Yu-Ju Hong7f01e9a2014-10-23 11:01:57 -0700121 equery_cmd = ['equery', 'm', '-U', '%s/%s' % (category, package)]
Mike Frysingerf5a3b2d2019-12-12 14:36:17 -0500122 lines = cros_build_lib.run(equery_cmd, check=False, print_cmd=False,
Mike Frysinger44b83812019-12-10 00:09:30 -0500123 stdout=True, encoding='utf-8').stdout.splitlines()
Jim Hebertcf870d72013-06-12 15:33:34 -0700124 # Look for lines like "Remote-ID: cpe:/a:kernel:linux-pam ID: cpe"
125 # and extract the cpe URI.
126 cpes = []
127 for line in lines:
Yu-Ju Hong7f01e9a2014-10-23 11:01:57 -0700128 if 'ID: cpe' not in line:
Jim Hebertcf870d72013-06-12 15:33:34 -0700129 continue
Yu-Ju Hong7f01e9a2014-10-23 11:01:57 -0700130 cpes.append('%s:%s' % (line.split()[1], version.replace('_', '')))
Jim Hebertcf870d72013-06-12 15:33:34 -0700131 # Note that we're assuming we can combine the root of the CPE, taken
132 # from metadata.xml, and tack on the version number as used by
133 # Portage, and come up with a legitimate CPE. This works so long as
134 # Portage and CPE agree on the precise formatting of the version
Jim Hebert96aff9c2013-07-16 15:43:17 -0700135 # number, which they almost always do. The major exception we've
136 # identified thus far is that our ebuilds have a pattern of inserting
137 # underscores prior to patchlevels, that neither upstream nor CPE
138 # use. For example, our code will decide we have
Jim Hebertcf870d72013-06-12 15:33:34 -0700139 # cpe:/a:todd_miller:sudo:1.8.6_p7 yet the advisories use a format
140 # like cpe:/a:todd_miller:sudo:1.8.6p7, without the underscore. (CPE
141 # is "right" in this example, in that it matches www.sudo.ws.)
142 #
Jim Hebert96aff9c2013-07-16 15:43:17 -0700143 # Removing underscores seems to improve our chances of correctly
144 # arriving at the CPE used by NVD. However, at the end of the day,
145 # ebuild version numbers are rev'd by people who don't have "try to
146 # match NVD" as one of their goals, and there is always going to be
147 # some risk of minor formatting disagreements at the version number
148 # level, if not from stray underscores then from something else.
149 #
Jim Hebertcf870d72013-06-12 15:33:34 -0700150 # This is livable so long as you do some fuzzy version number
151 # comparison in your vulnerability monitoring, between what-we-have
152 # and what-the-advisory-says-is-affected.
153 return cpes
154
155
Chris McDonalde69db662018-11-15 12:50:18 -0700156def GenerateCPEList(deps_list, sysroot):
Xuewei Zhang656f9932017-09-15 16:15:05 -0700157 """Generate all CPEs for the packages included in deps_list and SDK packages
158
159 Args:
160 deps_list: A flattened dependency tree (cros_extract_deps format).
Chris McDonalde69db662018-11-15 12:50:18 -0700161 sysroot: The board directory to use when finding SDK packages.
Xuewei Zhang656f9932017-09-15 16:15:05 -0700162
163 Returns:
164 A list of CPE info for packages in deps_list and SDK packages, e.g.
165 [
166 {
167 "ComponentName": "app-admin/sudo",
168 "Repository": "cros",
169 "Targets": [
170 "cpe:/a:todd_miller:sudo:1.8.19p2"
171 ]
172 },
173 {
174 "ComponentName": "sys-libs/glibc",
175 "Repository": "cros",
176 "Targets": [
177 "cpe:/a:gnu:glibc:2.23"
178 ]
179 }
180 ]
181 """
Jim Hebertcf870d72013-06-12 15:33:34 -0700182 cpe_dump = []
Xuewei Zhang656f9932017-09-15 16:15:05 -0700183
Alex Klein5cdd57f2020-11-23 11:53:28 -0700184 # Generate CPEs for SDK packages.
185 for pkg_info in sorted(
186 sysroot_lib.get_sdk_provided_packages(sysroot), key=lambda x: x.cpvr):
Xuewei Zhang656f9932017-09-15 16:15:05 -0700187 # Only add CPE for SDK CPVs missing in deps_list.
Alex Klein5cdd57f2020-11-23 11:53:28 -0700188 if deps_list.get(pkg_info.cpvr) is not None:
Xuewei Zhang656f9932017-09-15 16:15:05 -0700189 continue
190
Alex Klein5cdd57f2020-11-23 11:53:28 -0700191 cpes = GetCPEFromCPV(pkg_info.category, pkg_info.package, pkg_info.version)
Xuewei Zhang656f9932017-09-15 16:15:05 -0700192 if cpes:
Alex Klein5cdd57f2020-11-23 11:53:28 -0700193 cpe_dump.append({'ComponentName': '%s' % pkg_info.atom,
Xuewei Zhang656f9932017-09-15 16:15:05 -0700194 'Repository': 'cros',
195 'Targets': sorted(cpes)})
196 else:
Alex Klein5cdd57f2020-11-23 11:53:28 -0700197 logging.warning('No CPE entry for %s', pkg_info.cpvr)
Xuewei Zhang656f9932017-09-15 16:15:05 -0700198
199 # Generage CPEs for packages in deps_list.
Mike Frysinger1457e482019-01-03 04:04:02 -0500200 for cpv, record in sorted(deps_list.items()):
Yu-Ju Hong7f01e9a2014-10-23 11:01:57 -0700201 if record['cpes']:
202 name = '%s/%s' % (record['category'], record['name'])
203 cpe_dump.append({'ComponentName': name,
204 'Repository': 'cros',
205 'Targets': sorted(record['cpes'])})
Jim Hebertcf870d72013-06-12 15:33:34 -0700206 else:
Ralph Nathan446aee92015-03-23 14:44:56 -0700207 logging.warning('No CPE entry for %s', cpv)
Yu-Ju Hong7f01e9a2014-10-23 11:01:57 -0700208 return sorted(cpe_dump, key=lambda k: k['ComponentName'])
Jim Hebert91c052c2011-03-11 11:00:53 -0800209
210
Chris McDonalde69db662018-11-15 12:50:18 -0700211def ParseArgs(argv):
212 """Parse command line arguments."""
Mike Frysinger012371e2019-01-03 03:42:32 -0500213 parser = commandline.ArgumentParser(description=__doc__)
Chris McDonalde69db662018-11-15 12:50:18 -0700214 target = parser.add_mutually_exclusive_group()
215 target.add_argument('--sysroot', type='path', help='Path to the sysroot.')
216 target.add_argument('--board', help='Board name.')
217
Yu-Ju Hong7f01e9a2014-10-23 11:01:57 -0700218 parser.add_argument('--format', default='deps',
219 choices=['deps', 'cpe'],
Yu-Ju Hongf48d3982014-10-30 16:12:16 -0700220 help='Output either traditional deps or CPE-only JSON.')
221 parser.add_argument('--output-path', default=None,
222 help='Write output to the given path.')
Mike Frysinger012371e2019-01-03 03:42:32 -0500223 parser.add_argument('pkgs', nargs='*')
224 opts = parser.parse_args(argv)
225 opts.Freeze()
226 return opts
Jim Hebert91c052c2011-03-11 11:00:53 -0800227
Chris McDonalde69db662018-11-15 12:50:18 -0700228
Ned Nguyene16dcfb2019-03-22 10:36:05 -0600229def FilterObsoleteDeps(package_deps):
230 """Remove all the packages that are to be uninstalled from |package_deps|.
231
232 Returns:
233 None since this method mutates |package_deps| directly.
234 """
235 obsolete_package_deps = []
Mike Frysinger0bdbc102019-06-13 15:27:29 -0400236 for k, v in package_deps.items():
Ned Nguyene16dcfb2019-03-22 10:36:05 -0600237 if v['action'] in ('merge', 'nomerge'):
238 continue
239 elif v['action'] == 'uninstall':
240 obsolete_package_deps.append(k)
241 else:
242 assert False, (
243 'Unrecognized action. Package dep data: %s' % v)
244 for p in obsolete_package_deps:
245 del package_deps[p]
246
247
Chris McDonaldd8a7f112019-11-01 10:35:07 -0600248def ExtractDeps(sysroot,
249 package_list,
250 formatting='deps',
Chris McDonald76608422020-06-15 11:42:20 -0600251 include_bdepend=True,
252 backtrack=True):
Ned Nguyendd3e09f2019-03-14 18:54:03 -0600253 """Returns the set of dependencies for the packages in package_list.
254
255 For calculating dependencies graph, this should only consider packages
Chris McDonaldebc7ae72019-10-03 14:58:46 -0600256 that are DEPENDS, RDEPENDS, or BDEPENDS. Essentially, this should answer the
257 question "which are all the packages which changing them may change the
258 execution of any binaries produced by packages in |package_list|."
Ned Nguyendd3e09f2019-03-14 18:54:03 -0600259
260 Args:
261 sysroot: the path (string) to the root directory into which the package is
262 pretend to be merged. This value is also used for setting
263 PORTAGE_CONFIGROOT.
264 package_list: the list of packages (CP string) to extract their dependencies
265 from.
266 formatting: can either be 'deps' or 'cpe'. For 'deps', see the return
267 format in docstring of FlattenDepTree, for 'cpe', see the return format in
268 docstring of GenerateCPEList.
Chris McDonaldd8a7f112019-11-01 10:35:07 -0600269 include_bdepend: Controls whether BDEPEND packages that would be installed
270 to BROOT (usually "/" instead of ROOT) are included in the output.
Chris McDonald76608422020-06-15 11:42:20 -0600271 backtrack: Setting to False disables backtracking in Portage's dependency
272 solver. If the highest available version of dependencies doesn't produce
273 a solvable graph Portage will give up and return an error instead of
274 trying other candidates.
Ned Nguyendd3e09f2019-03-14 18:54:03 -0600275
276 Returns:
277 A JSON-izable object that either follows 'deps' or 'cpe' format.
278 """
Chris McDonaldd8a7f112019-11-01 10:35:07 -0600279 lib_argv = ['--quiet', '--pretend', '--emptytree']
280 if include_bdepend:
281 lib_argv += ['--include-bdepend']
Chris McDonald76608422020-06-15 11:42:20 -0600282 if not backtrack:
283 lib_argv += ['--backtrack=0']
Chris McDonalde69db662018-11-15 12:50:18 -0700284 lib_argv += ['--sysroot=%s' % sysroot]
285 lib_argv.extend(package_list)
Jim Hebert91c052c2011-03-11 11:00:53 -0800286
287 deps = DepGraphGenerator()
Jim Hebertcf870d72013-06-12 15:33:34 -0700288 deps.Initialize(lib_argv)
Chris McDonaldd8a7f112019-11-01 10:35:07 -0600289
290 deps_tree, _deps_info, bdeps_tree = deps.GenDependencyTree()
291 trees = (deps_tree, bdeps_tree)
292
293 flattened_trees = tuple(
294 FlattenDepTree(tree, get_cpe=(formatting == 'cpe')) for tree in trees)
Ned Nguyene16dcfb2019-03-22 10:36:05 -0600295
296 # Workaround: since emerge doesn't honor the --emptytree flag, for now we need
297 # to manually filter out packages that are obsolete (meant to be
298 # uninstalled by emerge)
299 # TODO(crbug.com/938605): remove this work around once
300 # https://bugs.gentoo.org/681308 is addressed.
Chris McDonaldd8a7f112019-11-01 10:35:07 -0600301 for tree in flattened_trees:
302 FilterObsoleteDeps(tree)
Ned Nguyene16dcfb2019-03-22 10:36:05 -0600303
Chris McDonalde69db662018-11-15 12:50:18 -0700304 if formatting == 'cpe':
Chris McDonaldd8a7f112019-11-01 10:35:07 -0600305 flattened_trees = tuple(
306 GenerateCPEList(tree, sysroot) for tree in flattened_trees)
307 return flattened_trees
Chris McDonalde69db662018-11-15 12:50:18 -0700308
309
310def main(argv):
Mike Frysinger012371e2019-01-03 03:42:32 -0500311 opts = ParseArgs(argv)
Chris McDonalde69db662018-11-15 12:50:18 -0700312
Jack Rosenthalb984e102021-04-07 21:18:29 +0000313 sysroot = opts.sysroot or cros_build_lib.GetSysroot(opts.board)
Chris McDonald486508d2019-11-04 09:20:07 -0700314 deps_list, _ = ExtractDeps(sysroot, opts.pkgs, opts.format)
Yu-Ju Hongf48d3982014-10-30 16:12:16 -0700315
Mike Frysingerd0960812020-06-09 01:53:32 -0400316 pformat.json(deps_list,
317 fp=opts.output_path if opts.output_path else sys.stdout)