blob: 78c84853a3bcaa6881f2615e55a4b03a7bf2619c [file] [log] [blame]
David Pursell9476bf42015-03-30 13:34:27 -07001# Copyright 2015 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
Alex Kleinaaddc932020-01-30 15:02:24 -07005"""Deploy packages onto a target device.
6
7Integration tests for this file can be found at cli/cros/tests/cros_vm_tests.py.
8See that file for more information.
9"""
David Pursell9476bf42015-03-30 13:34:27 -070010
Mike Frysinger93e8ffa2019-07-03 20:24:18 -040011from __future__ import division
David Pursell9476bf42015-03-30 13:34:27 -070012
Xiaochu Liu2726e7c2019-07-18 10:28:10 -070013import bz2
David Pursell9476bf42015-03-30 13:34:27 -070014import fnmatch
Ralph Nathane01ccf12015-04-16 10:40:32 -070015import functools
David Pursell9476bf42015-03-30 13:34:27 -070016import json
Chris McDonald14ac61d2021-07-21 11:49:56 -060017import logging
David Pursell9476bf42015-03-30 13:34:27 -070018import os
Jae Hoon Kim2376e142022-09-03 00:18:58 +000019from pathlib import Path
Xiaochu Liu2726e7c2019-07-18 10:28:10 -070020import tempfile
David Pursell9476bf42015-03-30 13:34:27 -070021
Ralph Nathane01ccf12015-04-16 10:40:32 -070022from chromite.cli import command
Mike Frysinger06a51c82021-04-06 11:39:17 -040023from chromite.lib import build_target_lib
Ram Chandrasekar56152ec2021-11-22 17:10:41 +000024from chromite.lib import constants
David Pursell9476bf42015-03-30 13:34:27 -070025from chromite.lib import cros_build_lib
Alex Klein18a60af2020-06-11 12:08:47 -060026from chromite.lib import dlc_lib
Ralph Nathane01ccf12015-04-16 10:40:32 -070027from chromite.lib import operation
Xiaochu Liu2726e7c2019-07-18 10:28:10 -070028from chromite.lib import osutils
David Pursell9476bf42015-03-30 13:34:27 -070029from chromite.lib import portage_util
David Pursell9476bf42015-03-30 13:34:27 -070030from chromite.lib import remote_access
Kimiyuki Onakaa4ec7f62020-08-25 13:58:48 +090031from chromite.lib import workon_helper
Alex Klein18a60af2020-06-11 12:08:47 -060032from chromite.lib.parser import package_info
33
Chris McDonald14ac61d2021-07-21 11:49:56 -060034
David Pursell9476bf42015-03-30 13:34:27 -070035try:
36 import portage
37except ImportError:
38 if cros_build_lib.IsInsideChroot():
39 raise
40
41
42_DEVICE_BASE_DIR = '/usr/local/tmp/cros-deploy'
43# This is defined in src/platform/dev/builder.py
44_STRIPPED_PACKAGES_DIR = 'stripped-packages'
45
46_MAX_UPDATES_NUM = 10
47_MAX_UPDATES_WARNING = (
48 'You are about to update a large number of installed packages, which '
49 'might take a long time, fail midway, or leave the target in an '
50 'inconsistent state. It is highly recommended that you flash a new image '
51 'instead.')
52
Xiaochu Liu2726e7c2019-07-18 10:28:10 -070053_DLC_ID = 'DLC_ID'
54_DLC_PACKAGE = 'DLC_PACKAGE'
Andrew67b5fa72020-02-05 14:14:48 -080055_DLC_ENABLED = 'DLC_ENABLED'
Xiaochu Liu2726e7c2019-07-18 10:28:10 -070056_ENVIRONMENT_FILENAME = 'environment.bz2'
57_DLC_INSTALL_ROOT = '/var/cache/dlc'
58
David Pursell9476bf42015-03-30 13:34:27 -070059
60class DeployError(Exception):
61 """Thrown when an unrecoverable error is encountered during deploy."""
62
63
Ralph Nathane01ccf12015-04-16 10:40:32 -070064class BrilloDeployOperation(operation.ProgressBarOperation):
65 """ProgressBarOperation specific for brillo deploy."""
Alex Kleinaaddc932020-01-30 15:02:24 -070066 # These two variables are used to validate the output in the VM integration
67 # tests. Changes to the output must be reflected here.
Mike Frysinger63d35512021-01-26 23:16:13 -050068 MERGE_EVENTS = (
69 'Preparing local packages',
70 'NOTICE: Copying binpkgs',
71 'NOTICE: Installing',
72 'been installed.',
73 'Please restart any updated',
74 )
Mike Frysinger22bb5502021-01-29 13:05:46 -050075 UNMERGE_EVENTS = (
76 'NOTICE: Unmerging',
77 'been uninstalled.',
78 'Please restart any updated',
79 )
Ralph Nathane01ccf12015-04-16 10:40:32 -070080
Mike Frysinger63d35512021-01-26 23:16:13 -050081 def __init__(self, emerge):
Ralph Nathane01ccf12015-04-16 10:40:32 -070082 """Construct BrilloDeployOperation object.
83
84 Args:
Ralph Nathane01ccf12015-04-16 10:40:32 -070085 emerge: True if emerge, False is unmerge.
86 """
Jae Hoon Kimad176b82021-07-26 19:29:29 +000087 super().__init__()
Ralph Nathane01ccf12015-04-16 10:40:32 -070088 if emerge:
Ralph Nathan90475a12015-05-20 13:19:01 -070089 self._events = self.MERGE_EVENTS
Ralph Nathane01ccf12015-04-16 10:40:32 -070090 else:
Ralph Nathan90475a12015-05-20 13:19:01 -070091 self._events = self.UNMERGE_EVENTS
Mike Frysinger63d35512021-01-26 23:16:13 -050092 self._total = len(self._events)
Ralph Nathane01ccf12015-04-16 10:40:32 -070093 self._completed = 0
94
Ralph Nathandc14ed92015-04-22 11:17:40 -070095 def ParseOutput(self, output=None):
Ralph Nathane01ccf12015-04-16 10:40:32 -070096 """Parse the output of brillo deploy to update a progress bar."""
97 stdout = self._stdout.read()
98 stderr = self._stderr.read()
99 output = stdout + stderr
100 for event in self._events:
101 self._completed += output.count(event)
Mike Frysinger93e8ffa2019-07-03 20:24:18 -0400102 self.ProgressBar(self._completed / self._total)
Ralph Nathane01ccf12015-04-16 10:40:32 -0700103
104
David Pursell9476bf42015-03-30 13:34:27 -0700105class _InstallPackageScanner(object):
106 """Finds packages that need to be installed on a target device.
107
108 Scans the sysroot bintree, beginning with a user-provided list of packages,
109 to find all packages that need to be installed. If so instructed,
110 transitively scans forward (mandatory) and backward (optional) dependencies
111 as well. A package will be installed if missing on the target (mandatory
112 packages only), or it will be updated if its sysroot version and build time
113 are different from the target. Common usage:
114
115 pkg_scanner = _InstallPackageScanner(sysroot)
116 pkgs = pkg_scanner.Run(...)
117 """
118
119 class VartreeError(Exception):
120 """An error in the processing of the installed packages tree."""
121
122 class BintreeError(Exception):
123 """An error in the processing of the source binpkgs tree."""
124
125 class PkgInfo(object):
126 """A record containing package information."""
127
128 __slots__ = ('cpv', 'build_time', 'rdeps_raw', 'rdeps', 'rev_rdeps')
129
130 def __init__(self, cpv, build_time, rdeps_raw, rdeps=None, rev_rdeps=None):
131 self.cpv = cpv
132 self.build_time = build_time
133 self.rdeps_raw = rdeps_raw
134 self.rdeps = set() if rdeps is None else rdeps
135 self.rev_rdeps = set() if rev_rdeps is None else rev_rdeps
136
137 # Python snippet for dumping vartree info on the target. Instantiate using
138 # _GetVartreeSnippet().
139 _GET_VARTREE = """
David Pursell9476bf42015-03-30 13:34:27 -0700140import json
Gwendal Grignou99e6f532018-10-25 12:16:28 -0700141import os
142import portage
143
144# Normalize the path to match what portage will index.
145target_root = os.path.normpath('%(root)s')
146if not target_root.endswith('/'):
147 target_root += '/'
148trees = portage.create_trees(target_root=target_root, config_root='/')
149vartree = trees[target_root]['vartree']
David Pursell9476bf42015-03-30 13:34:27 -0700150pkg_info = []
151for cpv in vartree.dbapi.cpv_all():
152 slot, rdep_raw, build_time = vartree.dbapi.aux_get(
153 cpv, ('SLOT', 'RDEPEND', 'BUILD_TIME'))
154 pkg_info.append((cpv, slot, rdep_raw, build_time))
155
156print(json.dumps(pkg_info))
157"""
158
159 def __init__(self, sysroot):
160 self.sysroot = sysroot
161 # Members containing the sysroot (binpkg) and target (installed) package DB.
162 self.target_db = None
163 self.binpkgs_db = None
164 # Members for managing the dependency resolution work queue.
165 self.queue = None
166 self.seen = None
167 self.listed = None
168
169 @staticmethod
170 def _GetCP(cpv):
171 """Returns the CP value for a given CPV string."""
Alex Klein9742cb62020-10-12 19:22:10 +0000172 attrs = package_info.SplitCPV(cpv, strict=False)
Alex Klein9f93b482018-10-01 09:26:51 -0600173 if not attrs.cp:
David Pursell9476bf42015-03-30 13:34:27 -0700174 raise ValueError('Cannot get CP value for %s' % cpv)
Alex Klein9f93b482018-10-01 09:26:51 -0600175 return attrs.cp
David Pursell9476bf42015-03-30 13:34:27 -0700176
177 @staticmethod
178 def _InDB(cp, slot, db):
179 """Returns whether CP and slot are found in a database (if provided)."""
180 cp_slots = db.get(cp) if db else None
181 return cp_slots is not None and (not slot or slot in cp_slots)
182
183 @staticmethod
184 def _AtomStr(cp, slot):
185 """Returns 'CP:slot' if slot is non-empty, else just 'CP'."""
186 return '%s:%s' % (cp, slot) if slot else cp
187
188 @classmethod
189 def _GetVartreeSnippet(cls, root='/'):
190 """Returns a code snippet for dumping the vartree on the target.
191
192 Args:
193 root: The installation root.
194
195 Returns:
196 The said code snippet (string) with parameters filled in.
197 """
198 return cls._GET_VARTREE % {'root': root}
199
200 @classmethod
201 def _StripDepAtom(cls, dep_atom, installed_db=None):
202 """Strips a dependency atom and returns a (CP, slot) pair."""
203 # TODO(garnold) This is a gross simplification of ebuild dependency
204 # semantics, stripping and ignoring various qualifiers (versions, slots,
205 # USE flag, negation) and will likely need to be fixed. chromium:447366.
206
207 # Ignore unversioned blockers, leaving them for the user to resolve.
208 if dep_atom[0] == '!' and dep_atom[1] not in '<=>~':
209 return None, None
210
211 cp = dep_atom
212 slot = None
213 require_installed = False
214
215 # Versioned blockers should be updated, but only if already installed.
216 # These are often used for forcing cascaded updates of multiple packages,
217 # so we're treating them as ordinary constraints with hopes that it'll lead
218 # to the desired result.
219 if cp.startswith('!'):
220 cp = cp.lstrip('!')
221 require_installed = True
222
223 # Remove USE flags.
224 if '[' in cp:
225 cp = cp[:cp.index('[')] + cp[cp.index(']') + 1:]
226
227 # Separate the slot qualifier and strip off subslots.
228 if ':' in cp:
229 cp, slot = cp.split(':')
230 for delim in ('/', '='):
231 slot = slot.split(delim, 1)[0]
232
233 # Strip version wildcards (right), comparators (left).
234 cp = cp.rstrip('*')
235 cp = cp.lstrip('<=>~')
236
237 # Turn into CP form.
238 cp = cls._GetCP(cp)
239
240 if require_installed and not cls._InDB(cp, None, installed_db):
241 return None, None
242
243 return cp, slot
244
245 @classmethod
246 def _ProcessDepStr(cls, dep_str, installed_db, avail_db):
247 """Resolves and returns a list of dependencies from a dependency string.
248
249 This parses a dependency string and returns a list of package names and
250 slots. Other atom qualifiers (version, sub-slot, block) are ignored. When
251 resolving disjunctive deps, we include all choices that are fully present
252 in |installed_db|. If none is present, we choose an arbitrary one that is
253 available.
254
255 Args:
256 dep_str: A raw dependency string.
257 installed_db: A database of installed packages.
258 avail_db: A database of packages available for installation.
259
260 Returns:
261 A list of pairs (CP, slot).
262
263 Raises:
264 ValueError: the dependencies string is malformed.
265 """
266 def ProcessSubDeps(dep_exp, disjunct):
267 """Parses and processes a dependency (sub)expression."""
268 deps = set()
269 default_deps = set()
270 sub_disjunct = False
271 for dep_sub_exp in dep_exp:
272 sub_deps = set()
273
274 if isinstance(dep_sub_exp, (list, tuple)):
275 sub_deps = ProcessSubDeps(dep_sub_exp, sub_disjunct)
276 sub_disjunct = False
277 elif sub_disjunct:
278 raise ValueError('Malformed disjunctive operation in deps')
279 elif dep_sub_exp == '||':
280 sub_disjunct = True
281 elif dep_sub_exp.endswith('?'):
282 raise ValueError('Dependencies contain a conditional')
283 else:
284 cp, slot = cls._StripDepAtom(dep_sub_exp, installed_db)
285 if cp:
286 sub_deps = set([(cp, slot)])
287 elif disjunct:
288 raise ValueError('Atom in disjunct ignored')
289
290 # Handle sub-deps of a disjunctive expression.
291 if disjunct:
292 # Make the first available choice the default, for use in case that
293 # no option is installed.
294 if (not default_deps and avail_db is not None and
Mike Frysinger80ff4542022-05-06 23:52:04 -0400295 all(cls._InDB(cp, slot, avail_db) for cp, slot in sub_deps)):
David Pursell9476bf42015-03-30 13:34:27 -0700296 default_deps = sub_deps
297
298 # If not all sub-deps are installed, then don't consider them.
Mike Frysinger80ff4542022-05-06 23:52:04 -0400299 if not all(cls._InDB(cp, slot, installed_db)
300 for cp, slot in sub_deps):
David Pursell9476bf42015-03-30 13:34:27 -0700301 sub_deps = set()
302
303 deps.update(sub_deps)
304
305 return deps or default_deps
306
307 try:
308 return ProcessSubDeps(portage.dep.paren_reduce(dep_str), False)
309 except portage.exception.InvalidDependString as e:
310 raise ValueError('Invalid dep string: %s' % e)
311 except ValueError as e:
312 raise ValueError('%s: %s' % (e, dep_str))
313
314 def _BuildDB(self, cpv_info, process_rdeps, process_rev_rdeps,
315 installed_db=None):
316 """Returns a database of packages given a list of CPV info.
317
318 Args:
319 cpv_info: A list of tuples containing package CPV and attributes.
320 process_rdeps: Whether to populate forward dependencies.
321 process_rev_rdeps: Whether to populate reverse dependencies.
322 installed_db: A database of installed packages for filtering disjunctive
323 choices against; if None, using own built database.
324
325 Returns:
326 A map from CP values to another dictionary that maps slots to package
327 attribute tuples. Tuples contain a CPV value (string), build time
328 (string), runtime dependencies (set), and reverse dependencies (set,
329 empty if not populated).
330
331 Raises:
332 ValueError: If more than one CPV occupies a single slot.
333 """
334 db = {}
335 logging.debug('Populating package DB...')
336 for cpv, slot, rdeps_raw, build_time in cpv_info:
337 cp = self._GetCP(cpv)
338 cp_slots = db.setdefault(cp, dict())
339 if slot in cp_slots:
340 raise ValueError('More than one package found for %s' %
341 self._AtomStr(cp, slot))
342 logging.debug(' %s -> %s, built %s, raw rdeps: %s',
343 self._AtomStr(cp, slot), cpv, build_time, rdeps_raw)
344 cp_slots[slot] = self.PkgInfo(cpv, build_time, rdeps_raw)
345
346 avail_db = db
347 if installed_db is None:
348 installed_db = db
349 avail_db = None
350
351 # Add approximate forward dependencies.
352 if process_rdeps:
353 logging.debug('Populating forward dependencies...')
Mike Frysinger0bdbc102019-06-13 15:27:29 -0400354 for cp, cp_slots in db.items():
355 for slot, pkg_info in cp_slots.items():
David Pursell9476bf42015-03-30 13:34:27 -0700356 pkg_info.rdeps.update(self._ProcessDepStr(pkg_info.rdeps_raw,
357 installed_db, avail_db))
358 logging.debug(' %s (%s) processed rdeps: %s',
359 self._AtomStr(cp, slot), pkg_info.cpv,
360 ' '.join([self._AtomStr(rdep_cp, rdep_slot)
361 for rdep_cp, rdep_slot in pkg_info.rdeps]))
362
363 # Add approximate reverse dependencies (optional).
364 if process_rev_rdeps:
365 logging.debug('Populating reverse dependencies...')
Mike Frysinger0bdbc102019-06-13 15:27:29 -0400366 for cp, cp_slots in db.items():
367 for slot, pkg_info in cp_slots.items():
David Pursell9476bf42015-03-30 13:34:27 -0700368 for rdep_cp, rdep_slot in pkg_info.rdeps:
369 to_slots = db.get(rdep_cp)
370 if not to_slots:
371 continue
372
Mike Frysinger0bdbc102019-06-13 15:27:29 -0400373 for to_slot, to_pkg_info in to_slots.items():
David Pursell9476bf42015-03-30 13:34:27 -0700374 if rdep_slot and to_slot != rdep_slot:
375 continue
376 logging.debug(' %s (%s) added as rev rdep for %s (%s)',
377 self._AtomStr(cp, slot), pkg_info.cpv,
378 self._AtomStr(rdep_cp, to_slot), to_pkg_info.cpv)
379 to_pkg_info.rev_rdeps.add((cp, slot))
380
381 return db
382
383 def _InitTargetVarDB(self, device, root, process_rdeps, process_rev_rdeps):
384 """Initializes a dictionary of packages installed on |device|."""
385 get_vartree_script = self._GetVartreeSnippet(root)
386 try:
Mike Frysinger345666a2017-10-06 00:26:21 -0400387 result = device.GetAgent().RemoteSh(['python'], remote_sudo=True,
David Pursell67a82762015-04-30 17:26:59 -0700388 input=get_vartree_script)
David Pursell9476bf42015-03-30 13:34:27 -0700389 except cros_build_lib.RunCommandError as e:
Mike Frysingera2fac222022-08-17 21:06:33 -0400390 logging.error('Cannot get target vartree:\n%s', e.stderr)
David Pursell9476bf42015-03-30 13:34:27 -0700391 raise
392
393 try:
Mike Frysinger876a8e52022-06-23 18:07:30 -0400394 self.target_db = self._BuildDB(json.loads(result.stdout),
David Pursell9476bf42015-03-30 13:34:27 -0700395 process_rdeps, process_rev_rdeps)
396 except ValueError as e:
397 raise self.VartreeError(str(e))
398
399 def _InitBinpkgDB(self, process_rdeps):
400 """Initializes a dictionary of binary packages for updating the target."""
401 # Get build root trees; portage indexes require a trailing '/'.
402 build_root = os.path.join(self.sysroot, '')
403 trees = portage.create_trees(target_root=build_root, config_root=build_root)
404 bintree = trees[build_root]['bintree']
405 binpkgs_info = []
406 for cpv in bintree.dbapi.cpv_all():
407 slot, rdep_raw, build_time = bintree.dbapi.aux_get(
408 cpv, ['SLOT', 'RDEPEND', 'BUILD_TIME'])
409 binpkgs_info.append((cpv, slot, rdep_raw, build_time))
410
411 try:
412 self.binpkgs_db = self._BuildDB(binpkgs_info, process_rdeps, False,
413 installed_db=self.target_db)
414 except ValueError as e:
415 raise self.BintreeError(str(e))
416
417 def _InitDepQueue(self):
418 """Initializes the dependency work queue."""
419 self.queue = set()
420 self.seen = {}
421 self.listed = set()
422
423 def _EnqDep(self, dep, listed, optional):
424 """Enqueues a dependency if not seen before or if turned non-optional."""
425 if dep in self.seen and (optional or not self.seen[dep]):
426 return False
427
428 self.queue.add(dep)
429 self.seen[dep] = optional
430 if listed:
431 self.listed.add(dep)
432 return True
433
434 def _DeqDep(self):
435 """Dequeues and returns a dependency, its listed and optional flags.
436
437 This returns listed packages first, if any are present, to ensure that we
438 correctly mark them as such when they are first being processed.
439 """
440 if self.listed:
441 dep = self.listed.pop()
442 self.queue.remove(dep)
443 listed = True
444 else:
445 dep = self.queue.pop()
446 listed = False
447
448 return dep, listed, self.seen[dep]
449
450 def _FindPackageMatches(self, cpv_pattern):
451 """Returns list of binpkg (CP, slot) pairs that match |cpv_pattern|.
452
453 This is breaking |cpv_pattern| into its C, P and V components, each of
454 which may or may not be present or contain wildcards. It then scans the
455 binpkgs database to find all atoms that match these components, returning a
456 list of CP and slot qualifier. When the pattern does not specify a version,
457 or when a CP has only one slot in the binpkgs database, we omit the slot
458 qualifier in the result.
459
460 Args:
461 cpv_pattern: A CPV pattern, potentially partial and/or having wildcards.
462
463 Returns:
464 A list of (CPV, slot) pairs of packages in the binpkgs database that
465 match the pattern.
466 """
Alex Klein9742cb62020-10-12 19:22:10 +0000467 attrs = package_info.SplitCPV(cpv_pattern, strict=False)
David Pursell9476bf42015-03-30 13:34:27 -0700468 cp_pattern = os.path.join(attrs.category or '*', attrs.package or '*')
469 matches = []
Mike Frysinger0bdbc102019-06-13 15:27:29 -0400470 for cp, cp_slots in self.binpkgs_db.items():
David Pursell9476bf42015-03-30 13:34:27 -0700471 if not fnmatch.fnmatchcase(cp, cp_pattern):
472 continue
473
474 # If no version attribute was given or there's only one slot, omit the
475 # slot qualifier.
Alex Klein9742cb62020-10-12 19:22:10 +0000476 if not attrs.version or len(cp_slots) == 1:
David Pursell9476bf42015-03-30 13:34:27 -0700477 matches.append((cp, None))
478 else:
Alex Klein9742cb62020-10-12 19:22:10 +0000479 cpv_pattern = '%s-%s' % (cp, attrs.version)
Mike Frysinger0bdbc102019-06-13 15:27:29 -0400480 for slot, pkg_info in cp_slots.items():
David Pursell9476bf42015-03-30 13:34:27 -0700481 if fnmatch.fnmatchcase(pkg_info.cpv, cpv_pattern):
482 matches.append((cp, slot))
483
484 return matches
485
486 def _FindPackage(self, pkg):
487 """Returns the (CP, slot) pair for a package matching |pkg|.
488
489 Args:
490 pkg: Path to a binary package or a (partial) package CPV specifier.
491
492 Returns:
493 A (CP, slot) pair for the given package; slot may be None (unspecified).
494
495 Raises:
496 ValueError: if |pkg| is not a binpkg file nor does it match something
497 that's in the bintree.
498 """
499 if pkg.endswith('.tbz2') and os.path.isfile(pkg):
500 package = os.path.basename(os.path.splitext(pkg)[0])
501 category = os.path.basename(os.path.dirname(pkg))
502 return self._GetCP(os.path.join(category, package)), None
503
504 matches = self._FindPackageMatches(pkg)
505 if not matches:
506 raise ValueError('No package found for %s' % pkg)
507
508 idx = 0
509 if len(matches) > 1:
510 # Ask user to pick among multiple matches.
511 idx = cros_build_lib.GetChoice('Multiple matches found for %s: ' % pkg,
512 ['%s:%s' % (cp, slot) if slot else cp
513 for cp, slot in matches])
514
515 return matches[idx]
516
517 def _NeedsInstall(self, cpv, slot, build_time, optional):
518 """Returns whether a package needs to be installed on the target.
519
520 Args:
521 cpv: Fully qualified CPV (string) of the package.
522 slot: Slot identifier (string).
523 build_time: The BUILT_TIME value (string) of the binpkg.
524 optional: Whether package is optional on the target.
525
526 Returns:
527 A tuple (install, update) indicating whether to |install| the package and
528 whether it is an |update| to an existing package.
529
530 Raises:
531 ValueError: if slot is not provided.
532 """
533 # If not checking installed packages, always install.
534 if not self.target_db:
535 return True, False
536
537 cp = self._GetCP(cpv)
538 target_pkg_info = self.target_db.get(cp, dict()).get(slot)
539 if target_pkg_info is not None:
540 if cpv != target_pkg_info.cpv:
Alex Klein9742cb62020-10-12 19:22:10 +0000541 attrs = package_info.SplitCPV(cpv)
542 target_attrs = package_info.SplitCPV(target_pkg_info.cpv)
David Pursell9476bf42015-03-30 13:34:27 -0700543 logging.debug('Updating %s: version (%s) different on target (%s)',
Alex Klein9742cb62020-10-12 19:22:10 +0000544 cp, attrs.version, target_attrs.version)
David Pursell9476bf42015-03-30 13:34:27 -0700545 return True, True
546
547 if build_time != target_pkg_info.build_time:
548 logging.debug('Updating %s: build time (%s) different on target (%s)',
549 cpv, build_time, target_pkg_info.build_time)
550 return True, True
551
552 logging.debug('Not updating %s: already up-to-date (%s, built %s)',
553 cp, target_pkg_info.cpv, target_pkg_info.build_time)
554 return False, False
555
556 if optional:
557 logging.debug('Not installing %s: missing on target but optional', cp)
558 return False, False
559
560 logging.debug('Installing %s: missing on target and non-optional (%s)',
561 cp, cpv)
562 return True, False
563
564 def _ProcessDeps(self, deps, reverse):
565 """Enqueues dependencies for processing.
566
567 Args:
568 deps: List of dependencies to enqueue.
569 reverse: Whether these are reverse dependencies.
570 """
571 if not deps:
572 return
573
574 logging.debug('Processing %d %s dep(s)...', len(deps),
575 'reverse' if reverse else 'forward')
576 num_already_seen = 0
577 for dep in deps:
578 if self._EnqDep(dep, False, reverse):
579 logging.debug(' Queued dep %s', dep)
580 else:
581 num_already_seen += 1
582
583 if num_already_seen:
584 logging.debug('%d dep(s) already seen', num_already_seen)
585
586 def _ComputeInstalls(self, process_rdeps, process_rev_rdeps):
587 """Returns a dictionary of packages that need to be installed on the target.
588
589 Args:
590 process_rdeps: Whether to trace forward dependencies.
591 process_rev_rdeps: Whether to trace backward dependencies as well.
592
593 Returns:
594 A dictionary mapping CP values (string) to tuples containing a CPV
595 (string), a slot (string), a boolean indicating whether the package
596 was initially listed in the queue, and a boolean indicating whether this
597 is an update to an existing package.
598 """
599 installs = {}
600 while self.queue:
601 dep, listed, optional = self._DeqDep()
602 cp, required_slot = dep
603 if cp in installs:
604 logging.debug('Already updating %s', cp)
605 continue
606
607 cp_slots = self.binpkgs_db.get(cp, dict())
608 logging.debug('Checking packages matching %s%s%s...', cp,
609 ' (slot: %s)' % required_slot if required_slot else '',
610 ' (optional)' if optional else '')
611 num_processed = 0
Mike Frysinger0bdbc102019-06-13 15:27:29 -0400612 for slot, pkg_info in cp_slots.items():
David Pursell9476bf42015-03-30 13:34:27 -0700613 if required_slot and slot != required_slot:
614 continue
615
616 num_processed += 1
617 logging.debug(' Checking %s...', pkg_info.cpv)
618
619 install, update = self._NeedsInstall(pkg_info.cpv, slot,
620 pkg_info.build_time, optional)
621 if not install:
622 continue
623
624 installs[cp] = (pkg_info.cpv, slot, listed, update)
625
626 # Add forward and backward runtime dependencies to queue.
627 if process_rdeps:
628 self._ProcessDeps(pkg_info.rdeps, False)
629 if process_rev_rdeps:
630 target_pkg_info = self.target_db.get(cp, dict()).get(slot)
631 if target_pkg_info:
632 self._ProcessDeps(target_pkg_info.rev_rdeps, True)
633
634 if num_processed == 0:
635 logging.warning('No qualified bintree package corresponding to %s', cp)
636
637 return installs
638
639 def _SortInstalls(self, installs):
640 """Returns a sorted list of packages to install.
641
642 Performs a topological sort based on dependencies found in the binary
643 package database.
644
645 Args:
646 installs: Dictionary of packages to install indexed by CP.
647
648 Returns:
649 A list of package CPVs (string).
650
651 Raises:
652 ValueError: If dependency graph contains a cycle.
653 """
654 not_visited = set(installs.keys())
655 curr_path = []
656 sorted_installs = []
657
658 def SortFrom(cp):
659 """Traverses dependencies recursively, emitting nodes in reverse order."""
660 cpv, slot, _, _ = installs[cp]
661 if cpv in curr_path:
662 raise ValueError('Dependencies contain a cycle: %s -> %s' %
663 (' -> '.join(curr_path[curr_path.index(cpv):]), cpv))
664 curr_path.append(cpv)
665 for rdep_cp, _ in self.binpkgs_db[cp][slot].rdeps:
666 if rdep_cp in not_visited:
667 not_visited.remove(rdep_cp)
668 SortFrom(rdep_cp)
669
670 sorted_installs.append(cpv)
671 curr_path.pop()
672
673 # So long as there's more packages, keep expanding dependency paths.
674 while not_visited:
675 SortFrom(not_visited.pop())
676
677 return sorted_installs
678
679 def _EnqListedPkg(self, pkg):
680 """Finds and enqueues a listed package."""
681 cp, slot = self._FindPackage(pkg)
682 if cp not in self.binpkgs_db:
683 raise self.BintreeError('Package %s not found in binpkgs tree' % pkg)
684 self._EnqDep((cp, slot), True, False)
685
686 def _EnqInstalledPkgs(self):
687 """Enqueues all available binary packages that are already installed."""
Mike Frysinger0bdbc102019-06-13 15:27:29 -0400688 for cp, cp_slots in self.binpkgs_db.items():
David Pursell9476bf42015-03-30 13:34:27 -0700689 target_cp_slots = self.target_db.get(cp)
690 if target_cp_slots:
Mike Frysinger0bdbc102019-06-13 15:27:29 -0400691 for slot in cp_slots.keys():
David Pursell9476bf42015-03-30 13:34:27 -0700692 if slot in target_cp_slots:
693 self._EnqDep((cp, slot), True, False)
694
695 def Run(self, device, root, listed_pkgs, update, process_rdeps,
696 process_rev_rdeps):
697 """Computes the list of packages that need to be installed on a target.
698
699 Args:
700 device: Target handler object.
701 root: Package installation root.
702 listed_pkgs: Package names/files listed by the user.
703 update: Whether to read the target's installed package database.
704 process_rdeps: Whether to trace forward dependencies.
705 process_rev_rdeps: Whether to trace backward dependencies as well.
706
707 Returns:
Xiaochu Liu2726e7c2019-07-18 10:28:10 -0700708 A tuple (sorted, listed, num_updates, install_attrs) where |sorted| is a
709 list of package CPVs (string) to install on the target in an order that
710 satisfies their inter-dependencies, |listed| the subset that was
711 requested by the user, and |num_updates| the number of packages being
712 installed over preexisting versions. Note that installation order should
713 be reversed for removal, |install_attrs| is a dictionary mapping a package
714 CPV (string) to some of its extracted environment attributes.
David Pursell9476bf42015-03-30 13:34:27 -0700715 """
716 if process_rev_rdeps and not process_rdeps:
717 raise ValueError('Must processing forward deps when processing rev deps')
718 if process_rdeps and not update:
719 raise ValueError('Must check installed packages when processing deps')
720
721 if update:
722 logging.info('Initializing target intalled packages database...')
723 self._InitTargetVarDB(device, root, process_rdeps, process_rev_rdeps)
724
725 logging.info('Initializing binary packages database...')
726 self._InitBinpkgDB(process_rdeps)
727
728 logging.info('Finding listed package(s)...')
729 self._InitDepQueue()
730 for pkg in listed_pkgs:
731 if pkg == '@installed':
732 if not update:
733 raise ValueError(
734 'Must check installed packages when updating all of them.')
735 self._EnqInstalledPkgs()
736 else:
737 self._EnqListedPkg(pkg)
738
739 logging.info('Computing set of packages to install...')
740 installs = self._ComputeInstalls(process_rdeps, process_rev_rdeps)
741
742 num_updates = 0
743 listed_installs = []
Mike Frysinger8ab15bb2019-09-18 17:24:36 -0400744 for cpv, _, listed, isupdate in installs.values():
David Pursell9476bf42015-03-30 13:34:27 -0700745 if listed:
746 listed_installs.append(cpv)
Mike Frysinger8ab15bb2019-09-18 17:24:36 -0400747 if isupdate:
David Pursell9476bf42015-03-30 13:34:27 -0700748 num_updates += 1
749
750 logging.info('Processed %d package(s), %d will be installed, %d are '
751 'updating existing packages',
752 len(self.seen), len(installs), num_updates)
753
754 sorted_installs = self._SortInstalls(installs)
Xiaochu Liu2726e7c2019-07-18 10:28:10 -0700755
756 install_attrs = {}
757 for pkg in sorted_installs:
Mike Frysingerada2d1c2020-03-20 05:02:06 -0400758 pkg_path = os.path.join(root, portage_util.VDB_PATH, pkg)
Xiaochu Liu2726e7c2019-07-18 10:28:10 -0700759 dlc_id, dlc_package = _GetDLCInfo(device, pkg_path, from_dut=True)
760 install_attrs[pkg] = {}
761 if dlc_id and dlc_package:
762 install_attrs[pkg][_DLC_ID] = dlc_id
763
764 return sorted_installs, listed_installs, num_updates, install_attrs
David Pursell9476bf42015-03-30 13:34:27 -0700765
766
Mike Frysinger63d35512021-01-26 23:16:13 -0500767def _Emerge(device, pkg_paths, root, extra_args=None):
768 """Copies |pkg_paths| to |device| and emerges them.
David Pursell9476bf42015-03-30 13:34:27 -0700769
770 Args:
771 device: A ChromiumOSDevice object.
Mike Frysinger63d35512021-01-26 23:16:13 -0500772 pkg_paths: (Local) paths to binary packages.
David Pursell9476bf42015-03-30 13:34:27 -0700773 root: Package installation root path.
774 extra_args: Extra arguments to pass to emerge.
775
776 Raises:
777 DeployError: Unrecoverable error during emerge.
778 """
Mike Frysinger63d35512021-01-26 23:16:13 -0500779 def path_to_name(pkg_path):
780 return os.path.basename(pkg_path)
781 def path_to_category(pkg_path):
782 return os.path.basename(os.path.dirname(pkg_path))
783
784 pkg_names = ', '.join(path_to_name(x) for x in pkg_paths)
785
David Pursell9476bf42015-03-30 13:34:27 -0700786 pkgroot = os.path.join(device.work_dir, 'packages')
Mike Frysinger15a4e012015-05-21 22:18:45 -0400787 portage_tmpdir = os.path.join(device.work_dir, 'portage-tmp')
788 # Clean out the dirs first if we had a previous emerge on the device so as to
789 # free up space for this emerge. The last emerge gets implicitly cleaned up
790 # when the device connection deletes its work_dir.
Mike Frysinger3459bf52020-03-31 00:52:11 -0400791 device.run(
Mike Frysinger63d35512021-01-26 23:16:13 -0500792 f'cd {device.work_dir} && '
793 f'rm -rf packages portage-tmp && '
794 f'mkdir -p portage-tmp packages && '
795 f'cd packages && '
796 f'mkdir -p {" ".join(set(path_to_category(x) for x in pkg_paths))}',
797 shell=True, remote_sudo=True)
David Pursell9476bf42015-03-30 13:34:27 -0700798
David Pursell9476bf42015-03-30 13:34:27 -0700799 logging.info('Use portage temp dir %s', portage_tmpdir)
800
Ralph Nathane01ccf12015-04-16 10:40:32 -0700801 # This message is read by BrilloDeployOperation.
Mike Frysinger63d35512021-01-26 23:16:13 -0500802 logging.notice('Copying binpkgs to device.')
803 for pkg_path in pkg_paths:
804 pkg_name = path_to_name(pkg_path)
805 logging.info('Copying %s', pkg_name)
806 pkg_dir = os.path.join(pkgroot, path_to_category(pkg_path))
807 device.CopyToDevice(pkg_path, pkg_dir, mode='rsync', remote_sudo=True,
808 compress=False)
809
810 # This message is read by BrilloDeployOperation.
811 logging.notice('Installing: %s', pkg_names)
David Pursell9476bf42015-03-30 13:34:27 -0700812
813 # We set PORTAGE_CONFIGROOT to '/usr/local' because by default all
814 # chromeos-base packages will be skipped due to the configuration
815 # in /etc/protage/make.profile/package.provided. However, there is
816 # a known bug that /usr/local/etc/portage is not setup properly
817 # (crbug.com/312041). This does not affect `cros deploy` because
818 # we do not use the preset PKGDIR.
819 extra_env = {
820 'FEATURES': '-sandbox',
821 'PKGDIR': pkgroot,
822 'PORTAGE_CONFIGROOT': '/usr/local',
823 'PORTAGE_TMPDIR': portage_tmpdir,
824 'PORTDIR': device.work_dir,
825 'CONFIG_PROTECT': '-*',
826 }
Mike Frysinger63d35512021-01-26 23:16:13 -0500827
Alex Kleinaaddc932020-01-30 15:02:24 -0700828 # --ignore-built-slot-operator-deps because we don't rebuild everything.
829 # It can cause errors, but that's expected with cros deploy since it's just a
830 # best effort to prevent developers avoid rebuilding an image every time.
Mike Frysinger63d35512021-01-26 23:16:13 -0500831 cmd = ['emerge', '--usepkg', '--ignore-built-slot-operator-deps=y', '--root',
832 root] + [os.path.join(pkgroot, *x.split('/')[-2:]) for x in pkg_paths]
David Pursell9476bf42015-03-30 13:34:27 -0700833 if extra_args:
834 cmd.append(extra_args)
835
Alex Kleinaaddc932020-01-30 15:02:24 -0700836 logging.warning('Ignoring slot dependencies! This may break things! e.g. '
837 'packages built against the old version may not be able to '
838 'load the new .so. This is expected, and you will just need '
839 'to build and flash a new image if you have problems.')
David Pursell9476bf42015-03-30 13:34:27 -0700840 try:
Mike Frysinger3459bf52020-03-31 00:52:11 -0400841 result = device.run(cmd, extra_env=extra_env, remote_sudo=True,
842 capture_output=True, debug_level=logging.INFO)
Greg Kerrb96c02c2019-02-08 14:32:41 -0800843
844 pattern = ('A requested package will not be merged because '
845 'it is listed in package.provided')
Mike Frysinger876a8e52022-06-23 18:07:30 -0400846 output = result.stderr.replace('\n', ' ').replace('\r', '')
Greg Kerrb96c02c2019-02-08 14:32:41 -0800847 if pattern in output:
848 error = ('Package failed to emerge: %s\n'
849 'Remove %s from /etc/portage/make.profile/'
850 'package.provided/chromeos-base.packages\n'
851 '(also see crbug.com/920140 for more context)\n'
852 % (pattern, pkg_name))
853 cros_build_lib.Die(error)
David Pursell9476bf42015-03-30 13:34:27 -0700854 except Exception:
Mike Frysinger63d35512021-01-26 23:16:13 -0500855 logging.error('Failed to emerge packages %s', pkg_names)
David Pursell9476bf42015-03-30 13:34:27 -0700856 raise
857 else:
Mike Frysinger63d35512021-01-26 23:16:13 -0500858 # This message is read by BrilloDeployOperation.
859 logging.notice('Packages have been installed.')
David Pursell9476bf42015-03-30 13:34:27 -0700860
861
Qijiang Fand5958192019-07-26 12:32:36 +0900862def _RestoreSELinuxContext(device, pkgpath, root):
Andrewc7e1c6b2020-02-27 16:03:53 -0800863 """Restore SELinux context for files in a given package.
Qijiang Fan8a945032019-04-25 20:53:29 +0900864
865 This reads the tarball from pkgpath, and calls restorecon on device to
866 restore SELinux context for files listed in the tarball, assuming those files
867 are installed to /
868
869 Args:
870 device: a ChromiumOSDevice object
871 pkgpath: path to tarball
Qijiang Fand5958192019-07-26 12:32:36 +0900872 root: Package installation root path.
Qijiang Fan8a945032019-04-25 20:53:29 +0900873 """
Qijiang Fan8a945032019-04-25 20:53:29 +0900874 pkgroot = os.path.join(device.work_dir, 'packages')
875 pkg_dirname = os.path.basename(os.path.dirname(pkgpath))
876 pkgpath_device = os.path.join(pkgroot, pkg_dirname, os.path.basename(pkgpath))
877 # Testing shows restorecon splits on newlines instead of spaces.
Mike Frysinger3459bf52020-03-31 00:52:11 -0400878 device.run(
Qijiang Fand5958192019-07-26 12:32:36 +0900879 ['cd', root, '&&',
880 'tar', 'tf', pkgpath_device, '|',
881 'restorecon', '-i', '-f', '-'],
Qijiang Fan8a945032019-04-25 20:53:29 +0900882 remote_sudo=True)
Qijiang Fan352d0eb2019-02-25 13:10:08 +0900883
884
Gilad Arnold0e1b1da2015-06-10 06:41:05 -0700885def _GetPackagesByCPV(cpvs, strip, sysroot):
886 """Returns paths to binary packages corresponding to |cpvs|.
887
888 Args:
Alex Klein9742cb62020-10-12 19:22:10 +0000889 cpvs: List of CPV components given by package_info.SplitCPV().
Gilad Arnold0e1b1da2015-06-10 06:41:05 -0700890 strip: True to run strip_package.
891 sysroot: Sysroot path.
892
893 Returns:
894 List of paths corresponding to |cpvs|.
895
896 Raises:
897 DeployError: If a package is missing.
898 """
899 packages_dir = None
900 if strip:
901 try:
Mike Frysinger45602c72019-09-22 02:15:11 -0400902 cros_build_lib.run(
Ram Chandrasekar56152ec2021-11-22 17:10:41 +0000903 [os.path.join(constants.CHROMITE_SCRIPTS_DIR, 'strip_package'),
904 '--sysroot', sysroot] +
Alex Klein9742cb62020-10-12 19:22:10 +0000905 [cpv.cpf for cpv in cpvs])
Gilad Arnold0e1b1da2015-06-10 06:41:05 -0700906 packages_dir = _STRIPPED_PACKAGES_DIR
907 except cros_build_lib.RunCommandError:
908 logging.error('Cannot strip packages %s',
Alex Klein9742cb62020-10-12 19:22:10 +0000909 ' '.join([str(cpv) for cpv in cpvs]))
Gilad Arnold0e1b1da2015-06-10 06:41:05 -0700910 raise
911
912 paths = []
913 for cpv in cpvs:
914 path = portage_util.GetBinaryPackagePath(
915 cpv.category, cpv.package, cpv.version, sysroot=sysroot,
916 packages_dir=packages_dir)
917 if not path:
918 raise DeployError('Missing package %s.' % cpv)
919 paths.append(path)
920
921 return paths
922
923
924def _GetPackagesPaths(pkgs, strip, sysroot):
925 """Returns paths to binary |pkgs|.
926
Gilad Arnold0e1b1da2015-06-10 06:41:05 -0700927 Args:
Ned Nguyend0db4072019-02-22 14:19:21 -0700928 pkgs: List of package CPVs string.
Gilad Arnold0e1b1da2015-06-10 06:41:05 -0700929 strip: Whether or not to run strip_package for CPV packages.
930 sysroot: The sysroot path.
931
932 Returns:
933 List of paths corresponding to |pkgs|.
934 """
Alex Klein9742cb62020-10-12 19:22:10 +0000935 cpvs = [package_info.SplitCPV(p) for p in pkgs]
Ned Nguyend0db4072019-02-22 14:19:21 -0700936 return _GetPackagesByCPV(cpvs, strip, sysroot)
Gilad Arnold0e1b1da2015-06-10 06:41:05 -0700937
938
Mike Frysinger22bb5502021-01-29 13:05:46 -0500939def _Unmerge(device, pkgs, root):
940 """Unmerges |pkgs| on |device|.
David Pursell9476bf42015-03-30 13:34:27 -0700941
942 Args:
943 device: A RemoteDevice object.
Mike Frysinger22bb5502021-01-29 13:05:46 -0500944 pkgs: Package names.
David Pursell9476bf42015-03-30 13:34:27 -0700945 root: Package installation root path.
946 """
Mike Frysinger22bb5502021-01-29 13:05:46 -0500947 pkg_names = ', '.join(os.path.basename(x) for x in pkgs)
Ralph Nathane01ccf12015-04-16 10:40:32 -0700948 # This message is read by BrilloDeployOperation.
Mike Frysinger22bb5502021-01-29 13:05:46 -0500949 logging.notice('Unmerging %s.', pkg_names)
David Pursell9476bf42015-03-30 13:34:27 -0700950 cmd = ['qmerge', '--yes']
951 # Check if qmerge is available on the device. If not, use emerge.
Mike Frysinger3459bf52020-03-31 00:52:11 -0400952 if device.run(['qmerge', '--version'], check=False).returncode != 0:
David Pursell9476bf42015-03-30 13:34:27 -0700953 cmd = ['emerge']
954
Mike Frysinger22bb5502021-01-29 13:05:46 -0500955 cmd += ['--unmerge', '--root', root]
956 cmd.extend('f={x}' for x in pkgs)
David Pursell9476bf42015-03-30 13:34:27 -0700957 try:
Ralph Nathane01ccf12015-04-16 10:40:32 -0700958 # Always showing the emerge output for clarity.
Mike Frysinger3459bf52020-03-31 00:52:11 -0400959 device.run(cmd, capture_output=False, remote_sudo=True,
960 debug_level=logging.INFO)
David Pursell9476bf42015-03-30 13:34:27 -0700961 except Exception:
Mike Frysinger22bb5502021-01-29 13:05:46 -0500962 logging.error('Failed to unmerge packages %s', pkg_names)
David Pursell9476bf42015-03-30 13:34:27 -0700963 raise
964 else:
Mike Frysinger22bb5502021-01-29 13:05:46 -0500965 # This message is read by BrilloDeployOperation.
966 logging.notice('Packages have been uninstalled.')
David Pursell9476bf42015-03-30 13:34:27 -0700967
968
969def _ConfirmDeploy(num_updates):
970 """Returns whether we can continue deployment."""
971 if num_updates > _MAX_UPDATES_NUM:
972 logging.warning(_MAX_UPDATES_WARNING)
973 return cros_build_lib.BooleanPrompt(default=False)
974
975 return True
976
977
Andrew06a5f812020-01-23 08:08:32 -0800978def _EmergePackages(pkgs, device, strip, sysroot, root, board, emerge_args):
Andrewc7e1c6b2020-02-27 16:03:53 -0800979 """Call _Emerge for each package in pkgs."""
Mike Frysinger4eb5f4e2021-01-26 21:48:37 -0500980 if device.IsSELinuxAvailable():
981 enforced = device.IsSELinuxEnforced()
982 if enforced:
983 device.run(['setenforce', '0'])
984 else:
985 enforced = False
986
Xiaochu Liu2726e7c2019-07-18 10:28:10 -0700987 dlc_deployed = False
Mike Frysinger63d35512021-01-26 23:16:13 -0500988 # This message is read by BrilloDeployOperation.
989 logging.info('Preparing local packages for transfer.')
990 pkg_paths = _GetPackagesPaths(pkgs, strip, sysroot)
991 # Install all the packages in one pass so inter-package blockers work.
992 _Emerge(device, pkg_paths, root, extra_args=emerge_args)
993 logging.info('Updating SELinux settings & DLC images.')
994 for pkg_path in pkg_paths:
Ben Pastene5f03b052019-08-12 18:03:24 -0700995 if device.IsSELinuxAvailable():
Qijiang Fand5958192019-07-26 12:32:36 +0900996 _RestoreSELinuxContext(device, pkg_path, root)
Andrewc7e1c6b2020-02-27 16:03:53 -0800997
998 dlc_id, dlc_package = _GetDLCInfo(device, pkg_path, from_dut=False)
999 if dlc_id and dlc_package:
Andrew06a5f812020-01-23 08:08:32 -08001000 _DeployDLCImage(device, sysroot, board, dlc_id, dlc_package)
Xiaochu Liu2726e7c2019-07-18 10:28:10 -07001001 dlc_deployed = True
Mike Frysinger5f4c2742021-02-08 14:37:23 -05001002
1003 if dlc_deployed:
1004 # Clean up empty directories created by emerging DLCs.
1005 device.run(['test', '-d', '/build/rootfs', '&&', 'rmdir',
1006 '--ignore-fail-on-non-empty', '/build/rootfs', '/build'],
1007 check=False)
Xiaochu Liu2726e7c2019-07-18 10:28:10 -07001008
Mike Frysinger4eb5f4e2021-01-26 21:48:37 -05001009 if enforced:
1010 device.run(['setenforce', '1'])
1011
Xiaochu Liu2726e7c2019-07-18 10:28:10 -07001012 # Restart dlcservice so it picks up the newly installed DLC modules (in case
1013 # we installed new DLC images).
1014 if dlc_deployed:
Mike Frysinger3459bf52020-03-31 00:52:11 -04001015 device.run(['restart', 'dlcservice'])
Ralph Nathane01ccf12015-04-16 10:40:32 -07001016
1017
Xiaochu Liu2726e7c2019-07-18 10:28:10 -07001018def _UnmergePackages(pkgs, device, root, pkgs_attrs):
Ralph Nathane01ccf12015-04-16 10:40:32 -07001019 """Call _Unmege for each package in pkgs."""
Xiaochu Liu2726e7c2019-07-18 10:28:10 -07001020 dlc_uninstalled = False
Mike Frysinger22bb5502021-01-29 13:05:46 -05001021 _Unmerge(device, pkgs, root)
1022 logging.info('Cleaning up DLC images.')
Ralph Nathane01ccf12015-04-16 10:40:32 -07001023 for pkg in pkgs:
Xiaochu Liu2726e7c2019-07-18 10:28:10 -07001024 if _UninstallDLCImage(device, pkgs_attrs[pkg]):
1025 dlc_uninstalled = True
1026
1027 # Restart dlcservice so it picks up the uninstalled DLC modules (in case we
1028 # uninstalled DLC images).
1029 if dlc_uninstalled:
Mike Frysinger3459bf52020-03-31 00:52:11 -04001030 device.run(['restart', 'dlcservice'])
Xiaochu Liu2726e7c2019-07-18 10:28:10 -07001031
1032
1033def _UninstallDLCImage(device, pkg_attrs):
1034 """Uninstall a DLC image."""
1035 if _DLC_ID in pkg_attrs:
1036 dlc_id = pkg_attrs[_DLC_ID]
1037 logging.notice('Uninstalling DLC image for %s', dlc_id)
1038
Jae Hoon Kim964ed7e2020-05-15 13:59:23 -07001039 device.run(['dlcservice_util', '--uninstall', '--id=%s' % dlc_id])
Xiaochu Liu2726e7c2019-07-18 10:28:10 -07001040 return True
1041 else:
1042 logging.debug('DLC_ID not found in package')
1043 return False
1044
1045
Andrew06a5f812020-01-23 08:08:32 -08001046def _DeployDLCImage(device, sysroot, board, dlc_id, dlc_package):
Jae Hoon Kim2376e142022-09-03 00:18:58 +00001047 """Deploy (install and mount) a DLC image.
Xiaochu Liu2726e7c2019-07-18 10:28:10 -07001048
Jae Hoon Kim2376e142022-09-03 00:18:58 +00001049 Args:
1050 device: A device object.
1051 sysroot: The sysroot path.
1052 board: Board to use.
1053 dlc_id: The DLC ID.
1054 dlc_package: The DLC package name.
1055 """
1056 # Requires `sudo_rm` because installations of files are running with sudo.
1057 with osutils.TempDir(sudo_rm=True) as tempdir:
1058 temp_rootfs = Path(tempdir)
1059 # Build the DLC image if the image is outdated or doesn't exist.
1060 dlc_lib.InstallDlcImages(
1061 sysroot=sysroot, rootfs=temp_rootfs, dlc_id=dlc_id, board=board)
Xiaochu Liu2726e7c2019-07-18 10:28:10 -07001062
Jae Hoon Kim2376e142022-09-03 00:18:58 +00001063 logging.debug('Uninstall DLC %s if it is installed.', dlc_id)
1064 try:
1065 device.run(['dlcservice_util', '--uninstall', '--id=%s' % dlc_id])
1066 except cros_build_lib.RunCommandError as e:
1067 logging.info('Failed to uninstall DLC:%s. Continue anyway.', e.stderr)
1068 except Exception:
1069 logging.error('Failed to uninstall DLC.')
1070 raise
Andrewc7e1c6b2020-02-27 16:03:53 -08001071
Jae Hoon Kim2376e142022-09-03 00:18:58 +00001072 # TODO(andrewlassalle): Copy the DLC image to the preload location instead
1073 # of to dlc_a and dlc_b, and let dlcserive install the images to their final
1074 # location.
1075 logging.notice('Deploy the DLC image for %s', dlc_id)
1076 dlc_img_path_src = os.path.join(sysroot, dlc_lib.DLC_BUILD_DIR, dlc_id,
1077 dlc_package, dlc_lib.DLC_IMAGE)
1078 dlc_img_path = os.path.join(_DLC_INSTALL_ROOT, dlc_id, dlc_package)
1079 dlc_img_path_a = os.path.join(dlc_img_path, 'dlc_a')
1080 dlc_img_path_b = os.path.join(dlc_img_path, 'dlc_b')
1081 # Create directories for DLC images.
1082 device.run(['mkdir', '-p', dlc_img_path_a, dlc_img_path_b])
1083 # Copy images to the destination directories.
1084 device.CopyToDevice(dlc_img_path_src, os.path.join(dlc_img_path_a,
1085 dlc_lib.DLC_IMAGE),
1086 mode='rsync')
1087 device.run(['cp', os.path.join(dlc_img_path_a, dlc_lib.DLC_IMAGE),
1088 os.path.join(dlc_img_path_b, dlc_lib.DLC_IMAGE)])
Xiaochu Liu2726e7c2019-07-18 10:28:10 -07001089
Jae Hoon Kim2376e142022-09-03 00:18:58 +00001090 # Set the proper perms and ownership so dlcservice can access the image.
1091 device.run(['chmod', '-R', 'u+rwX,go+rX,go-w', _DLC_INSTALL_ROOT])
1092 device.run(['chown', '-R', 'dlcservice:dlcservice', _DLC_INSTALL_ROOT])
1093
1094 # Copy metadata to device.
1095 dest_meta_dir = Path('/') / dlc_lib.DLC_META_DIR / dlc_id / dlc_package
Jae Hoon Kima4b7e092022-09-08 20:12:59 +00001096 device.run(['mkdir', '-p', dest_meta_dir])
Jae Hoon Kim2376e142022-09-03 00:18:58 +00001097 src_meta_dir = os.path.join(sysroot, dlc_lib.DLC_BUILD_DIR, dlc_id,
1098 dlc_package, dlc_lib.DLC_TMP_META_DIR)
1099 device.CopyToDevice(src_meta_dir + '/',
1100 dest_meta_dir,
1101 mode='rsync',
1102 recursive=True,
1103 remote_sudo=True)
1104
1105 # TODO(kimjae): Make this generic so it recomputes all the DLCs + copies
1106 # over a fresh list of dm-verity digests instead of appending and keeping
1107 # the stale digests when developers are testing.
1108
1109 # Copy the LoadPin dm-verity digests to device.
1110 loadpin = dlc_lib.DLC_LOADPIN_TRUSTED_VERITY_DIGESTS
1111 dst_loadpin = Path('/') / dlc_lib.DLC_META_DIR / loadpin
1112 src_loadpin = temp_rootfs / dlc_lib.DLC_META_DIR / loadpin
1113 if src_loadpin.exists():
1114 digests = set(osutils.ReadFile(src_loadpin).split())
1115 try:
1116 digests.update(device.CatFile(dst_loadpin).split())
1117 except remote_access.CatFileError:
1118 pass
1119
1120 with tempfile.NamedTemporaryFile(dir=temp_rootfs) as f:
1121 osutils.WriteFile(f.name, '\n'.join(digests))
1122 device.CopyToDevice(f.name, dst_loadpin, mode='rsync', remote_sudo=True)
Andrew67b5fa72020-02-05 14:14:48 -08001123
Xiaochu Liu2726e7c2019-07-18 10:28:10 -07001124
1125def _GetDLCInfo(device, pkg_path, from_dut):
1126 """Returns information of a DLC given its package path.
1127
1128 Args:
1129 device: commandline.Device object; None to use the default device.
1130 pkg_path: path to the package.
1131 from_dut: True if extracting DLC info from DUT, False if extracting DLC
1132 info from host.
1133
1134 Returns:
1135 A tuple (dlc_id, dlc_package).
1136 """
1137 environment_content = ''
1138 if from_dut:
1139 # On DUT, |pkg_path| is the directory which contains environment file.
1140 environment_path = os.path.join(pkg_path, _ENVIRONMENT_FILENAME)
Mike Frysingeracd06cd2021-01-27 13:33:52 -05001141 try:
1142 environment_data = device.CatFile(
1143 environment_path, max_size=None, encoding=None)
1144 except remote_access.CatFileError:
Xiaochu Liu2726e7c2019-07-18 10:28:10 -07001145 # The package is not installed on DUT yet. Skip extracting info.
Mike Frysingeracd06cd2021-01-27 13:33:52 -05001146 return None, None
Xiaochu Liu2726e7c2019-07-18 10:28:10 -07001147 else:
1148 # On host, pkg_path is tbz2 file which contains environment file.
1149 # Extract the metadata of the package file.
1150 data = portage.xpak.tbz2(pkg_path).get_data()
Mike Frysingeracd06cd2021-01-27 13:33:52 -05001151 environment_data = data[_ENVIRONMENT_FILENAME.encode('utf-8')]
1152
1153 # Extract the environment metadata.
1154 environment_content = bz2.decompress(environment_data)
Xiaochu Liu2726e7c2019-07-18 10:28:10 -07001155
1156 with tempfile.NamedTemporaryFile() as f:
1157 # Dumps content into a file so we can use osutils.SourceEnvironment.
1158 path = os.path.realpath(f.name)
Woody Chowde57a322020-01-07 16:18:52 +09001159 osutils.WriteFile(path, environment_content, mode='wb')
Andrew67b5fa72020-02-05 14:14:48 -08001160 content = osutils.SourceEnvironment(path, (_DLC_ID, _DLC_PACKAGE,
1161 _DLC_ENABLED))
1162
1163 dlc_enabled = content.get(_DLC_ENABLED)
1164 if dlc_enabled is not None and (dlc_enabled is False or
1165 str(dlc_enabled) == 'false'):
1166 logging.info('Installing DLC in rootfs.')
1167 return None, None
Xiaochu Liu2726e7c2019-07-18 10:28:10 -07001168 return content.get(_DLC_ID), content.get(_DLC_PACKAGE)
Ralph Nathane01ccf12015-04-16 10:40:32 -07001169
1170
Gilad Arnolda0a98062015-07-07 08:34:27 -07001171def Deploy(device, packages, board=None, emerge=True, update=False, deep=False,
1172 deep_rev=False, clean_binpkg=True, root='/', strip=True,
1173 emerge_args=None, ssh_private_key=None, ping=True, force=False,
1174 dry_run=False):
David Pursell9476bf42015-03-30 13:34:27 -07001175 """Deploys packages to a device.
1176
1177 Args:
David Pursell2e773382015-04-03 14:30:47 -07001178 device: commandline.Device object; None to use the default device.
David Pursell9476bf42015-03-30 13:34:27 -07001179 packages: List of packages (strings) to deploy to device.
1180 board: Board to use; None to automatically detect.
David Pursell9476bf42015-03-30 13:34:27 -07001181 emerge: True to emerge package, False to unmerge.
1182 update: Check installed version on device.
1183 deep: Install dependencies also. Implies |update|.
1184 deep_rev: Install reverse dependencies. Implies |deep|.
1185 clean_binpkg: Clean outdated binary packages.
1186 root: Package installation root path.
1187 strip: Run strip_package to filter out preset paths in the package.
1188 emerge_args: Extra arguments to pass to emerge.
1189 ssh_private_key: Path to an SSH private key file; None to use test keys.
1190 ping: True to ping the device before trying to connect.
Sloan Johnsoncdd53b72022-06-07 20:29:24 +00001191 force: Ignore confidence checks and prompts.
David Pursell9476bf42015-03-30 13:34:27 -07001192 dry_run: Print deployment plan but do not deploy anything.
1193
1194 Raises:
1195 ValueError: Invalid parameter or parameter combination.
1196 DeployError: Unrecoverable failure during deploy.
1197 """
1198 if deep_rev:
1199 deep = True
1200 if deep:
1201 update = True
1202
Gilad Arnolda0a98062015-07-07 08:34:27 -07001203 if not packages:
1204 raise DeployError('No packages provided, nothing to deploy.')
1205
David Pursell9476bf42015-03-30 13:34:27 -07001206 if update and not emerge:
1207 raise ValueError('Cannot update and unmerge.')
1208
David Pursell2e773382015-04-03 14:30:47 -07001209 if device:
1210 hostname, username, port = device.hostname, device.username, device.port
1211 else:
1212 hostname, username, port = None, None, None
1213
Gilad Arnold4d3ade72015-04-28 15:13:35 -07001214 lsb_release = None
Bertrand SIMONNET60c94492015-04-30 17:46:28 -07001215 sysroot = None
Gilad Arnold4d3ade72015-04-28 15:13:35 -07001216 try:
Mike Frysinger17844a02019-08-24 18:21:02 -04001217 # Somewhat confusing to clobber, but here we are.
1218 # pylint: disable=redefined-argument-from-local
Gilad Arnold5dc243a2015-07-07 08:22:43 -07001219 with remote_access.ChromiumOSDeviceHandler(
1220 hostname, port=port, username=username, private_key=ssh_private_key,
1221 base_dir=_DEVICE_BASE_DIR, ping=ping) as device:
Mike Frysinger539db512015-05-21 18:14:01 -04001222 lsb_release = device.lsb_release
David Pursell9476bf42015-03-30 13:34:27 -07001223
Gilad Arnolda0a98062015-07-07 08:34:27 -07001224 board = cros_build_lib.GetBoard(device_board=device.board,
1225 override_board=board)
1226 if not force and board != device.board:
1227 raise DeployError('Device (%s) is incompatible with board %s. Use '
Brian Norrisbee77382016-06-02 14:50:29 -07001228 '--force to deploy anyway.' % (device.board, board))
Bertrand SIMONNET60c94492015-04-30 17:46:28 -07001229
Mike Frysinger06a51c82021-04-06 11:39:17 -04001230 sysroot = build_target_lib.get_default_sysroot_path(board)
David Pursell9476bf42015-03-30 13:34:27 -07001231
Mike Frysinger5c7b9512020-12-04 02:30:56 -05001232 # Don't bother trying to clean for unmerges. We won't use the local db,
1233 # and it just slows things down for the user.
1234 if emerge and clean_binpkg:
Ralph Nathane01ccf12015-04-16 10:40:32 -07001235 logging.notice('Cleaning outdated binary packages from %s', sysroot)
Bertrand SIMONNET0f6029f2015-04-30 17:44:13 -07001236 portage_util.CleanOutdatedBinaryPackages(sysroot)
David Pursell9476bf42015-03-30 13:34:27 -07001237
Achuith Bhandarkar0487c312019-04-22 12:19:25 -07001238 # Remount rootfs as writable if necessary.
1239 if not device.MountRootfsReadWrite():
1240 raise DeployError('Cannot remount rootfs as read-write. Exiting.')
David Pursell9476bf42015-03-30 13:34:27 -07001241
1242 # Obtain list of packages to upgrade/remove.
1243 pkg_scanner = _InstallPackageScanner(sysroot)
Xiaochu Liu2726e7c2019-07-18 10:28:10 -07001244 pkgs, listed, num_updates, pkgs_attrs = pkg_scanner.Run(
Mike Frysinger539db512015-05-21 18:14:01 -04001245 device, root, packages, update, deep, deep_rev)
David Pursell9476bf42015-03-30 13:34:27 -07001246 if emerge:
1247 action_str = 'emerge'
1248 else:
1249 pkgs.reverse()
1250 action_str = 'unmerge'
1251
1252 if not pkgs:
Ralph Nathane01ccf12015-04-16 10:40:32 -07001253 logging.notice('No packages to %s', action_str)
David Pursell9476bf42015-03-30 13:34:27 -07001254 return
1255
Mike Frysinger5c7b9512020-12-04 02:30:56 -05001256 # Warn when the user installs & didn't `cros workon start`.
1257 if emerge:
Brian Norris2eee8892021-04-06 16:23:23 -07001258 all_workon = workon_helper.WorkonHelper(sysroot).ListAtoms(use_all=True)
Mike Frysinger5c7b9512020-12-04 02:30:56 -05001259 worked_on_cps = workon_helper.WorkonHelper(sysroot).ListAtoms()
1260 for package in listed:
1261 cp = package_info.SplitCPV(package).cp
Brian Norris2eee8892021-04-06 16:23:23 -07001262 if cp in all_workon and cp not in worked_on_cps:
Mike Frysinger5c7b9512020-12-04 02:30:56 -05001263 logging.warning(
1264 'Are you intentionally deploying unmodified packages, or did '
1265 'you forget to run `cros workon --board=$BOARD start %s`?', cp)
Kimiyuki Onakaa4ec7f62020-08-25 13:58:48 +09001266
Ralph Nathane01ccf12015-04-16 10:40:32 -07001267 logging.notice('These are the packages to %s:', action_str)
David Pursell9476bf42015-03-30 13:34:27 -07001268 for i, pkg in enumerate(pkgs):
Ralph Nathane01ccf12015-04-16 10:40:32 -07001269 logging.notice('%s %d) %s', '*' if pkg in listed else ' ', i + 1, pkg)
David Pursell9476bf42015-03-30 13:34:27 -07001270
1271 if dry_run or not _ConfirmDeploy(num_updates):
1272 return
1273
Ralph Nathane01ccf12015-04-16 10:40:32 -07001274 # Select function (emerge or unmerge) and bind args.
1275 if emerge:
Mike Frysinger539db512015-05-21 18:14:01 -04001276 func = functools.partial(_EmergePackages, pkgs, device, strip,
Andrew06a5f812020-01-23 08:08:32 -08001277 sysroot, root, board, emerge_args)
Ralph Nathane01ccf12015-04-16 10:40:32 -07001278 else:
Xiaochu Liu2726e7c2019-07-18 10:28:10 -07001279 func = functools.partial(_UnmergePackages, pkgs, device, root,
1280 pkgs_attrs)
Ralph Nathane01ccf12015-04-16 10:40:32 -07001281
1282 # Call the function with the progress bar or with normal output.
1283 if command.UseProgressBar():
Mike Frysinger63d35512021-01-26 23:16:13 -05001284 op = BrilloDeployOperation(emerge)
Ralph Nathane01ccf12015-04-16 10:40:32 -07001285 op.Run(func, log_level=logging.DEBUG)
1286 else:
1287 func()
David Pursell9476bf42015-03-30 13:34:27 -07001288
Ben Pastene5f03b052019-08-12 18:03:24 -07001289 if device.IsSELinuxAvailable():
Qijiang Fan8a945032019-04-25 20:53:29 +09001290 if sum(x.count('selinux-policy') for x in pkgs):
1291 logging.warning(
1292 'Deploying SELinux policy will not take effect until reboot. '
Ian Barkley-Yeung6b2d8672020-08-13 18:58:10 -07001293 'SELinux policy is loaded by init. Also, changing the security '
1294 'contexts (labels) of a file will require building a new image '
1295 'and flashing the image onto the device.')
Qijiang Fan352d0eb2019-02-25 13:10:08 +09001296
Mike Frysinger63d35512021-01-26 23:16:13 -05001297 # This message is read by BrilloDeployOperation.
David Pursell9476bf42015-03-30 13:34:27 -07001298 logging.warning('Please restart any updated services on the device, '
1299 'or just reboot it.')
Gilad Arnold4d3ade72015-04-28 15:13:35 -07001300 except Exception:
1301 if lsb_release:
1302 lsb_entries = sorted(lsb_release.items())
1303 logging.info('Following are the LSB version details of the device:\n%s',
1304 '\n'.join('%s=%s' % (k, v) for k, v in lsb_entries))
1305 raise