blob: cc1bde493fe50caf58c15d41e50a5a1c14243a38 [file] [log] [blame]
Amin Hassani92f6c4a2021-02-20 17:36:09 -08001# Copyright 2021 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Library containing functions to install an image on a Chromium OS device."""
6
Amin Hassanid4b3ff82021-02-20 23:05:14 -08007import abc
Amin Hassani92f6c4a2021-02-20 17:36:09 -08008import enum
9import os
10import re
11import sys
Amin Hassanid4b3ff82021-02-20 23:05:14 -080012import tempfile
13import threading
Amin Hassani55970562021-02-22 20:49:13 -080014import time
Amin Hassani92f6c4a2021-02-20 17:36:09 -080015from typing import Tuple, Dict
16
Amin Hassani55970562021-02-22 20:49:13 -080017from chromite.cli import command
Amin Hassanicf8f0042021-03-12 10:42:13 -080018from chromite.cli import flash
Amin Hassanid4b3ff82021-02-20 23:05:14 -080019from chromite.lib import constants
Amin Hassani92f6c4a2021-02-20 17:36:09 -080020from chromite.lib import cros_build_lib
21from chromite.lib import cros_logging as logging
22from chromite.lib import gs
Amin Hassanid4b3ff82021-02-20 23:05:14 -080023from chromite.lib import image_lib
Amin Hassani55970562021-02-22 20:49:13 -080024from chromite.lib import operation
Amin Hassanid4b3ff82021-02-20 23:05:14 -080025from chromite.lib import osutils
Amin Hassani92f6c4a2021-02-20 17:36:09 -080026from chromite.lib import parallel
27from chromite.lib import remote_access
28from chromite.lib import retry_util
Amin Hassani74403082021-02-22 11:40:09 -080029from chromite.lib import stateful_updater
Amin Hassani75c5f942021-02-20 23:56:53 -080030from chromite.lib.paygen import partition_lib
Amin Hassani74403082021-02-22 11:40:09 -080031from chromite.lib.paygen import paygen_stateful_payload_lib
Amin Hassani92f6c4a2021-02-20 17:36:09 -080032from chromite.lib.xbuddy import devserver_constants
33from chromite.lib.xbuddy import xbuddy
34
35
36assert sys.version_info >= (3, 6), 'This module requires Python 3.6+'
37
38
39class Error(Exception):
40 """Thrown when there is a general Chromium OS-specific flash error."""
41
42
43class ImageType(enum.Enum):
44 """Type of the image that is used for flashing the device."""
45
46 # The full image on disk (e.g. chromiumos_test_image.bin).
47 FULL = 0
48 # The remote directory path
49 # (e.g gs://chromeos-image-archive/eve-release/R90-x.x.x)
50 REMOTE_DIRECTORY = 1
51
52
53class Partition(enum.Enum):
54 """An enum for partition types like kernel and rootfs."""
55 KERNEL = 0
56 ROOTFS = 1
57
58
59class DeviceImager(object):
60 """A class to flash a Chromium OS device.
61
62 This utility uses parallelism as much as possible to achieve its goal as fast
63 as possible. For example, it uses parallel compressors, parallel transfers,
64 and simultaneous pipes.
65 """
66
67 # The parameters of the kernel and rootfs's two main partitions.
68 A = {Partition.KERNEL: 2, Partition.ROOTFS: 3}
69 B = {Partition.KERNEL: 4, Partition.ROOTFS: 5}
70
71 def __init__(self, device, image: str,
Amin Hassanicf8f0042021-03-12 10:42:13 -080072 board: str = None,
73 version: str = None,
Amin Hassani92f6c4a2021-02-20 17:36:09 -080074 no_rootfs_update: bool = False,
75 no_stateful_update: bool = False,
76 no_reboot: bool = False,
77 disable_verification: bool = False,
78 clobber_stateful: bool = False,
79 clear_tpm_owner: bool = False):
80 """Initialize DeviceImager for flashing a Chromium OS device.
81
82 Args:
83 device: The ChromiumOSDevice to be updated.
84 image: The target image path (can be xBuddy path).
Amin Hassanicf8f0042021-03-12 10:42:13 -080085 board: Board to use.
86 version: Image version to use.
Amin Hassani92f6c4a2021-02-20 17:36:09 -080087 no_rootfs_update: Whether to do rootfs partition update.
88 no_stateful_update: Whether to do stateful partition update.
89 no_reboot: Whether to reboot device after update. The default is True.
90 disable_verification: Whether to disabling rootfs verification on the
91 device.
92 clobber_stateful: Whether to do a clean stateful partition.
93 clear_tpm_owner: If true, it will clear the TPM owner on reboot.
94 """
95
96 self._device = device
97 self._image = image
Amin Hassanicf8f0042021-03-12 10:42:13 -080098 self._board = board
99 self._version = version
Amin Hassani92f6c4a2021-02-20 17:36:09 -0800100 self._no_rootfs_update = no_rootfs_update
101 self._no_stateful_update = no_stateful_update
102 self._no_reboot = no_reboot
103 self._disable_verification = disable_verification
104 self._clobber_stateful = clobber_stateful
105 self._clear_tpm_owner = clear_tpm_owner
106
Amin Hassani0972d392021-03-31 19:04:19 -0700107 self._compression = cros_build_lib.COMP_GZIP
Amin Hassani92f6c4a2021-02-20 17:36:09 -0800108 self._inactive_state = None
109
110 def Run(self):
111 """Update the device with image of specific version."""
112
113 try:
Amin Hassani55970562021-02-22 20:49:13 -0800114 if command.UseProgressBar():
115 op = DeviceImagerOperation()
116 op.Run(self._Run)
117 else:
118 self._Run()
Amin Hassani92f6c4a2021-02-20 17:36:09 -0800119 except Exception as e:
120 raise Error(f'DeviceImager Failed with error: {e}')
121
Amin Hassani55970562021-02-22 20:49:13 -0800122 # DeviceImagerOperation will look for this log.
Amin Hassani92f6c4a2021-02-20 17:36:09 -0800123 logging.info('DeviceImager completed.')
124
125 def _Run(self):
126 """Runs the various operations to install the image on device."""
127 image, image_type = self._GetImage()
128 logging.info('Using image %s of type %s', image, image_type )
129
130 if image_type == ImageType.REMOTE_DIRECTORY:
131 self._compression = cros_build_lib.COMP_GZIP
132
133 self._InstallPartitions(image, image_type)
134
135 if self._clear_tpm_owner:
136 self._device.ClearTpmOwner()
137
138 if not self._no_reboot:
139 self._Reboot()
140 self._VerifyBootExpectations()
141
142 if self._disable_verification:
143 self._device.DisableRootfsVerification()
144
145 def _GetImage(self) -> Tuple[str, ImageType]:
146 """Returns the path to the final image(s) that need to be installed.
147
148 If the paths is local, the image should be the Chromium OS GPT image
149 (e.g. chromiumos_test_image.bin). If the path is remote, it should be the
150 remote directory where we can find the quick-provision and stateful update
151 files (e.g. gs://chromeos-image-archive/eve-release/R90-x.x.x).
152
153 NOTE: At this point there is no caching involved. Hence we always download
154 the partition payloads or extract them from the Chromium OS image.
155
156 Returns:
157 A tuple of image path and image type.
158 """
159 if os.path.isfile(self._image):
160 return self._image, ImageType.FULL
161
162 # TODO(b/172212406): We could potentially also allow this by searching
163 # through the directory to see whether we have quick-provision and stateful
164 # payloads. This only makes sense when a user has their workstation at home
165 # and doesn't want to incur the bandwidth cost of downloading the same
166 # image multiple times. For that, they can simply download the GPT image
167 # image first and flash that instead.
168 if os.path.isdir(self._image):
169 raise ValueError(
170 f'{self._image}: input must be a disk image, not a directory.')
171
172 if gs.PathIsGs(self._image):
173 # TODO(b/172212406): Check whether it is a directory. If it wasn't a
174 # directory download the image into some temp location and use it instead.
175 return self._image, ImageType.REMOTE_DIRECTORY
176
177 # Assuming it is an xBuddy path.
Amin Hassanicf8f0042021-03-12 10:42:13 -0800178 board = cros_build_lib.GetBoard(
179 device_board=self._device.board or flash.GetDefaultBoard(),
180 override_board=self._board, force=True)
181
Amin Hassania20d20d2021-04-28 10:18:18 -0700182 xb = xbuddy.XBuddy(board=board, version=self._version)
Amin Hassani92f6c4a2021-02-20 17:36:09 -0800183 build_id, local_file = xb.Translate([self._image])
184 if build_id is None:
185 raise Error(f'{self._image}: unable to find matching xBuddy path.')
186 logging.info('XBuddy path translated to build ID %s', build_id)
187
188 if local_file:
189 return local_file, ImageType.FULL
190
191 return (f'{devserver_constants.GS_IMAGE_DIR}/{build_id}',
192 ImageType.REMOTE_DIRECTORY)
193
194 def _SplitDevPath(self, path: str) -> Tuple[str, int]:
195 """Splits the given /dev/x path into prefix and the dev number.
196
197 Args:
198 path: The path to a block dev device.
199
200 Returns:
201 A tuple of representing the prefix and the index of the dev path.
202 e.g.: '/dev/mmcblk0p1' -> ['/dev/mmcblk0p', 1]
203 """
204 match = re.search(r'(.*)([0-9]+)$', path)
205 if match is None:
206 raise Error(f'{path}: Could not parse root dev path.')
207
208 return match.group(1), int(match.group(2))
209
210 def _GetKernelState(self, root_num: int) -> Tuple[Dict, Dict]:
211 """Returns the kernel state.
212
213 Returns:
214 A tuple of two dictionaries: The current active kernel state and the
215 inactive kernel state. (Look at A and B constants in this class.)
216 """
217 if root_num == self.A[Partition.ROOTFS]:
218 return self.A, self.B
219 elif root_num == self.B[Partition.ROOTFS]:
220 return self.B, self.A
221 else:
222 raise Error(f'Invalid root partition number {root_num}')
223
224 def _InstallPartitions(self, image: str, image_type):
225 """The main method that installs the partitions of a Chrome OS device.
226
227 It uses parallelism to install the partitions as fast as possible.
228
229 Args:
230 image: The image path (local file or remote directory).
231 image_type: The type of the image (ImageType).
232 """
Amin Hassanid684e982021-02-26 11:10:58 -0800233 prefix, root_num = self._SplitDevPath(self._device.root_dev)
234 active_state, self._inactive_state = self._GetKernelState(root_num)
235
Amin Hassani92f6c4a2021-02-20 17:36:09 -0800236 updaters = []
Amin Hassanid684e982021-02-26 11:10:58 -0800237 if not self._no_rootfs_update:
Amin Hassani75c5f942021-02-20 23:56:53 -0800238 current_root = prefix + str(active_state[Partition.ROOTFS])
239 target_root = prefix + str(self._inactive_state[Partition.ROOTFS])
240 updaters.append(RootfsUpdater(current_root, self._device, image,
241 image_type, target_root, self._compression))
242
Amin Hassanid684e982021-02-26 11:10:58 -0800243 target_kernel = prefix + str(self._inactive_state[Partition.KERNEL])
244 updaters.append(KernelUpdater(self._device, image, image_type,
245 target_kernel, self._compression))
Amin Hassani92f6c4a2021-02-20 17:36:09 -0800246
Amin Hassani74403082021-02-22 11:40:09 -0800247 if not self._no_stateful_update:
248 updaters.append(StatefulUpdater(self._clobber_stateful, self._device,
249 image, image_type, None, None))
250
Amin Hassani92f6c4a2021-02-20 17:36:09 -0800251 # Retry the partitions updates that failed, in case a transient error (like
252 # SSH drop, etc) caused the error.
253 num_retries = 1
254 try:
255 retry_util.RetryException(Error, num_retries,
256 parallel.RunParallelSteps,
257 (x.Run for x in updaters if not x.IsFinished()),
258 halt_on_error=True)
259 except Exception:
260 # If one of the partitions failed to be installed, revert all partitions.
261 parallel.RunParallelSteps(x.Revert for x in updaters)
262 raise
263
264 def _Reboot(self):
265 """Reboots the device."""
266 try:
Amin Hassani9281b682021-03-08 16:38:25 -0800267 self._device.Reboot(timeout_sec=300)
Amin Hassani92f6c4a2021-02-20 17:36:09 -0800268 except remote_access.RebootError:
269 raise Error('Could not recover from reboot. Once example reason'
270 ' could be the image provided was a non-test image'
271 ' or the system failed to boot after the update.')
272 except Exception as e:
273 raise Error(f'Failed to reboot to the device with error: {e}')
274
275 def _VerifyBootExpectations(self):
276 """Verify that we fully booted into the expected kernel state."""
277 # Discover the newly active kernel.
278 _, root_num = self._SplitDevPath(self._device.root_dev)
279 active_state, _ = self._GetKernelState(root_num)
280
281 # If this happens, we should rollback.
282 if active_state != self._inactive_state:
283 raise Error('The expected kernel state after update is invalid.')
284
285 logging.info('Verified boot expectations.')
Amin Hassanid4b3ff82021-02-20 23:05:14 -0800286
287
288class ReaderBase(threading.Thread):
289 """The base class for reading different inputs and writing into output.
290
291 This class extends threading.Thread, so it will be run on its own thread. Also
292 it can be used as a context manager. Internally, it opens necessary files for
293 writing to and reading from. This class cannot be instantiated, it needs to be
294 sub-classed first to provide necessary function implementations.
295 """
296
297 def __init__(self, use_named_pipes: bool = False):
298 """Initializes the class.
299
300 Args:
301 use_named_pipes: Whether to use a named pipe or anonymous file
302 descriptors.
303 """
304 super().__init__()
305 self._use_named_pipes = use_named_pipes
306 self._pipe_target = None
307 self._pipe_source = None
308
309 def __del__(self):
310 """Destructor.
311
312 Make sure to clean up any named pipes we might have created.
313 """
314 if self._use_named_pipes:
315 osutils.SafeUnlink(self._pipe_target)
316
317 def __enter__(self):
318 """Enters the context manager"""
319 if self._use_named_pipes:
320 # There is no need for the temp file, we only need its path. So the named
321 # pipe is created after this temp file is deleted.
322 with tempfile.NamedTemporaryFile(prefix='chromite-device-imager') as fp:
323 self._pipe_target = self._pipe_source = fp.name
324 os.mkfifo(self._pipe_target)
325 else:
326 self._pipe_target, self._pipe_source = os.pipe()
327
328 self.start()
329 return self
330
331 def __exit__(self, *args, **kwargs):
332 """Exits the context manager."""
333 self.join()
334
335 def _Source(self):
336 """Returns the source pipe to write data into.
337
338 Sub-classes can use this function to determine where to write their data
339 into.
340 """
341 return self._pipe_source
342
343 def _CloseSource(self):
344 """Closes the source pipe.
345
346 Sub-classes should use this function to close the pipe after they are done
347 writing into it. Failure to do so may result reader of the data to hang
348 indefinitely.
349 """
350 if not self._use_named_pipes:
351 os.close(self._pipe_source)
352
353 def Target(self):
354 """Returns the target pipe to read data from.
355
356 Users of this class can use this path to read data from.
357 """
358 return self._pipe_target
359
360 def CloseTarget(self):
361 """Closes the target pipe.
362
363 Users of this class should use this function to close the pipe after they
364 are done reading from it.
365 """
366 if self._use_named_pipes:
367 os.remove(self._pipe_target)
368 else:
369 os.close(self._pipe_target)
370
371
372class PartialFileReader(ReaderBase):
373 """A class to read specific offset and length from a file and compress it.
374
375 This class can be used to read from specific location and length in a file
376 (e.g. A partition in a GPT image). Then it compresses the input and writes it
377 out (to a pipe). Look at the base class for more information.
378 """
379
380 # The offset of different partitions in a Chromium OS image does not always
381 # align to larger values like 4096. It seems that 512 is the maximum value to
382 # be divisible by partition offsets. This size should not be increased just
383 # for 'performance reasons'. Since we are doing everything in parallel, in
384 # practice there is not much difference between this and larger block sizes as
385 # parallelism hides the possible extra latency provided by smaller block
386 # sizes.
387 _BLOCK_SIZE = 512
388
389 def __init__(self, image: str, offset: int, length: int, compression):
390 """Initializes the class.
391
392 Args:
393 image: The path to an image (local or remote directory).
394 offset: The offset (in bytes) to read from the image.
395 length: The length (in bytes) to read from the image.
396 compression: The compression type (see cros_build_lib.COMP_XXX).
397 """
398 super().__init__()
399
400 self._image = image
401 self._offset = offset
402 self._length = length
403 self._compression = compression
404
405 def run(self):
406 """Runs the reading and compression."""
407 cmd = [
408 'dd',
409 'status=none',
410 f'if={self._image}',
411 f'ibs={self._BLOCK_SIZE}',
412 f'skip={int(self._offset/self._BLOCK_SIZE)}',
413 f'count={int(self._length/self._BLOCK_SIZE)}',
414 '|',
415 cros_build_lib.FindCompressor(self._compression),
416 ]
417
418 try:
419 cros_build_lib.run(' '.join(cmd), stdout=self._Source(), shell=True)
420 finally:
421 self._CloseSource()
422
423
Amin Hassani0fe49ae2021-02-21 23:41:58 -0800424class GsFileCopier(ReaderBase):
425 """A class for downloading gzip compressed file from GS bucket into a pipe."""
426
427 def __init__(self, image: str):
428 """Initializes the class.
429
430 Args:
431 image: The path to an image (local or remote directory).
432 """
433 super().__init__(use_named_pipes=True)
434 self._image = image
435
436 def run(self):
437 """Runs the download and write into the output pipe."""
438 try:
439 gs.GSContext().Copy(self._image, self._Source())
440 finally:
441 self._CloseSource()
442
443
Amin Hassanid4b3ff82021-02-20 23:05:14 -0800444class PartitionUpdaterBase(object):
445 """A base abstract class to use for installing an image into a partition.
446
447 Sub-classes should implement the abstract methods to provide the core
448 functionality.
449 """
450 def __init__(self, device, image: str, image_type, target: str, compression):
451 """Initializes this base class with values that most sub-classes will need.
452
453 Args:
454 device: The ChromiumOSDevice to be updated.
455 image: The target image path for the partition update.
456 image_type: The type of the image (ImageType).
457 target: The target path (e.g. block dev) to install the update.
458 compression: The compression used for compressing the update payload.
459 """
460 self._device = device
461 self._image = image
462 self._image_type = image_type
463 self._target = target
464 self._compression = compression
465 self._finished = False
466
467 def Run(self):
468 """The main function that does the partition update job."""
469 with cros_build_lib.TimedSection() as timer:
470 try:
471 self._Run()
472 finally:
473 self._finished = True
474
475 logging.debug('Completed %s in %s', self.__class__.__name__, timer.delta)
476
477 @abc.abstractmethod
478 def _Run(self):
479 """The method that need to be implemented by sub-classes."""
480 raise NotImplementedError('Sub-classes need to implement this.')
481
482 def IsFinished(self):
483 """Returns whether the partition update has been successful."""
484 return self._finished
485
486 @abc.abstractmethod
487 def Revert(self):
488 """Reverts the partition update.
489
490 Sub-classes need to implement this function to provide revert capability.
491 """
492 raise NotImplementedError('Sub-classes need to implement this.')
493
494
495class RawPartitionUpdater(PartitionUpdaterBase):
496 """A class to update a raw partition on a Chromium OS device."""
497
498 def _Run(self):
499 """The function that does the job of kernel partition update."""
500 if self._image_type == ImageType.FULL:
501 self._CopyPartitionFromImage(self._GetPartitionName())
502 elif self._image_type == ImageType.REMOTE_DIRECTORY:
Amin Hassani0fe49ae2021-02-21 23:41:58 -0800503 self._RedirectPartition(self._GetRemotePartitionName())
Amin Hassanid4b3ff82021-02-20 23:05:14 -0800504 else:
505 raise ValueError(f'Invalid image type {self._image_type}')
506
507 def _GetPartitionName(self):
508 """Returns the name of the partition in a Chromium OS GPT layout.
509
510 Subclasses should override this function to return correct name.
511 """
512 raise NotImplementedError('Subclasses need to implement this.')
513
514 def _CopyPartitionFromImage(self, part_name: str):
515 """Updates the device's partition from a local Chromium OS image.
516
517 Args:
518 part_name: The name of the partition in the source image that needs to be
519 extracted.
520 """
521 cmd = self._GetWriteToTargetCommand()
522
523 offset, length = self._GetPartLocation(part_name)
524 offset, length = self._OptimizePartLocation(offset, length)
525 with PartialFileReader(self._image, offset, length,
526 self._compression) as generator:
527 try:
528 self._device.run(cmd, input=generator.Target(), shell=True)
529 finally:
530 generator.CloseTarget()
531
532 def _GetWriteToTargetCommand(self):
533 """Returns a write to target command to run on a Chromium OS device.
534
535 Returns:
536 A string command to run on a device to read data from stdin, uncompress it
537 and write it to the target partition.
538 """
539 cmd = self._device.GetDecompressor(self._compression)
540 # Using oflag=direct to tell the OS not to cache the writes (faster).
541 cmd += ['|', 'dd', 'bs=1M', 'oflag=direct', f'of={self._target}']
542 return ' '.join(cmd)
543
544 def _GetPartLocation(self, part_name: str):
545 """Extracts the location and size of the kernel partition from the image.
546
547 Args:
548 part_name: The name of the partition in the source image that needs to be
549 extracted.
550
551 Returns:
552 A tuple of offset and length (in bytes) from the image.
553 """
554 try:
555 parts = image_lib.GetImageDiskPartitionInfo(self._image)
556 part_info = [p for p in parts if p.name == part_name][0]
557 except IndexError:
558 raise Error(f'No partition named {part_name} found.')
559
560 return int(part_info.start), int(part_info.size)
561
Amin Hassani0fe49ae2021-02-21 23:41:58 -0800562 def _GetRemotePartitionName(self):
563 """Returns the name of the quick-provision partition file.
564
565 Subclasses should override this function to return correct name.
566 """
567 raise NotImplementedError('Subclasses need to implement this.')
568
Amin Hassanid4b3ff82021-02-20 23:05:14 -0800569 def _OptimizePartLocation(self, offset: int, length: int):
570 """Optimizes the offset and length of the partition.
571
572 Subclasses can override this to provide better offset/length than what is
573 defined in the PGT partition layout.
574
575 Args:
576 offset: The offset (in bytes) of the partition in the image.
577 length: The length (in bytes) of the partition.
578
579 Returns:
580 A tuple of offset and length (in bytes) from the image.
581 """
582 return offset, length
Amin Hassanid684e982021-02-26 11:10:58 -0800583
Amin Hassani0fe49ae2021-02-21 23:41:58 -0800584 def _RedirectPartition(self, file_name: str):
585 """Downloads the partition from a remote path and writes it into target.
586
587 Args:
588 file_name: The file name in the remote directory self._image.
589 """
590 cmd = self._GetWriteToTargetCommand()
591
592 image_path = os.path.join(self._image, file_name)
593 with GsFileCopier(image_path) as generator:
594 try:
595 with open(generator.Target(), 'rb') as fp:
596 self._device.run(cmd, input=fp, shell=True)
597 finally:
598 generator.CloseTarget()
599
Amin Hassanid684e982021-02-26 11:10:58 -0800600
601class KernelUpdater(RawPartitionUpdater):
602 """A class to update the kernel partition on a Chromium OS device."""
603
604 def _GetPartitionName(self):
605 """See RawPartitionUpdater._GetPartitionName()."""
606 return constants.PART_KERN_B
607
Amin Hassani0fe49ae2021-02-21 23:41:58 -0800608 def _GetRemotePartitionName(self):
609 """See RawPartitionUpdater._GetRemotePartitionName()."""
610 return constants.QUICK_PROVISION_PAYLOAD_KERNEL
611
Amin Hassanid684e982021-02-26 11:10:58 -0800612 def Revert(self):
613 """Reverts the kernel partition update."""
614 # There is nothing to do for reverting kernel partition.
Amin Hassani75c5f942021-02-20 23:56:53 -0800615
616
617class RootfsUpdater(RawPartitionUpdater):
618 """A class to update the root partition on a Chromium OS device."""
619
620 def __init__(self, current_root: str, *args):
621 """Initializes the class.
622
623 Args:
624 current_root: The current root device path.
625 *args: See PartitionUpdaterBase
626 """
627 super().__init__(*args)
628
629 self._current_root = current_root
630 self._ran_postinst = False
631
632 def _GetPartitionName(self):
633 """See RawPartitionUpdater._GetPartitionName()."""
634 return constants.PART_ROOT_A
635
Amin Hassani0fe49ae2021-02-21 23:41:58 -0800636 def _GetRemotePartitionName(self):
637 """See RawPartitionUpdater._GetRemotePartitionName()."""
638 return constants.QUICK_PROVISION_PAYLOAD_ROOTFS
639
Amin Hassani75c5f942021-02-20 23:56:53 -0800640 def _Run(self):
641 """The function that does the job of rootfs partition update."""
Amin Hassani55970562021-02-22 20:49:13 -0800642 with ProgressWatcher(self._device, self._target):
643 super()._Run()
Amin Hassani75c5f942021-02-20 23:56:53 -0800644
645 self._RunPostInst()
646
647 def _OptimizePartLocation(self, offset: int, length: int):
648 """Optimizes the size of the root partition of the image.
649
650 Normally the file system does not occupy the entire partition. Furthermore
651 we don't need the verity hash tree at the end of the root file system
652 because postinst will recreate it. This function reads the (approximate)
653 superblock of the ext4 partition and extracts the actual file system size in
654 the root partition.
655 """
656 superblock_size = 4096 * 2
657 with open(self._image, 'rb') as r:
658 r.seek(offset)
659 with tempfile.NamedTemporaryFile(delete=False) as fp:
660 fp.write(r.read(superblock_size))
661 fp.close()
662 return offset, partition_lib.Ext2FileSystemSize(fp.name)
663
664 def _RunPostInst(self, on_target: bool = True):
665 """Runs the postinst process in the root partition.
666
667 Args:
668 on_target: If true the postinst is run on the target (inactive)
669 partition. This is used when doing normal updates. If false, the
670 postinst is run on the current (active) partition. This is used when
671 reverting an update.
672 """
673 try:
674 postinst_dir = '/'
675 partition = self._current_root
676 if on_target:
677 postinst_dir = self._device.run(
678 ['mktemp', '-d', '-p', self._device.work_dir],
679 capture_output=True).stdout.strip()
680 self._device.run(['mount', '-o', 'ro', self._target, postinst_dir])
681 partition = self._target
682
683 self._ran_postinst = True
684 postinst = os.path.join(postinst_dir, 'postinst')
685 result = self._device.run([postinst, partition], capture_output=True)
686
687 logging.debug('Postinst result on %s: \n%s', postinst, result.stdout)
Amin Hassani55970562021-02-22 20:49:13 -0800688 # DeviceImagerOperation will look for this log.
Amin Hassani75c5f942021-02-20 23:56:53 -0800689 logging.info('Postinstall completed.')
690 finally:
691 if on_target:
692 self._device.run(['umount', postinst_dir])
693
694 def Revert(self):
695 """Reverts the root update install."""
696 logging.info('Reverting the rootfs partition update.')
697 if self._ran_postinst:
698 # We don't have to do anything for revert if we haven't changed the kernel
699 # priorities yet.
700 self._RunPostInst(on_target=False)
Amin Hassani74403082021-02-22 11:40:09 -0800701
702
703class StatefulPayloadGenerator(ReaderBase):
704 """A class for generating a stateful update payload in a separate thread."""
705 def __init__(self, image: str):
706 """Initializes that class.
707
708 Args:
709 image: The path to a local Chromium OS image.
710 """
711 super().__init__()
712 self._image = image
713
714 def run(self):
715 """Generates the stateful update and writes it into the output pipe."""
716 try:
717 paygen_stateful_payload_lib.GenerateStatefulPayload(
718 self._image, self._Source())
719 finally:
720 self._CloseSource()
721
722
723class StatefulUpdater(PartitionUpdaterBase):
724 """A class to update the stateful partition on a device."""
725 def __init__(self, clobber_stateful: bool, *args):
726 """Initializes the class
727
728 Args:
729 clobber_stateful: Whether to clobber the stateful or not.
730 *args: Look at PartitionUpdaterBase.
731 """
732 super().__init__(*args)
733 self._clobber_stateful = clobber_stateful
734
735 def _Run(self):
736 """Reads/Downloads the stateful updates and writes it into the device."""
737 if self._image_type == ImageType.FULL:
738 generator_cls = StatefulPayloadGenerator
739 elif self._image_type == ImageType.REMOTE_DIRECTORY:
740 generator_cls = GsFileCopier
741 self._image = os.path.join(self._image,
742 paygen_stateful_payload_lib.STATEFUL_FILE)
743 else:
744 raise ValueError(f'Invalid image type {self._image_type}')
745
746 with generator_cls(self._image) as generator:
747 try:
748 updater = stateful_updater.StatefulUpdater(self._device)
749 updater.Update(
750 generator.Target(),
751 is_payload_on_device=False,
752 update_type=(stateful_updater.StatefulUpdater.UPDATE_TYPE_CLOBBER if
753 self._clobber_stateful else None))
754 finally:
755 generator.CloseTarget()
756
757 def Revert(self):
758 """Reverts the stateful partition update."""
759 logging.info('Reverting the stateful update.')
760 stateful_updater.StatefulUpdater(self._device).Reset()
Amin Hassani55970562021-02-22 20:49:13 -0800761
762
763class ProgressWatcher(threading.Thread):
764 """A class used for watching the progress of rootfs update."""
765
766 def __init__(self, device, target_root: str):
767 """Initializes the class.
768
769 Args:
770 device: The ChromiumOSDevice to be updated.
771 target_root: The target root partition to monitor the progress of.
772 """
773 super().__init__()
774
775 self._device = device
776 self._target_root = target_root
777 self._exit = False
778
779 def __enter__(self):
780 """Starts the thread."""
781 self.start()
782 return self
783
784 def __exit__(self, *args, **kwargs):
785 """Exists the thread."""
786 self._exit = True
787 self.join()
788
789 def _ShouldExit(self):
790 return self._exit
791
792 def run(self):
793 """Monitors the progress of the target root partitions' update.
794
795 This is done by periodically, reading the fd position of the process that is
796 writing into the target partition and reporting it back. Then the position
797 is divided by the size of the block device to report an approximate
798 progress.
799 """
800 cmd = ['blockdev', '--getsize64', self._target_root]
801 output = self._device.run(cmd, capture_output=True).stdout.strip()
802 if output is None:
803 raise Error(f'Cannot get the block device size from {output}.')
804 dev_size = int(output)
805
806 # Using lsof to find out which process is writing to the target rootfs.
807 cmd = f'lsof 2>/dev/null | grep {self._target_root}'
808 while not self._ShouldExit():
809 try:
810 output = self._device.run(cmd, capture_output=True,
811 shell=True).stdout.strip()
812 if output:
813 break
814 except cros_build_lib.RunCommandError:
815 continue
816 finally:
817 time.sleep(1)
818
819 # Now that we know which process is writing to it, we can look the fdinfo of
820 # stdout of that process to get its offset. We're assuming there will be no
821 # seek, which is correct.
822 pid = output.split()[1]
823 cmd = ['cat', f'/proc/{pid}/fdinfo/1']
824 while not self._ShouldExit():
825 try:
826 output = self._device.run(cmd, capture_output=True).stdout.strip()
827 m = re.search(r'^pos:\s*(\d+)$', output, flags=re.M)
828 if m:
829 offset = int(m.group(1))
830 # DeviceImagerOperation will look for this log.
831 logging.info('RootFS progress: %f', offset/dev_size)
832 except cros_build_lib.RunCommandError:
833 continue
834 finally:
835 time.sleep(1)
836
837
838class DeviceImagerOperation(operation.ProgressBarOperation):
839 """A class to provide a progress bar for DeviceImager operation."""
840
841 def __init__(self):
842 """Initializes the class."""
843 super().__init__()
844
845 self._progress = 0.0
846
847 def ParseOutput(self, output=None):
848 """Override function to parse the output and provide progress.
849
850 Args:
851 output: The stderr or stdout.
852 """
853 output = self._stdout.read()
854 match = re.findall(r'RootFS progress: (\d+(?:\.\d+)?)', output)
855 if match:
856 progress = float(match[0])
857 self._progress = max(self._progress, progress)
858
859 # If postinstall completes, move half of the remaining progress.
860 if re.findall(r'Postinstall completed', output):
861 self._progress += (1.0 - self._progress) / 2
862
863 # While waiting for reboot, each time, move half of the remaining progress.
864 if re.findall(r'Unable to get new boot_id', output):
865 self._progress += (1.0 - self._progress) / 2
866
867 if re.findall(r'DeviceImager completed.', output):
868 self._progress = 1.0
869
870 self.ProgressBar(self._progress)