cros_update.py

# Copyright 2016 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""An executable function cros-update for auto-update of a CrOS host.

The reason to create this file is to let devserver to trigger a background
process for CrOS auto-update. Therefore, when devserver service is restarted
sometimes, the CrOS auto-update process is still running and the corresponding
provision task won't claim failure.

It includes two classes:
  a. CrOSUpdateTrigger:
    1. Includes all logics which identify which types of update need to be
       performed in the current DUT.
    2. Responsible for write current status of CrOS auto-update process into
       progress_tracker.

  b. CrOSAUParser:
    1. Pre-setup the required args for CrOS auto-update.
    2. Parse the input parameters for cmd that runs 'cros_update.py'.
"""

from __future__ import print_function

import argparse
import cros_update_logging
import cros_update_progress
import logging # pylint: disable=cros-logging-import
import os
import re
import sys
import time
import traceback

# only import setup_chromite before chromite import.
import setup_chromite # pylint: disable=unused-import
try:
  from chromite.lib import auto_updater
  from chromite.lib import cros_build_lib
  from chromite.lib import remote_access
  from chromite.lib import timeout_util
except ImportError as e:
  logging.debug('chromite cannot be imported: %r', e)
  auto_updater = None
  remote_access = None
  timeout_util = None

# The build channel for recovering host's stateful partition
STABLE_BUILD_CHANNEL = 'stable-channel'

# Timeout for CrOS auto-update process.
CROS_UPDATE_TIMEOUT_MIN = 30

# The preserved path in remote device, won't be deleted after rebooting.
CROS_PRESERVED_PATH = ('/mnt/stateful_partition/unencrypted/'
                       'preserve/cros-update')

# Standard error tmeplate to be written into status tracking log.
CROS_ERROR_TEMPLATE = cros_update_progress.ERROR_TAG + ' %s'

# How long after a quick provision fails to wait before falling back to the
# standard provisioning flow.
QUICK_PROVISION_FAILURE_DELAY_SEC = 45

# Setting logging level
logConfig = cros_update_logging.loggingConfig()
logConfig.ConfigureLogging()

class CrOSAUParser(object):
  """Custom command-line options parser for cros-update."""
  def __init__(self):
    self.args = sys.argv[1:]
    self.parser = argparse.ArgumentParser(
        usage='%(prog)s [options] [control-file]')
    self.SetupOptions()
    self.removed_args = []

    # parse an empty list of arguments in order to set self.options
    # to default values.
    self.options = self.parser.parse_args(args=[])

  def SetupOptions(self):
    """Setup options to call cros-update command."""
    self.parser.add_argument('-d', action='store', type=str,
                             dest='host_name',
                             help='host_name of a DUT')
    self.parser.add_argument('-b', action='store', type=str,
                             dest='build_name',
                             help='build name to be auto-updated')
    self.parser.add_argument('--static_dir', action='store', type=str,
                             help='static directory of the devserver')
    self.parser.add_argument('--force_update', action='store_true',
                             default=False,
                             help=('force an update even if the version '
                                   'installed is the same'))
    self.parser.add_argument('--full_update', action='store_true',
                             default=False,
                             help='force a rootfs update, skip stateful update')
    self.parser.add_argument('--original_build', action='store', type=str,
                             default='',
                             help=('force stateful update with the same '
                                   'version of previous rootfs partition'))
    self.parser.add_argument('--payload_filename', action='store', type=str,
                             default=None, help='A custom payload filename')
    self.parser.add_argument('--clobber_stateful', action='store_true',
                             default=False, help='Whether to clobber stateful')
    self.parser.add_argument('--quick_provision', action='store_true',
                             default=False,
                             help='Whether to attempt quick provisioning path')
    self.parser.add_argument('--devserver_url', action='store', type=str,
                             default=None, help='Devserver URL base for RPCs')
    self.parser.add_argument('--static_url', action='store', type=str,
                             default=None,
                             help='Devserver URL base for static files')

  def ParseArgs(self):
    """Parse and process command line arguments."""
    # Positional arguments from the end of the command line will be included
    # in the list of unknown_args.
    self.options, unknown_args = self.parser.parse_known_args()
    # Filter out none-positional arguments
    while unknown_args and unknown_args[0][0] == '-':
      self.removed_args.append(unknown_args.pop(0))
      # Always assume the argument has a value.
      if unknown_args:
        self.removed_args.append(unknown_args.pop(0))
    if self.removed_args:
      logging.warn('Unknown arguments are removed from the options: %s',
                   self.removed_args)


class CrOSUpdateTrigger(object):
  """The class for CrOS auto-updater trigger.

  This class is used for running all CrOS auto-update trigger logic.
  """
  def __init__(self, host_name, build_name, static_dir, progress_tracker=None,
               log_file=None, au_tempdir=None, force_update=False,
               full_update=False, original_build=None, payload_filename=None,
               clobber_stateful=True, quick_provision=False,
               devserver_url=None, static_url=None):
    self.host_name = host_name
    self.build_name = build_name
    self.static_dir = static_dir
    self.progress_tracker = progress_tracker
    self.log_file = log_file
    self.au_tempdir = au_tempdir
    self.force_update = force_update
    self.full_update = full_update
    self.original_build = original_build
    self.payload_filename = payload_filename
    self.clobber_stateful = clobber_stateful
    self.quick_provision = quick_provision
    self.devserver_url = devserver_url
    self.static_url = static_url

  def _WriteAUStatus(self, content):
    if self.progress_tracker:
      self.progress_tracker.WriteStatus(content)

  def _StatefulUpdate(self, cros_updater):
    """The detailed process in stateful update.

    Args:
      cros_updater: The CrOS auto updater for auto-update.
    """
    self._WriteAUStatus('pre-setup stateful update')
    cros_updater.PreSetupStatefulUpdate()
    self._WriteAUStatus('perform stateful update')
    cros_updater.UpdateStateful()
    self._WriteAUStatus('post-check stateful update')
    cros_updater.PostCheckStatefulUpdate()

  def _RootfsUpdate(self, cros_updater):
    """The detailed process in rootfs update.

    Args:
      cros_updater: The CrOS auto updater for auto-update.
    """
    self._WriteAUStatus('Check whether devserver can run before rootfs update')
    cros_updater.CheckDevserverRun()
    self._WriteAUStatus('transfer rootfs update package')
    cros_updater.TransferRootfsUpdate()
    self._WriteAUStatus('pre-setup rootfs update')
    cros_updater.PreSetupRootfsUpdate()
    self._WriteAUStatus('rootfs update')
    cros_updater.UpdateRootfs()
    self._WriteAUStatus('post-check rootfs update')
    cros_updater.PostCheckRootfsUpdate()

  def _GetOriginalPayloadDir(self):
    """Get the directory of original payload.

    Returns:
      The directory of original payload, whose format is like:
          'static/stable-channel/link/3428.210.0'
    """
    if self.original_build:
      return os.path.join(self.static_dir, '%s/%s' % (STABLE_BUILD_CHANNEL,
                                                      self.original_build))
    else:
      return None

  def _MakeStatusUrl(self, devserver_url, host_name, pid):
    """Generates a URL to post auto update status to.

    Args:
      devserver_url: URL base for devserver RPCs.
      host_name: Host to post status for.
      pid: pid of the update process.

    Returns:
      An unescaped URL.
    """
    return '%s/post_au_status?host_name=%s&pid=%d' % (devserver_url, host_name,
                                                      pid)

  def _QuickProvision(self, device):
    """Performs a quick provision of device.

    Returns:
      A dictionary of extracted key-value pairs returned from the script
      execution.

    Raises:
      cros_build_lib.RunCommandError: error executing command or script
      remote_access.SSHConnectionError: SSH connection error
    """
    pid = os.getpid()
    pgid = os.getpgid(pid)
    if self.progress_tracker is None:
      self.progress_tracker = cros_update_progress.AUProgress(self.host_name,
                                                              pgid)

    dut_script = '/tmp/quick-provision'
    status_url = self._MakeStatusUrl(self.devserver_url, self.host_name, pgid)
    cmd = ('curl -o %s %s && bash '
           '%s --status_url %s %s %s') % (
               dut_script, os.path.join(self.static_url, 'quick-provision'),
               dut_script, cros_build_lib.ShellQuote(status_url),
               self.build_name, self.static_url
           )
    # Quick provision script issues a reboot and might result in the SSH
    # connection being terminated so set ssh_error_ok so that output can
    # still be captured.
    results = device.RunCommand(cmd, log_output=True, capture_output=True,
                                ssh_error_ok=True)
    key_re = re.compile(r'^KEYVAL: ([^\d\W]\w*)=(.*)$')
    matches = [key_re.match(l) for l in results.output.splitlines()]
    keyvals = {m.group(1): m.group(2) for m in matches if m}
    logging.info("DUT returned keyvals: %s", keyvals)

    # If there was an SSH error, check the keyvals to see if it actually
    # completed and suppress the error if so.
    if results.returncode == remote_access.SSH_ERROR_CODE:
      if 'COMPLETED' in keyvals:
        logging.warning('Quick provision completed with ssh error, ignoring...')
      else:
        logging.error('Incomplete quick provision failed with ssh error')
        raise remote_access.SSHConnectionError(results.error)

    return keyvals

  def TriggerAU(self):
    """Execute auto update for cros_host.

    The auto update includes 4 steps:
    1. if devserver cannot run, restore the stateful partition.
    2. if possible, do stateful update first, but never raise errors, except
       for timeout_util.TimeoutError caused by system.signal.
    3. If required or stateful_update fails, first do rootfs update, then do
       stateful_update.
    4. Post-check for the whole update.
    """
    try:
      with remote_access.ChromiumOSDeviceHandler(
          self.host_name, port=None,
          base_dir=CROS_PRESERVED_PATH,
          ping=False) as device:

        logging.debug('Remote device %s is connected', self.host_name)
        payload_dir = os.path.join(self.static_dir, self.build_name)
        original_payload_dir = self._GetOriginalPayloadDir()

        chromeos_AU = auto_updater.ChromiumOSUpdater(
            device, self.build_name, payload_dir,
            dev_dir=os.path.abspath(os.path.dirname(__file__)),
            tempdir=self.au_tempdir,
            log_file=self.log_file,
            original_payload_dir=original_payload_dir,
            yes=True,
            payload_filename=self.payload_filename,
            clobber_stateful=self.clobber_stateful)

        # Allow fall back if the quick provision does not succeed.
        invoke_autoupdate = True

        if (self.quick_provision and self.clobber_stateful and
            not self.full_update):
          try:
            logging.debug('Start CrOS quick provision process...')
            self._WriteAUStatus('Start Quick Provision')
            keyvals = self._QuickProvision(device)
            logging.debug('Start CrOS check process...')
            self._WriteAUStatus('Finish Quick Provision, reboot')
            chromeos_AU.AwaitReboot(keyvals.get('BOOT_ID'))
            self._WriteAUStatus('Finish Quick Provision, post-check')
            chromeos_AU.PostCheckCrOSUpdate()
            self._WriteAUStatus(cros_update_progress.FINISHED)
            invoke_autoupdate = False
          except (cros_build_lib.RunCommandError,
                  remote_access.SSHConnectionError,
                  auto_updater.RebootVerificationError) as e:
            logging.warning(
                'Error during quick provision, falling back to legacy: %s: %s',
                type(e).__name__, e)
            time.sleep(QUICK_PROVISION_FAILURE_DELAY_SEC)

        if invoke_autoupdate:
          chromeos_AU.CheckPayloads()

          version_match = chromeos_AU.PreSetupCrOSUpdate()
          self._WriteAUStatus('Transfer Devserver/Stateful Update Package')
          chromeos_AU.TransferDevServerPackage()
          chromeos_AU.TransferStatefulUpdate()

          restore_stateful = chromeos_AU.CheckRestoreStateful()
          do_stateful_update = (not self.full_update) and (
              version_match and self.force_update)
          stateful_update_complete = False
          logging.debug('Start CrOS update process...')
          try:
            if restore_stateful:
              self._WriteAUStatus('Restore Stateful Partition')
              chromeos_AU.RestoreStateful()
              stateful_update_complete = True
            else:
              # Whether to execute stateful update depends on:
              # a. full_update=False: No full reimage is required.
              # b. The update version is matched to the current version, And
              #    force_update=True: Update is forced even if the version
              #    installed is the same.
              if do_stateful_update:
                self._StatefulUpdate(chromeos_AU)
                stateful_update_complete = True

          except timeout_util.TimeoutError:
            raise
          except Exception as e:
            logging.debug('Error happens in stateful update: %r', e)

          # Whether to execute rootfs update depends on:
          # a. stateful update is not completed, or completed by
          #    update action 'restore_stateful'.
          # b. force_update=True: Update is forced no matter what the current
          #    version is. Or, the update version is not matched to the current
          #    version.
          require_rootfs_update = self.force_update or (
              not chromeos_AU.CheckVersion())
          if (not (do_stateful_update and stateful_update_complete)
              and require_rootfs_update):
            self._RootfsUpdate(chromeos_AU)
            self._StatefulUpdate(chromeos_AU)

          self._WriteAUStatus('post-check for CrOS auto-update')
          chromeos_AU.PostCheckCrOSUpdate()
          self._WriteAUStatus(cros_update_progress.FINISHED)

        logging.debug('Provision successfully completed (%s)',
                      'legacy' if invoke_autoupdate else 'quick provision')
    except Exception as e:
      logging.debug('Error happens in CrOS auto-update: %r', e)
      self._WriteAUStatus(CROS_ERROR_TEMPLATE % str(traceback.format_exc()))
      raise


def main():
  # Create one cros_update_parser instance for parsing CrOS auto-update cmd.
  AU_parser = CrOSAUParser()
  try:
    AU_parser.ParseArgs()
  except Exception as e:
    logging.error('Error in Parsing Args: %r', e)
    raise

  if len(sys.argv) == 1:
    AU_parser.parser.print_help()
    sys.exit(1)

  options = AU_parser.options

  # Use process group id as the unique id in track and log files, since
  # os.setsid is executed before the current process is run.
  pid = os.getpid()
  pgid = os.getpgid(pid)

  # Setting log files for CrOS auto-update process.
  # Log file:  file to record every details of CrOS auto-update process.
  log_file = cros_update_progress.GetExecuteLogFile(options.host_name, pgid)
  logging.info('Writing executing logs into file: %s', log_file)
  logConfig.SetFileHandler(log_file)

  # Create a progress_tracker for tracking CrOS auto-update progress.
  progress_tracker = cros_update_progress.AUProgress(options.host_name, pgid)

  # Create a dir for temporarily storing devserver codes and logs.
  au_tempdir = cros_update_progress.GetAUTempDirectory(options.host_name, pgid)

  # Create cros_update instance to run CrOS auto-update.
  cros_updater_trigger = CrOSUpdateTrigger(
      options.host_name, options.build_name, options.static_dir,
      progress_tracker=progress_tracker,
      log_file=log_file,
      au_tempdir=au_tempdir,
      force_update=options.force_update,
      full_update=options.full_update,
      original_build=options.original_build,
      payload_filename=options.payload_filename,
      clobber_stateful=options.clobber_stateful,
      quick_provision=options.quick_provision,
      devserver_url=options.devserver_url,
      static_url=options.static_url)

  # Set timeout the cros-update process.
  try:
    with timeout_util.Timeout(CROS_UPDATE_TIMEOUT_MIN*60):
      cros_updater_trigger.TriggerAU()
  except timeout_util.TimeoutError as e:
    error_msg = ('%s. The CrOS auto-update process is timed out, thus will be '
                 'terminated' % str(e))
    progress_tracker.WriteStatus(CROS_ERROR_TEMPLATE % error_msg)


if __name__ == '__main__':
  main()