Chris Sosa | 5e4246b | 2012-05-22 18:05:22 -0700 | [diff] [blame] | 1 | # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. |
Sean O'Connor | 5346e4e | 2010-08-12 18:49:24 +0200 | [diff] [blame] | 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | |
Don Garrett | 56b1cc8 | 2013-12-06 17:49:20 -0800 | [diff] [blame] | 5 | import glob |
Sean O'Connor | 5346e4e | 2010-08-12 18:49:24 +0200 | [diff] [blame] | 6 | import logging |
Dale Curtis | 5c32c72 | 2011-05-04 19:24:23 -0700 | [diff] [blame] | 7 | import os |
Sean O'Connor | 5346e4e | 2010-08-12 18:49:24 +0200 | [diff] [blame] | 8 | import re |
Richard Barnette | 0beb14b | 2018-05-15 18:07:52 +0000 | [diff] [blame] | 9 | import time |
Prashanth B | 32baa9b | 2014-03-13 13:23:01 -0700 | [diff] [blame] | 10 | import urllib2 |
Richard Barnette | 0beb14b | 2018-05-15 18:07:52 +0000 | [diff] [blame] | 11 | import urlparse |
Sean O'Connor | 5346e4e | 2010-08-12 18:49:24 +0200 | [diff] [blame] | 12 | |
Chris Sosa | 6542508 | 2013-10-16 13:26:22 -0700 | [diff] [blame] | 13 | from autotest_lib.client.bin import utils |
Dale Curtis | 5c32c72 | 2011-05-04 19:24:23 -0700 | [diff] [blame] | 14 | from autotest_lib.client.common_lib import error, global_config |
Prashanth B | 32baa9b | 2014-03-13 13:23:01 -0700 | [diff] [blame] | 15 | from autotest_lib.client.common_lib.cros import dev_server |
Richard Barnette | 0beb14b | 2018-05-15 18:07:52 +0000 | [diff] [blame] | 16 | from autotest_lib.server import autotest |
Shelley Chen | 61d2898 | 2016-10-28 09:40:20 -0700 | [diff] [blame] | 17 | from autotest_lib.server import utils as server_utils |
Richard Barnette | 0beb14b | 2018-05-15 18:07:52 +0000 | [diff] [blame] | 18 | from autotest_lib.server.cros.dynamic_suite import constants as ds_constants |
| 19 | from autotest_lib.server.cros.dynamic_suite import tools |
Luigi Semenzato | e76d9f8 | 2016-11-21 11:15:10 -0800 | [diff] [blame] | 20 | from chromite.lib import retry_util |
Dan Shi | f3a35f7 | 2016-01-25 11:18:14 -0800 | [diff] [blame] | 21 | |
Shelley Chen | 16b8df3 | 2016-10-27 16:24:21 -0700 | [diff] [blame] | 22 | try: |
| 23 | from chromite.lib import metrics |
Dan Shi | 5e2efb7 | 2017-02-07 11:40:23 -0800 | [diff] [blame] | 24 | except ImportError: |
| 25 | metrics = utils.metrics_mock |
Sean O'Connor | 5346e4e | 2010-08-12 18:49:24 +0200 | [diff] [blame] | 26 | |
Gwendal Grignou | 3e96cc2 | 2017-06-07 16:22:51 -0700 | [diff] [blame] | 27 | try: |
| 28 | import devserver |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 29 | _STATEFUL_UPDATE_PATH = devserver.__path__[0] |
Gwendal Grignou | 3e96cc2 | 2017-06-07 16:22:51 -0700 | [diff] [blame] | 30 | except ImportError: |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 31 | _STATEFUL_UPDATE_PATH = '/usr/bin' |
Gwendal Grignou | 3e96cc2 | 2017-06-07 16:22:51 -0700 | [diff] [blame] | 32 | |
Dale Curtis | 5c32c72 | 2011-05-04 19:24:23 -0700 | [diff] [blame] | 33 | # Local stateful update path is relative to the CrOS source directory. |
Sean O'Connor | 5346e4e | 2010-08-12 18:49:24 +0200 | [diff] [blame] | 34 | UPDATER_IDLE = 'UPDATE_STATUS_IDLE' |
Sean O | c053dfe | 2010-08-23 18:22:26 +0200 | [diff] [blame] | 35 | UPDATER_NEED_REBOOT = 'UPDATE_STATUS_UPDATED_NEED_REBOOT' |
beeps | 5e8c45a | 2013-12-17 22:05:11 -0800 | [diff] [blame] | 36 | # A list of update engine client states that occur after an update is triggered. |
| 37 | UPDATER_PROCESSING_UPDATE = ['UPDATE_STATUS_CHECKING_FORUPDATE', |
| 38 | 'UPDATE_STATUS_UPDATE_AVAILABLE', |
| 39 | 'UPDATE_STATUS_DOWNLOADING', |
| 40 | 'UPDATE_STATUS_FINALIZING'] |
Sean O'Connor | 5346e4e | 2010-08-12 18:49:24 +0200 | [diff] [blame] | 41 | |
Richard Barnette | 0beb14b | 2018-05-15 18:07:52 +0000 | [diff] [blame] | 42 | |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 43 | _STATEFUL_UPDATE_SCRIPT = 'stateful_update' |
| 44 | _REMOTE_STATEFUL_UPDATE_PATH = os.path.join( |
| 45 | '/usr/local/bin', _STATEFUL_UPDATE_SCRIPT) |
| 46 | _REMOTE_TMP_STATEFUL_UPDATE = os.path.join( |
| 47 | '/tmp', _STATEFUL_UPDATE_SCRIPT) |
| 48 | |
| 49 | _UPDATER_BIN = '/usr/bin/update_engine_client' |
| 50 | _UPDATER_LOGS = ['/var/log/messages', '/var/log/update_engine'] |
| 51 | |
| 52 | _KERNEL_A = {'name': 'KERN-A', 'kernel': 2, 'root': 3} |
| 53 | _KERNEL_B = {'name': 'KERN-B', 'kernel': 4, 'root': 5} |
| 54 | |
| 55 | # Time to wait for new kernel to be marked successful after |
| 56 | # auto update. |
| 57 | _KERNEL_UPDATE_TIMEOUT = 120 |
| 58 | |
| 59 | |
Richard Barnette | 0beb14b | 2018-05-15 18:07:52 +0000 | [diff] [blame] | 60 | # PROVISION_FAILED - A flag file to indicate provision failures. The |
| 61 | # file is created at the start of any AU procedure (see |
| 62 | # `ChromiumOSUpdater.run_full_update()`). The file's location in |
| 63 | # stateful means that on successul update it will be removed. Thus, if |
| 64 | # this file exists, it indicates that we've tried and failed in a |
| 65 | # previous attempt to update. |
| 66 | PROVISION_FAILED = '/var/tmp/provision_failed' |
| 67 | |
| 68 | |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 69 | # A flag file used to enable special handling in lab DUTs. Some |
| 70 | # parts of the system in Chromium OS test images will behave in ways |
| 71 | # convenient to the test lab when this file is present. Generally, |
| 72 | # we create this immediately after any update completes. |
| 73 | _LAB_MACHINE_FILE = '/mnt/stateful_partition/.labmachine' |
| 74 | |
| 75 | |
Sean O'Connor | 5346e4e | 2010-08-12 18:49:24 +0200 | [diff] [blame] | 76 | class ChromiumOSError(error.InstallError): |
| 77 | """Generic error for ChromiumOS-specific exceptions.""" |
Gilad Arnold | d6adeb8 | 2015-09-21 07:10:03 -0700 | [diff] [blame] | 78 | |
| 79 | |
Chris Sosa | 77556d8 | 2012-04-05 15:23:14 -0700 | [diff] [blame] | 80 | class RootFSUpdateError(ChromiumOSError): |
| 81 | """Raised when the RootFS fails to update.""" |
Chris Sosa | 77556d8 | 2012-04-05 15:23:14 -0700 | [diff] [blame] | 82 | |
| 83 | |
| 84 | class StatefulUpdateError(ChromiumOSError): |
| 85 | """Raised when the stateful partition fails to update.""" |
Chris Sosa | 77556d8 | 2012-04-05 15:23:14 -0700 | [diff] [blame] | 86 | |
| 87 | |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 88 | def _url_to_version(update_url): |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 89 | """Return the version based on update_url. |
| 90 | |
| 91 | @param update_url: url to the image to update to. |
| 92 | |
| 93 | """ |
Dale Curtis | ddfdb94 | 2011-07-14 13:59:24 -0700 | [diff] [blame] | 94 | # The Chrome OS version is generally the last element in the URL. The only |
| 95 | # exception is delta update URLs, which are rooted under the version; e.g., |
| 96 | # http://.../update/.../0.14.755.0/au/0.14.754.0. In this case we want to |
| 97 | # strip off the au section of the path before reading the version. |
Dan Shi | 5002cfc | 2013-04-29 10:45:05 -0700 | [diff] [blame] | 98 | return re.sub('/au/.*', '', |
| 99 | urlparse.urlparse(update_url).path).split('/')[-1].strip() |
Sean O'Connor | 5346e4e | 2010-08-12 18:49:24 +0200 | [diff] [blame] | 100 | |
| 101 | |
Scott Zawalski | eadbf70 | 2013-03-14 09:23:06 -0400 | [diff] [blame] | 102 | def url_to_image_name(update_url): |
| 103 | """Return the image name based on update_url. |
| 104 | |
| 105 | From a URL like: |
| 106 | http://172.22.50.205:8082/update/lumpy-release/R27-3837.0.0 |
| 107 | return lumpy-release/R27-3837.0.0 |
| 108 | |
| 109 | @param update_url: url to the image to update to. |
| 110 | @returns a string representing the image name in the update_url. |
| 111 | |
| 112 | """ |
| 113 | return '/'.join(urlparse.urlparse(update_url).path.split('/')[-2:]) |
| 114 | |
| 115 | |
Prashanth B | 32baa9b | 2014-03-13 13:23:01 -0700 | [diff] [blame] | 116 | def _get_devserver_build_from_update_url(update_url): |
| 117 | """Get the devserver and build from the update url. |
| 118 | |
| 119 | @param update_url: The url for update. |
| 120 | Eg: http://devserver:port/update/build. |
| 121 | |
| 122 | @return: A tuple of (devserver url, build) or None if the update_url |
| 123 | doesn't match the expected pattern. |
| 124 | |
| 125 | @raises ValueError: If the update_url doesn't match the expected pattern. |
| 126 | @raises ValueError: If no global_config was found, or it doesn't contain an |
| 127 | image_url_pattern. |
| 128 | """ |
| 129 | pattern = global_config.global_config.get_config_value( |
| 130 | 'CROS', 'image_url_pattern', type=str, default='') |
| 131 | if not pattern: |
| 132 | raise ValueError('Cannot parse update_url, the global config needs ' |
| 133 | 'an image_url_pattern.') |
| 134 | re_pattern = pattern.replace('%s', '(\S+)') |
| 135 | parts = re.search(re_pattern, update_url) |
| 136 | if not parts or len(parts.groups()) < 2: |
| 137 | raise ValueError('%s is not an update url' % update_url) |
| 138 | return parts.groups() |
| 139 | |
| 140 | |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 141 | def _list_image_dir_contents(update_url): |
Prashanth B | 32baa9b | 2014-03-13 13:23:01 -0700 | [diff] [blame] | 142 | """Lists the contents of the devserver for a given build/update_url. |
| 143 | |
| 144 | @param update_url: An update url. Eg: http://devserver:port/update/build. |
| 145 | """ |
| 146 | if not update_url: |
| 147 | logging.warning('Need update_url to list contents of the devserver.') |
| 148 | return |
| 149 | error_msg = 'Cannot check contents of devserver, update url %s' % update_url |
| 150 | try: |
| 151 | devserver_url, build = _get_devserver_build_from_update_url(update_url) |
| 152 | except ValueError as e: |
| 153 | logging.warning('%s: %s', error_msg, e) |
| 154 | return |
| 155 | devserver = dev_server.ImageServer(devserver_url) |
| 156 | try: |
| 157 | devserver.list_image_dir(build) |
| 158 | # The devserver will retry on URLError to avoid flaky connections, but will |
| 159 | # eventually raise the URLError if it persists. All HTTPErrors get |
| 160 | # converted to DevServerExceptions. |
| 161 | except (dev_server.DevServerException, urllib2.URLError) as e: |
| 162 | logging.warning('%s: %s', error_msg, e) |
| 163 | |
| 164 | |
Gilad Arnold | d6adeb8 | 2015-09-21 07:10:03 -0700 | [diff] [blame] | 165 | # TODO(garnold) This implements shared updater functionality needed for |
| 166 | # supporting the autoupdate_EndToEnd server-side test. We should probably |
| 167 | # migrate more of the existing ChromiumOSUpdater functionality to it as we |
| 168 | # expand non-CrOS support in other tests. |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 169 | class ChromiumOSUpdater(object): |
| 170 | """Chromium OS specific DUT update functionality.""" |
Gilad Arnold | d6adeb8 | 2015-09-21 07:10:03 -0700 | [diff] [blame] | 171 | |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 172 | def __init__(self, update_url, host=None, interactive=True): |
Gilad Arnold | d6adeb8 | 2015-09-21 07:10:03 -0700 | [diff] [blame] | 173 | """Initializes the object. |
| 174 | |
Gilad Arnold | d6adeb8 | 2015-09-21 07:10:03 -0700 | [diff] [blame] | 175 | @param update_url: The URL we want the update to use. |
| 176 | @param host: A client.common_lib.hosts.Host implementation. |
David Haddock | 76a4c88 | 2017-12-13 18:50:09 -0800 | [diff] [blame] | 177 | @param interactive: Bool whether we are doing an interactive update. |
Gilad Arnold | d6adeb8 | 2015-09-21 07:10:03 -0700 | [diff] [blame] | 178 | """ |
Gilad Arnold | d6adeb8 | 2015-09-21 07:10:03 -0700 | [diff] [blame] | 179 | self.update_url = update_url |
| 180 | self.host = host |
David Haddock | 76a4c88 | 2017-12-13 18:50:09 -0800 | [diff] [blame] | 181 | self.interactive = interactive |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 182 | self.update_version = _url_to_version(update_url) |
| 183 | |
| 184 | |
| 185 | def _run(self, cmd, *args, **kwargs): |
| 186 | """Abbreviated form of self.host.run(...)""" |
| 187 | return self.host.run(cmd, *args, **kwargs) |
Gilad Arnold | d6adeb8 | 2015-09-21 07:10:03 -0700 | [diff] [blame] | 188 | |
| 189 | |
| 190 | def check_update_status(self): |
| 191 | """Returns the current update engine state. |
| 192 | |
| 193 | We use the `update_engine_client -status' command and parse the line |
| 194 | indicating the update state, e.g. "CURRENT_OP=UPDATE_STATUS_IDLE". |
| 195 | """ |
Luigi Semenzato | f15c8fc | 2017-03-03 14:12:40 -0800 | [diff] [blame] | 196 | update_status = self.host.run(command='%s -status | grep CURRENT_OP' % |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 197 | _UPDATER_BIN) |
Gilad Arnold | d6adeb8 | 2015-09-21 07:10:03 -0700 | [diff] [blame] | 198 | return update_status.stdout.strip().split('=')[-1] |
| 199 | |
| 200 | |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 201 | def _get_last_update_error(self): |
Shuqian Zhao | d999272 | 2016-02-29 12:26:38 -0800 | [diff] [blame] | 202 | """Get the last autoupdate error code.""" |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 203 | command_result = self._run( |
| 204 | '%s --last_attempt_error' % _UPDATER_BIN) |
| 205 | return command_result.stdout.strip().replace('\n', ', ') |
Shuqian Zhao | d999272 | 2016-02-29 12:26:38 -0800 | [diff] [blame] | 206 | |
| 207 | |
Luigi Semenzato | e76d9f8 | 2016-11-21 11:15:10 -0800 | [diff] [blame] | 208 | def _base_update_handler_no_retry(self, run_args): |
Shuqian Zhao | d999272 | 2016-02-29 12:26:38 -0800 | [diff] [blame] | 209 | """Base function to handle a remote update ssh call. |
| 210 | |
| 211 | @param run_args: Dictionary of args passed to ssh_host.run function. |
Shuqian Zhao | d999272 | 2016-02-29 12:26:38 -0800 | [diff] [blame] | 212 | |
Luigi Semenzato | e76d9f8 | 2016-11-21 11:15:10 -0800 | [diff] [blame] | 213 | @throws: intercepts and re-throws all exceptions |
Shuqian Zhao | d999272 | 2016-02-29 12:26:38 -0800 | [diff] [blame] | 214 | """ |
Shuqian Zhao | d999272 | 2016-02-29 12:26:38 -0800 | [diff] [blame] | 215 | try: |
| 216 | self.host.run(**run_args) |
Shuqian Zhao | d999272 | 2016-02-29 12:26:38 -0800 | [diff] [blame] | 217 | except Exception as e: |
Luigi Semenzato | e76d9f8 | 2016-11-21 11:15:10 -0800 | [diff] [blame] | 218 | logging.debug('exception in update handler: %s', e) |
| 219 | raise e |
Shuqian Zhao | d999272 | 2016-02-29 12:26:38 -0800 | [diff] [blame] | 220 | |
Luigi Semenzato | e76d9f8 | 2016-11-21 11:15:10 -0800 | [diff] [blame] | 221 | |
| 222 | def _base_update_handler(self, run_args, err_msg_prefix=None): |
| 223 | """Handle a remote update ssh call, possibly with retries. |
| 224 | |
| 225 | @param run_args: Dictionary of args passed to ssh_host.run function. |
| 226 | @param err_msg_prefix: Prefix of the exception error message. |
| 227 | """ |
| 228 | def exception_handler(e): |
| 229 | """Examines exceptions and returns True if the update handler |
| 230 | should be retried. |
| 231 | |
| 232 | @param e: the exception intercepted by the retry util. |
| 233 | """ |
| 234 | return (isinstance(e, error.AutoservSSHTimeout) or |
| 235 | (isinstance(e, error.GenericHostRunError) and |
| 236 | hasattr(e, 'description') and |
| 237 | (re.search('ERROR_CODE=37', e.description) or |
| 238 | re.search('generic error .255.', e.description)))) |
| 239 | |
| 240 | try: |
| 241 | # Try the update twice (arg 2 is max_retry, not including the first |
| 242 | # call). Some exceptions may be caught by the retry handler. |
| 243 | retry_util.GenericRetry(exception_handler, 1, |
| 244 | self._base_update_handler_no_retry, |
| 245 | run_args) |
| 246 | except Exception as e: |
| 247 | message = err_msg_prefix + ': ' + str(e) |
| 248 | raise RootFSUpdateError(message) |
Shuqian Zhao | d999272 | 2016-02-29 12:26:38 -0800 | [diff] [blame] | 249 | |
| 250 | |
Luigi Semenzato | f15c8fc | 2017-03-03 14:12:40 -0800 | [diff] [blame] | 251 | def _wait_for_update_service(self): |
| 252 | """Ensure that the update engine daemon is running, possibly |
| 253 | by waiting for it a bit in case the DUT just rebooted and the |
| 254 | service hasn't started yet. |
| 255 | """ |
| 256 | def handler(e): |
| 257 | """Retry exception handler. |
| 258 | |
| 259 | Assumes that the error is due to the update service not having |
| 260 | started yet. |
| 261 | |
| 262 | @param e: the exception intercepted by the retry util. |
| 263 | """ |
| 264 | if isinstance(e, error.AutoservRunError): |
| 265 | logging.debug('update service check exception: %s\n' |
| 266 | 'retrying...', e) |
| 267 | return True |
| 268 | else: |
| 269 | return False |
| 270 | |
| 271 | # Retry at most three times, every 5s. |
| 272 | status = retry_util.GenericRetry(handler, 3, |
| 273 | self.check_update_status, |
| 274 | sleep=5) |
| 275 | |
| 276 | # Expect the update engine to be idle. |
| 277 | if status != UPDATER_IDLE: |
| 278 | raise ChromiumOSError('%s is not in an installable state' % |
| 279 | self.host.hostname) |
| 280 | |
| 281 | |
Gilad Arnold | d6adeb8 | 2015-09-21 07:10:03 -0700 | [diff] [blame] | 282 | def trigger_update(self): |
| 283 | """Triggers a background update. |
| 284 | |
Shuqian Zhao | d999272 | 2016-02-29 12:26:38 -0800 | [diff] [blame] | 285 | @raise RootFSUpdateError or unknown Exception if anything went wrong. |
Gilad Arnold | d6adeb8 | 2015-09-21 07:10:03 -0700 | [diff] [blame] | 286 | """ |
Luigi Semenzato | f15c8fc | 2017-03-03 14:12:40 -0800 | [diff] [blame] | 287 | # If this function is called immediately after reboot (which it is at |
| 288 | # this time), there is no guarantee that the update service is up and |
| 289 | # running yet, so wait for it. |
| 290 | self._wait_for_update_service() |
| 291 | |
Gilad Arnold | d6adeb8 | 2015-09-21 07:10:03 -0700 | [diff] [blame] | 292 | autoupdate_cmd = ('%s --check_for_update --omaha_url=%s' % |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 293 | (_UPDATER_BIN, self.update_url)) |
Shuqian Zhao | d999272 | 2016-02-29 12:26:38 -0800 | [diff] [blame] | 294 | run_args = {'command': autoupdate_cmd} |
| 295 | err_prefix = 'Failed to trigger an update on %s. ' % self.host.hostname |
Gilad Arnold | d6adeb8 | 2015-09-21 07:10:03 -0700 | [diff] [blame] | 296 | logging.info('Triggering update via: %s', autoupdate_cmd) |
Allen Li | 1a5cc0a | 2017-06-20 14:08:59 -0700 | [diff] [blame] | 297 | metric_fields = {'success': False} |
Luigi Semenzato | e76d9f8 | 2016-11-21 11:15:10 -0800 | [diff] [blame] | 298 | try: |
Luigi Semenzato | e76d9f8 | 2016-11-21 11:15:10 -0800 | [diff] [blame] | 299 | self._base_update_handler(run_args, err_prefix) |
Allen Li | 1a5cc0a | 2017-06-20 14:08:59 -0700 | [diff] [blame] | 300 | metric_fields['success'] = True |
| 301 | finally: |
Allen Li | 1a5cc0a | 2017-06-20 14:08:59 -0700 | [diff] [blame] | 302 | c = metrics.Counter('chromeos/autotest/autoupdater/trigger') |
Allen Li | b5420a7 | 2017-06-20 14:14:07 -0700 | [diff] [blame] | 303 | metric_fields.update(self._get_metric_fields()) |
Allen Li | 1a5cc0a | 2017-06-20 14:08:59 -0700 | [diff] [blame] | 304 | c.increment(fields=metric_fields) |
Gilad Arnold | d6adeb8 | 2015-09-21 07:10:03 -0700 | [diff] [blame] | 305 | |
| 306 | |
Allen Li | b5420a7 | 2017-06-20 14:14:07 -0700 | [diff] [blame] | 307 | def _get_metric_fields(self): |
| 308 | """Return a dict of metric fields. |
| 309 | |
| 310 | This is used for sending autoupdate metrics for this instance. |
| 311 | """ |
| 312 | build_name = url_to_image_name(self.update_url) |
| 313 | try: |
| 314 | board, build_type, milestone, _ = server_utils.ParseBuildName( |
| 315 | build_name) |
| 316 | except server_utils.ParseBuildNameException: |
| 317 | logging.warning('Unable to parse build name %s for metrics. ' |
| 318 | 'Continuing anyway.', build_name) |
| 319 | board, build_type, milestone = ('', '', '') |
| 320 | return { |
| 321 | 'dev_server': dev_server.get_hostname(self.update_url), |
| 322 | 'board': board, |
| 323 | 'build_type': build_type, |
| 324 | 'milestone': milestone, |
| 325 | } |
| 326 | |
| 327 | |
Gilad Arnold | d6adeb8 | 2015-09-21 07:10:03 -0700 | [diff] [blame] | 328 | def _verify_update_completed(self): |
| 329 | """Verifies that an update has completed. |
| 330 | |
| 331 | @raise RootFSUpdateError: if verification fails. |
| 332 | """ |
| 333 | status = self.check_update_status() |
| 334 | if status != UPDATER_NEED_REBOOT: |
Shuqian Zhao | d999272 | 2016-02-29 12:26:38 -0800 | [diff] [blame] | 335 | error_msg = '' |
| 336 | if status == UPDATER_IDLE: |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 337 | error_msg = 'Update error: %s' % self._get_last_update_error() |
Gilad Arnold | d6adeb8 | 2015-09-21 07:10:03 -0700 | [diff] [blame] | 338 | raise RootFSUpdateError('Update did not complete with correct ' |
Shuqian Zhao | d999272 | 2016-02-29 12:26:38 -0800 | [diff] [blame] | 339 | 'status. Expecting %s, actual %s. %s' % |
| 340 | (UPDATER_NEED_REBOOT, status, error_msg)) |
Gilad Arnold | d6adeb8 | 2015-09-21 07:10:03 -0700 | [diff] [blame] | 341 | |
| 342 | |
| 343 | def update_image(self): |
| 344 | """Updates the device image and verifies success.""" |
Shuqian Zhao | fe4d62e | 2016-06-23 14:46:45 -0700 | [diff] [blame] | 345 | autoupdate_cmd = ('%s --update --omaha_url=%s' % |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 346 | (_UPDATER_BIN, self.update_url)) |
David Haddock | 76a4c88 | 2017-12-13 18:50:09 -0800 | [diff] [blame] | 347 | if not self.interactive: |
| 348 | autoupdate_cmd = '%s --interactive=false' % autoupdate_cmd |
Shuqian Zhao | d999272 | 2016-02-29 12:26:38 -0800 | [diff] [blame] | 349 | run_args = {'command': autoupdate_cmd, 'timeout': 3600} |
| 350 | err_prefix = ('Failed to install device image using payload at %s ' |
| 351 | 'on %s. ' % (self.update_url, self.host.hostname)) |
| 352 | logging.info('Updating image via: %s', autoupdate_cmd) |
Allen Li | 1a5cc0a | 2017-06-20 14:08:59 -0700 | [diff] [blame] | 353 | metric_fields = {'success': False} |
Luigi Semenzato | e76d9f8 | 2016-11-21 11:15:10 -0800 | [diff] [blame] | 354 | try: |
Luigi Semenzato | e76d9f8 | 2016-11-21 11:15:10 -0800 | [diff] [blame] | 355 | self._base_update_handler(run_args, err_prefix) |
Allen Li | 1a5cc0a | 2017-06-20 14:08:59 -0700 | [diff] [blame] | 356 | metric_fields['success'] = True |
| 357 | finally: |
Allen Li | 1a5cc0a | 2017-06-20 14:08:59 -0700 | [diff] [blame] | 358 | c = metrics.Counter('chromeos/autotest/autoupdater/update') |
Allen Li | b5420a7 | 2017-06-20 14:14:07 -0700 | [diff] [blame] | 359 | metric_fields.update(self._get_metric_fields()) |
Allen Li | 1a5cc0a | 2017-06-20 14:08:59 -0700 | [diff] [blame] | 360 | c.increment(fields=metric_fields) |
Aviv Keshet | f37b2d7 | 2016-06-01 19:27:59 -0700 | [diff] [blame] | 361 | self._verify_update_completed() |
Gilad Arnold | d6adeb8 | 2015-09-21 07:10:03 -0700 | [diff] [blame] | 362 | |
| 363 | |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 364 | def _reset_update_engine(self): |
Chris Sosa | e92399e | 2015-04-24 11:32:59 -0700 | [diff] [blame] | 365 | """Resets the host to prepare for a clean update regardless of state.""" |
Chris Sosa | e92399e | 2015-04-24 11:32:59 -0700 | [diff] [blame] | 366 | self._run('stop ui || true') |
| 367 | self._run('stop update-engine || true') |
| 368 | self._run('start update-engine') |
Dale Curtis | 5c32c72 | 2011-05-04 19:24:23 -0700 | [diff] [blame] | 369 | |
Shuqian Zhao | fe4d62e | 2016-06-23 14:46:45 -0700 | [diff] [blame] | 370 | # Wait for update engine to be ready. |
Luigi Semenzato | f15c8fc | 2017-03-03 14:12:40 -0800 | [diff] [blame] | 371 | self._wait_for_update_service() |
Sean O | c053dfe | 2010-08-23 18:22:26 +0200 | [diff] [blame] | 372 | |
| 373 | |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 374 | def _rootdev(self, options=''): |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 375 | """Returns the stripped output of rootdev <options>. |
| 376 | |
| 377 | @param options: options to run rootdev. |
| 378 | |
| 379 | """ |
Dale Curtis | a94c19c | 2011-05-02 15:05:17 -0700 | [diff] [blame] | 380 | return self._run('rootdev %s' % options).stdout.strip() |
| 381 | |
| 382 | |
| 383 | def get_kernel_state(self): |
| 384 | """Returns the (<active>, <inactive>) kernel state as a pair.""" |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 385 | active_root = int(re.findall('\d+\Z', self._rootdev('-s'))[0]) |
| 386 | if active_root == _KERNEL_A['root']: |
| 387 | return _KERNEL_A, _KERNEL_B |
| 388 | elif active_root == _KERNEL_B['root']: |
| 389 | return _KERNEL_B, _KERNEL_A |
Dale Curtis | a94c19c | 2011-05-02 15:05:17 -0700 | [diff] [blame] | 390 | else: |
Dale Curtis | 5c32c72 | 2011-05-04 19:24:23 -0700 | [diff] [blame] | 391 | raise ChromiumOSError('Encountered unknown root partition: %s' % |
Dale Curtis | a94c19c | 2011-05-02 15:05:17 -0700 | [diff] [blame] | 392 | active_root) |
| 393 | |
| 394 | |
| 395 | def _cgpt(self, flag, kernel, dev='$(rootdev -s -d)'): |
| 396 | """Return numeric cgpt value for the specified flag, kernel, device. """ |
| 397 | return int(self._run('cgpt show -n -i %d %s %s' % ( |
| 398 | kernel['kernel'], flag, dev)).stdout.strip()) |
| 399 | |
| 400 | |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 401 | def _get_kernel_priority(self, kernel): |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 402 | """Return numeric priority for the specified kernel. |
| 403 | |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 404 | @param kernel: information of the given kernel, either _KERNEL_A |
| 405 | or _KERNEL_B. |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 406 | """ |
Dale Curtis | a94c19c | 2011-05-02 15:05:17 -0700 | [diff] [blame] | 407 | return self._cgpt('-P', kernel) |
| 408 | |
| 409 | |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 410 | def _get_kernel_success(self, kernel): |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 411 | """Return boolean success flag for the specified kernel. |
| 412 | |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 413 | @param kernel: information of the given kernel, either _KERNEL_A |
| 414 | or _KERNEL_B. |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 415 | """ |
Dale Curtis | a94c19c | 2011-05-02 15:05:17 -0700 | [diff] [blame] | 416 | return self._cgpt('-S', kernel) != 0 |
| 417 | |
| 418 | |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 419 | def _get_kernel_tries(self, kernel): |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 420 | """Return tries count for the specified kernel. |
| 421 | |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 422 | @param kernel: information of the given kernel, either _KERNEL_A |
| 423 | or _KERNEL_B. |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 424 | """ |
Dale Curtis | a94c19c | 2011-05-02 15:05:17 -0700 | [diff] [blame] | 425 | return self._cgpt('-T', kernel) |
Sean O | 267c00b | 2010-08-31 15:54:55 +0200 | [diff] [blame] | 426 | |
| 427 | |
Chris Sosa | 5e4246b | 2012-05-22 18:05:22 -0700 | [diff] [blame] | 428 | def get_stateful_update_script(self): |
Gwendal Grignou | 3e96cc2 | 2017-06-07 16:22:51 -0700 | [diff] [blame] | 429 | """Returns the path to the stateful update script on the target. |
Chris Sosa | 5e4246b | 2012-05-22 18:05:22 -0700 | [diff] [blame] | 430 | |
Gwendal Grignou | 3e96cc2 | 2017-06-07 16:22:51 -0700 | [diff] [blame] | 431 | When runnning test_that, stateful_update is in chroot /usr/sbin, |
| 432 | as installed by chromeos-base/devserver packages. |
| 433 | In the lab, it is installed with the python module devserver, by |
| 434 | build_externals.py command. |
Chris Sosa | a3ac215 | 2012-05-23 22:23:13 -0700 | [diff] [blame] | 435 | |
Gwendal Grignou | 3e96cc2 | 2017-06-07 16:22:51 -0700 | [diff] [blame] | 436 | If we can find it, we hope it exists already on the DUT, we assert |
| 437 | otherwise. |
| 438 | """ |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 439 | stateful_update_file = os.path.join(_STATEFUL_UPDATE_PATH, |
| 440 | _STATEFUL_UPDATE_SCRIPT) |
Gwendal Grignou | 3e96cc2 | 2017-06-07 16:22:51 -0700 | [diff] [blame] | 441 | if os.path.exists(stateful_update_file): |
Chris Sosa | 5e4246b | 2012-05-22 18:05:22 -0700 | [diff] [blame] | 442 | self.host.send_file( |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 443 | stateful_update_file, _REMOTE_TMP_STATEFUL_UPDATE, |
Gilad Arnold | 0c0df73 | 2015-09-21 06:37:59 -0700 | [diff] [blame] | 444 | delete_dest=True) |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 445 | return _REMOTE_TMP_STATEFUL_UPDATE |
Chris Sosa | 5e4246b | 2012-05-22 18:05:22 -0700 | [diff] [blame] | 446 | |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 447 | if self.host.path_exists(_REMOTE_STATEFUL_UPDATE_PATH): |
Gwendal Grignou | 3e96cc2 | 2017-06-07 16:22:51 -0700 | [diff] [blame] | 448 | logging.warning('Could not chroot %s script, falling back on %s', |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 449 | _STATEFUL_UPDATE_SCRIPT, |
| 450 | _REMOTE_STATEFUL_UPDATE_PATH) |
| 451 | return _REMOTE_STATEFUL_UPDATE_PATH |
Gwendal Grignou | 3e96cc2 | 2017-06-07 16:22:51 -0700 | [diff] [blame] | 452 | else: |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 453 | raise ChromiumOSError('Could not locate %s' % |
| 454 | _STATEFUL_UPDATE_SCRIPT) |
Chris Sosa | 5e4246b | 2012-05-22 18:05:22 -0700 | [diff] [blame] | 455 | |
| 456 | |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 457 | def _reset_stateful_partition(self): |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 458 | """Clear any pending stateful update request.""" |
Chris Sosa | 5e4246b | 2012-05-22 18:05:22 -0700 | [diff] [blame] | 459 | statefuldev_cmd = [self.get_stateful_update_script()] |
| 460 | statefuldev_cmd += ['--stateful_change=reset', '2>&1'] |
Chris Sosa | 66d7407 | 2013-09-19 11:21:29 -0700 | [diff] [blame] | 461 | self._run(' '.join(statefuldev_cmd)) |
Chris Sosa | 5e4246b | 2012-05-22 18:05:22 -0700 | [diff] [blame] | 462 | |
| 463 | |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 464 | def _revert_boot_partition(self): |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 465 | """Revert the boot partition.""" |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 466 | part = self._rootdev('-s') |
Ilja H. Friedel | 04be2bd | 2014-05-07 21:29:59 -0700 | [diff] [blame] | 467 | logging.warning('Reverting update; Boot partition will be %s', part) |
Sean O | 267c00b | 2010-08-31 15:54:55 +0200 | [diff] [blame] | 468 | return self._run('/postinst %s 2>&1' % part) |
| 469 | |
| 470 | |
Chris Sosa | c193217 | 2013-10-16 13:28:53 -0700 | [diff] [blame] | 471 | def rollback_rootfs(self, powerwash): |
| 472 | """Triggers rollback and waits for it to complete. |
| 473 | |
| 474 | @param powerwash: If true, powerwash as part of rollback. |
| 475 | |
| 476 | @raise RootFSUpdateError if anything went wrong. |
| 477 | |
| 478 | """ |
Dan Shi | 549fb82 | 2015-03-24 18:01:11 -0700 | [diff] [blame] | 479 | version = self.host.get_release_version() |
Chris Sosa | c861752 | 2014-06-09 23:22:26 +0000 | [diff] [blame] | 480 | # Introduced can_rollback in M36 (build 5772). # etc/lsb-release matches |
| 481 | # X.Y.Z. This version split just pulls the first part out. |
| 482 | try: |
| 483 | build_number = int(version.split('.')[0]) |
| 484 | except ValueError: |
| 485 | logging.error('Could not parse build number.') |
| 486 | build_number = 0 |
| 487 | |
| 488 | if build_number >= 5772: |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 489 | can_rollback_cmd = '%s --can_rollback' % _UPDATER_BIN |
Chris Sosa | c861752 | 2014-06-09 23:22:26 +0000 | [diff] [blame] | 490 | logging.info('Checking for rollback.') |
| 491 | try: |
| 492 | self._run(can_rollback_cmd) |
| 493 | except error.AutoservRunError as e: |
| 494 | raise RootFSUpdateError("Rollback isn't possible on %s: %s" % |
| 495 | (self.host.hostname, str(e))) |
| 496 | |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 497 | rollback_cmd = '%s --rollback --follow' % _UPDATER_BIN |
Chris Sosa | c193217 | 2013-10-16 13:28:53 -0700 | [diff] [blame] | 498 | if not powerwash: |
Dan Shi | f3a35f7 | 2016-01-25 11:18:14 -0800 | [diff] [blame] | 499 | rollback_cmd += ' --nopowerwash' |
Chris Sosa | c193217 | 2013-10-16 13:28:53 -0700 | [diff] [blame] | 500 | |
Chris Sosa | c861752 | 2014-06-09 23:22:26 +0000 | [diff] [blame] | 501 | logging.info('Performing rollback.') |
Chris Sosa | c193217 | 2013-10-16 13:28:53 -0700 | [diff] [blame] | 502 | try: |
| 503 | self._run(rollback_cmd) |
Chris Sosa | c193217 | 2013-10-16 13:28:53 -0700 | [diff] [blame] | 504 | except error.AutoservRunError as e: |
| 505 | raise RootFSUpdateError('Rollback failed on %s: %s' % |
| 506 | (self.host.hostname, str(e))) |
| 507 | |
| 508 | self._verify_update_completed() |
| 509 | |
Gilad Arnold | 0ed760c | 2012-11-05 23:42:53 -0800 | [diff] [blame] | 510 | |
Chris Sosa | 7231260 | 2013-04-16 15:01:56 -0700 | [diff] [blame] | 511 | def update_stateful(self, clobber=True): |
| 512 | """Updates the stateful partition. |
| 513 | |
| 514 | @param clobber: If True, a clean stateful installation. |
| 515 | """ |
Chris Sosa | 77556d8 | 2012-04-05 15:23:14 -0700 | [diff] [blame] | 516 | logging.info('Updating stateful partition...') |
joychen | 03eaad9 | 2013-06-26 09:55:21 -0700 | [diff] [blame] | 517 | statefuldev_url = self.update_url.replace('update', |
| 518 | 'static') |
Chris Sosa | a3ac215 | 2012-05-23 22:23:13 -0700 | [diff] [blame] | 519 | |
Dale Curtis | 5c32c72 | 2011-05-04 19:24:23 -0700 | [diff] [blame] | 520 | # Attempt stateful partition update; this must succeed so that the newly |
| 521 | # installed host is testable after update. |
Chris Sosa | 7231260 | 2013-04-16 15:01:56 -0700 | [diff] [blame] | 522 | statefuldev_cmd = [self.get_stateful_update_script(), statefuldev_url] |
| 523 | if clobber: |
| 524 | statefuldev_cmd.append('--stateful_change=clean') |
| 525 | |
| 526 | statefuldev_cmd.append('2>&1') |
Dale Curtis | 5c32c72 | 2011-05-04 19:24:23 -0700 | [diff] [blame] | 527 | try: |
Dan Shi | 205b873 | 2016-01-25 10:56:22 -0800 | [diff] [blame] | 528 | self._run(' '.join(statefuldev_cmd), timeout=1200) |
Dale Curtis | 5c32c72 | 2011-05-04 19:24:23 -0700 | [diff] [blame] | 529 | except error.AutoservRunError: |
Gilad Arnold | 62cf3a4 | 2015-10-01 09:15:25 -0700 | [diff] [blame] | 530 | update_error = StatefulUpdateError( |
| 531 | 'Failed to perform stateful update on %s' % |
| 532 | self.host.hostname) |
Chris Sosa | 77556d8 | 2012-04-05 15:23:14 -0700 | [diff] [blame] | 533 | raise update_error |
Dale Curtis | 5c32c72 | 2011-05-04 19:24:23 -0700 | [diff] [blame] | 534 | |
Chris Sosa | a3ac215 | 2012-05-23 22:23:13 -0700 | [diff] [blame] | 535 | |
Richard Barnette | 0beb14b | 2018-05-15 18:07:52 +0000 | [diff] [blame] | 536 | def _install_update(self, update_root=True): |
| 537 | """Install the requested image on the DUT, but don't start it. |
| 538 | |
| 539 | This downloads all content needed for the requested update, and |
| 540 | installs it in place on the DUT. This does not reboot the DUT, |
| 541 | so the update is merely pending when the function returns. |
| 542 | |
| 543 | @param update_root: When true, force a rootfs update; otherwise |
| 544 | update the stateful partition only. |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 545 | """ |
Dan Shi | 549fb82 | 2015-03-24 18:01:11 -0700 | [diff] [blame] | 546 | booted_version = self.host.get_release_version() |
Richard Barnette | 0173ea8 | 2018-05-04 21:13:57 +0000 | [diff] [blame] | 547 | logging.info('Updating from version %s to %s.', |
| 548 | booted_version, self.update_version) |
Dale Curtis | 53d5586 | 2011-05-16 12:17:59 -0700 | [diff] [blame] | 549 | |
Dale Curtis | 5c32c72 | 2011-05-04 19:24:23 -0700 | [diff] [blame] | 550 | # Check that Dev Server is accepting connections (from autoserv's host). |
Sean O'Connor | 5346e4e | 2010-08-12 18:49:24 +0200 | [diff] [blame] | 551 | # If we can't talk to it, the machine host probably can't either. |
xixuan | ccf2e72 | 2016-06-10 16:42:38 -0700 | [diff] [blame] | 552 | auserver_host = 'http://%s' % urlparse.urlparse(self.update_url)[1] |
Sean O'Connor | 5346e4e | 2010-08-12 18:49:24 +0200 | [diff] [blame] | 553 | try: |
xixuan | ccf2e72 | 2016-06-10 16:42:38 -0700 | [diff] [blame] | 554 | if not dev_server.ImageServer.devserver_healthy(auserver_host): |
| 555 | raise ChromiumOSError( |
| 556 | 'Update server at %s not healthy' % auserver_host) |
| 557 | except Exception as e: |
| 558 | logging.debug('Error happens in connection to devserver: %r', e) |
Dale Curtis | 5c32c72 | 2011-05-04 19:24:23 -0700 | [diff] [blame] | 559 | raise ChromiumOSError( |
| 560 | 'Update server at %s not available' % auserver_host) |
Sean O'Connor | 5346e4e | 2010-08-12 18:49:24 +0200 | [diff] [blame] | 561 | |
Chris Sosa | a3ac215 | 2012-05-23 22:23:13 -0700 | [diff] [blame] | 562 | logging.info('Installing from %s to %s', self.update_url, |
Chris Sosa | 77556d8 | 2012-04-05 15:23:14 -0700 | [diff] [blame] | 563 | self.host.hostname) |
| 564 | |
Chris Sosa | 5e4246b | 2012-05-22 18:05:22 -0700 | [diff] [blame] | 565 | # Reset update state. |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 566 | self._reset_update_engine() |
| 567 | self._reset_stateful_partition() |
Sean O | c053dfe | 2010-08-23 18:22:26 +0200 | [diff] [blame] | 568 | |
Dale Curtis | 1e97318 | 2011-07-12 18:21:36 -0700 | [diff] [blame] | 569 | try: |
Aviv Keshet | 2610d3e | 2016-06-01 16:37:01 -0700 | [diff] [blame] | 570 | try: |
| 571 | if not update_root: |
| 572 | logging.info('Root update is skipped.') |
| 573 | else: |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 574 | self.update_image() |
Sean O'Connor | 5346e4e | 2010-08-12 18:49:24 +0200 | [diff] [blame] | 575 | |
Aviv Keshet | 2610d3e | 2016-06-01 16:37:01 -0700 | [diff] [blame] | 576 | self.update_stateful() |
| 577 | except: |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 578 | self._revert_boot_partition() |
| 579 | self._reset_stateful_partition() |
Aviv Keshet | 2610d3e | 2016-06-01 16:37:01 -0700 | [diff] [blame] | 580 | raise |
Sean O | c053dfe | 2010-08-23 18:22:26 +0200 | [diff] [blame] | 581 | |
Dale Curtis | 1e97318 | 2011-07-12 18:21:36 -0700 | [diff] [blame] | 582 | logging.info('Update complete.') |
Dale Curtis | 1e97318 | 2011-07-12 18:21:36 -0700 | [diff] [blame] | 583 | except: |
| 584 | # Collect update engine logs in the event of failure. |
| 585 | if self.host.job: |
Aviv Keshet | 2610d3e | 2016-06-01 16:37:01 -0700 | [diff] [blame] | 586 | logging.info('Collecting update engine logs due to failure...') |
Dale Curtis | 1e97318 | 2011-07-12 18:21:36 -0700 | [diff] [blame] | 587 | self.host.get_file( |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 588 | _UPDATER_LOGS, self.host.job.sysinfo.sysinfodir, |
Gilad Arnold | 0c0df73 | 2015-09-21 06:37:59 -0700 | [diff] [blame] | 589 | preserve_perm=False) |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 590 | _list_image_dir_contents(self.update_url) |
Dale Curtis | 1e97318 | 2011-07-12 18:21:36 -0700 | [diff] [blame] | 591 | raise |
Dan Shi | 10e992b | 2013-08-30 11:02:59 -0700 | [diff] [blame] | 592 | finally: |
Shuqian Zhao | d999272 | 2016-02-29 12:26:38 -0800 | [diff] [blame] | 593 | logging.info('Update engine log has downloaded in ' |
| 594 | 'sysinfo/update_engine dir. Check the lastest.') |
Sean O'Connor | 5346e4e | 2010-08-12 18:49:24 +0200 | [diff] [blame] | 595 | |
| 596 | |
Richard Barnette | 0beb14b | 2018-05-15 18:07:52 +0000 | [diff] [blame] | 597 | def _try_stateful_update(self): |
| 598 | """Try to use stateful update to initialize DUT. |
| 599 | |
| 600 | When DUT is already running the same version that machine_install |
| 601 | tries to install, stateful update is a much faster way to clean up |
| 602 | the DUT for testing, compared to a full reimage. It is implemeted |
| 603 | by calling autoupdater._run_full_update, but skipping updating root, |
| 604 | as updating the kernel is time consuming and not necessary. |
| 605 | |
| 606 | @param update_url: url of the image. |
| 607 | @param updater: ChromiumOSUpdater instance used to update the DUT. |
| 608 | @returns: True if the DUT was updated with stateful update. |
| 609 | |
| 610 | """ |
| 611 | self.host.prepare_for_update() |
| 612 | |
| 613 | # TODO(jrbarnette): Yes, I hate this re.match() test case. |
| 614 | # It's better than the alternative: see crbug.com/360944. |
| 615 | image_name = url_to_image_name(self.update_url) |
| 616 | release_pattern = r'^.*-release/R[0-9]+-[0-9]+\.[0-9]+\.0$' |
| 617 | if not re.match(release_pattern, image_name): |
| 618 | return False |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 619 | if not self._check_version(): |
Richard Barnette | 0beb14b | 2018-05-15 18:07:52 +0000 | [diff] [blame] | 620 | return False |
| 621 | # Following folders should be rebuilt after stateful update. |
| 622 | # A test file is used to confirm each folder gets rebuilt after |
| 623 | # the stateful update. |
| 624 | folders_to_check = ['/var', '/home', '/mnt/stateful_partition'] |
| 625 | test_file = '.test_file_to_be_deleted' |
| 626 | paths = [os.path.join(folder, test_file) for folder in folders_to_check] |
| 627 | self._run('touch %s' % ' '.join(paths)) |
| 628 | |
| 629 | self._install_update(update_root=False) |
| 630 | |
| 631 | # Reboot to complete stateful update. |
| 632 | self.host.reboot(timeout=self.host.REBOOT_TIMEOUT, wait=True) |
| 633 | |
| 634 | # After stateful update and a reboot, all of the test_files shouldn't |
| 635 | # exist any more. Otherwise the stateful update is failed. |
| 636 | return not any( |
| 637 | self.host.path_exists(os.path.join(folder, test_file)) |
| 638 | for folder in folders_to_check) |
| 639 | |
| 640 | |
| 641 | def _post_update_processing(self, expected_kernel): |
| 642 | """After the DUT is updated, confirm machine_install succeeded. |
| 643 | |
| 644 | @param updater: ChromiumOSUpdater instance used to update the DUT. |
| 645 | @param expected_kernel: kernel expected to be active after reboot, |
| 646 | or `None` to skip rollback checking. |
| 647 | |
| 648 | """ |
| 649 | # Touch the lab machine file to leave a marker that |
| 650 | # distinguishes this image from other test images. |
| 651 | # Afterwards, we must re-run the autoreboot script because |
| 652 | # it depends on the _LAB_MACHINE_FILE. |
| 653 | autoreboot_cmd = ('FILE="%s" ; [ -f "$FILE" ] || ' |
| 654 | '( touch "$FILE" ; start autoreboot )') |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 655 | self._run(autoreboot_cmd % _LAB_MACHINE_FILE) |
Richard Barnette | 0beb14b | 2018-05-15 18:07:52 +0000 | [diff] [blame] | 656 | self.verify_boot_expectations( |
| 657 | expected_kernel, rollback_message= |
| 658 | 'Build %s failed to boot on %s; system rolled back to previous ' |
| 659 | 'build' % (self.update_version, self.host.hostname)) |
| 660 | |
| 661 | logging.debug('Cleaning up old autotest directories.') |
| 662 | try: |
| 663 | installed_autodir = autotest.Autotest.get_installed_autodir( |
| 664 | self.host) |
| 665 | self._run('rm -rf ' + installed_autodir) |
| 666 | except autotest.AutodirNotFoundError: |
| 667 | logging.debug('No autotest installed directory found.') |
| 668 | |
| 669 | |
| 670 | def run_update(self, force_full_update): |
| 671 | """Perform a full update of a DUT in the test lab. |
| 672 | |
| 673 | This downloads and installs the root FS and stateful partition |
| 674 | content needed for the update specified in `self.host` and |
| 675 | `self.update_url`. The update is performed according to the |
| 676 | requirements for provisioning a DUT for testing the requested |
| 677 | build. |
| 678 | |
| 679 | @param force_full_update: When true, update the root file |
| 680 | system to the new build, even if the target DUT already has |
| 681 | that build installed. |
| 682 | @returns A tuple of the form `(image_name, attributes)`, where |
| 683 | `image_name` is the name of the image installed, and |
| 684 | `attributes` is new attributes to be applied to the DUT. |
| 685 | """ |
| 686 | logging.debug('Update URL is %s', self.update_url) |
| 687 | |
| 688 | # Report provision stats. |
| 689 | server_name = dev_server.get_hostname(self.update_url) |
| 690 | (metrics.Counter('chromeos/autotest/provision/install') |
| 691 | .increment(fields={'devserver': server_name})) |
| 692 | |
| 693 | # Create a file to indicate if provision fails. The file will be |
| 694 | # removed by any successful update. |
| 695 | self._run('touch %s' % PROVISION_FAILED) |
| 696 | |
| 697 | update_complete = False |
| 698 | if not force_full_update: |
| 699 | try: |
| 700 | # If the DUT is already running the same build, try stateful |
| 701 | # update first as it's much quicker than a full re-image. |
| 702 | update_complete = self._try_stateful_update() |
| 703 | except Exception as e: |
| 704 | logging.exception(e) |
| 705 | |
| 706 | inactive_kernel = None |
| 707 | if update_complete: |
| 708 | logging.info('Install complete without full update') |
| 709 | else: |
| 710 | logging.info('DUT requires full update.') |
| 711 | self.host.reboot(timeout=self.host.REBOOT_TIMEOUT, wait=True) |
| 712 | self.host.prepare_for_update() |
| 713 | |
| 714 | self._install_update() |
| 715 | |
| 716 | # Give it some time in case of IO issues. |
| 717 | time.sleep(10) |
| 718 | |
| 719 | # Figure out active and inactive kernel. |
| 720 | active_kernel, inactive_kernel = self.get_kernel_state() |
| 721 | |
| 722 | # Ensure inactive kernel has higher priority than active. |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 723 | if (self._get_kernel_priority(inactive_kernel) |
| 724 | < self._get_kernel_priority(active_kernel)): |
Richard Barnette | 0beb14b | 2018-05-15 18:07:52 +0000 | [diff] [blame] | 725 | raise ChromiumOSError( |
| 726 | 'Update failed. The priority of the inactive kernel' |
| 727 | ' partition is less than that of the active kernel' |
| 728 | ' partition.') |
| 729 | |
| 730 | # Update has returned successfully; reboot the host. |
| 731 | # |
| 732 | # Regarding the 'crossystem' command below: In some cases, |
| 733 | # the update flow puts the TPM into a state such that it |
| 734 | # fails verification. We don't know why. However, this |
| 735 | # call papers over the problem by clearing the TPM during |
| 736 | # the reboot. |
| 737 | # |
| 738 | # We ignore failures from 'crossystem'. Although failure |
| 739 | # here is unexpected, and could signal a bug, the point of |
| 740 | # the exercise is to paper over problems; allowing this to |
| 741 | # fail would defeat the purpose. |
| 742 | self._run('crossystem clear_tpm_owner_request=1', |
| 743 | ignore_status=True) |
| 744 | self.host.reboot(timeout=self.host.REBOOT_TIMEOUT, wait=True) |
| 745 | |
| 746 | self._post_update_processing(inactive_kernel) |
| 747 | image_name = url_to_image_name(self.update_url) |
| 748 | # update_url is different from devserver url needed to stage autotest |
| 749 | # packages, therefore, resolve a new devserver url here. |
| 750 | devserver_url = dev_server.ImageServer.resolve( |
| 751 | image_name, self.host.hostname).url() |
| 752 | repo_url = tools.get_package_url(devserver_url, image_name) |
| 753 | return image_name, {ds_constants.JOB_REPO_URL: repo_url} |
| 754 | |
| 755 | |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 756 | def _check_version(self): |
Dan Shi | 0f466e8 | 2013-02-22 15:44:58 -0800 | [diff] [blame] | 757 | """Check the image running in DUT has the desired version. |
| 758 | |
| 759 | @returns: True if the DUT's image version matches the version that |
| 760 | the autoupdater tries to update to. |
| 761 | |
| 762 | """ |
Dan Shi | 549fb82 | 2015-03-24 18:01:11 -0700 | [diff] [blame] | 763 | booted_version = self.host.get_release_version() |
Richard Barnette | 0173ea8 | 2018-05-04 21:13:57 +0000 | [diff] [blame] | 764 | return self.update_version.endswith(booted_version) |
Dan Shi | b95bb86 | 2013-03-22 16:29:28 -0700 | [diff] [blame] | 765 | |
| 766 | |
Chris Sosa | 6542508 | 2013-10-16 13:26:22 -0700 | [diff] [blame] | 767 | def verify_boot_expectations(self, expected_kernel_state, rollback_message): |
| 768 | """Verifies that we fully booted given expected kernel state. |
| 769 | |
| 770 | This method both verifies that we booted using the correct kernel |
| 771 | state and that the OS has marked the kernel as good. |
| 772 | |
| 773 | @param expected_kernel_state: kernel state that we are verifying with |
| 774 | i.e. I expect to be booted onto partition 4 etc. See output of |
| 775 | get_kernel_state. |
| 776 | @param rollback_message: string to raise as a ChromiumOSError |
| 777 | if we booted with the wrong partition. |
| 778 | |
| 779 | @raises ChromiumOSError: If we didn't. |
| 780 | """ |
| 781 | # Figure out the newly active kernel. |
| 782 | active_kernel_state = self.get_kernel_state()[0] |
| 783 | |
| 784 | # Check for rollback due to a bad build. |
| 785 | if (expected_kernel_state and |
| 786 | active_kernel_state != expected_kernel_state): |
Don Garrett | 56b1cc8 | 2013-12-06 17:49:20 -0800 | [diff] [blame] | 787 | |
| 788 | # Kernel crash reports should be wiped between test runs, but |
| 789 | # may persist from earlier parts of the test, or from problems |
| 790 | # with provisioning. |
| 791 | # |
| 792 | # Kernel crash reports will NOT be present if the crash happened |
| 793 | # before encrypted stateful is mounted. |
| 794 | # |
| 795 | # TODO(dgarrett): Integrate with server/crashcollect.py at some |
| 796 | # point. |
| 797 | kernel_crashes = glob.glob('/var/spool/crash/kernel.*.kcrash') |
| 798 | if kernel_crashes: |
| 799 | rollback_message += ': kernel_crash' |
| 800 | logging.debug('Found %d kernel crash reports:', |
| 801 | len(kernel_crashes)) |
| 802 | # The crash names contain timestamps that may be useful: |
| 803 | # kernel.20131207.005945.0.kcrash |
| 804 | for crash in kernel_crashes: |
Dan Shi | 0942b1d | 2015-03-31 11:07:00 -0700 | [diff] [blame] | 805 | logging.debug(' %s', os.path.basename(crash)) |
Don Garrett | 56b1cc8 | 2013-12-06 17:49:20 -0800 | [diff] [blame] | 806 | |
Chris Sosa | 6542508 | 2013-10-16 13:26:22 -0700 | [diff] [blame] | 807 | # Print out some information to make it easier to debug |
| 808 | # the rollback. |
| 809 | logging.debug('Dumping partition table.') |
| 810 | self._run('cgpt show $(rootdev -s -d)') |
| 811 | logging.debug('Dumping crossystem for firmware debugging.') |
| 812 | self._run('crossystem --all') |
| 813 | raise ChromiumOSError(rollback_message) |
| 814 | |
| 815 | # Make sure chromeos-setgoodkernel runs. |
| 816 | try: |
| 817 | utils.poll_for_condition( |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 818 | lambda: (self._get_kernel_tries(active_kernel_state) == 0 |
| 819 | and self._get_kernel_success(active_kernel_state)), |
Chris Sosa | 6542508 | 2013-10-16 13:26:22 -0700 | [diff] [blame] | 820 | exception=ChromiumOSError(), |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 821 | timeout=_KERNEL_UPDATE_TIMEOUT, sleep_interval=5) |
Chris Sosa | 6542508 | 2013-10-16 13:26:22 -0700 | [diff] [blame] | 822 | except ChromiumOSError: |
| 823 | services_status = self._run('status system-services').stdout |
| 824 | if services_status != 'system-services start/running\n': |
| 825 | event = ('Chrome failed to reach login screen') |
| 826 | else: |
| 827 | event = ('update-engine failed to call ' |
| 828 | 'chromeos-setgoodkernel') |
| 829 | raise ChromiumOSError( |
| 830 | 'After update and reboot, %s ' |
Richard Barnette | 3e8b228 | 2018-05-15 20:42:20 +0000 | [diff] [blame^] | 831 | 'within %d seconds' % (event, _KERNEL_UPDATE_TIMEOUT)) |