blob: 848f2d9ea7fc20d213b738d1b2a427bfbc6bc8b9 [file] [log] [blame]
Chris Sosa5e4246b2012-05-22 18:05:22 -07001# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
Sean O'Connor5346e4e2010-08-12 18:49:24 +02002# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
Don Garrett56b1cc82013-12-06 17:49:20 -08005import glob
Sean O'Connor5346e4e2010-08-12 18:49:24 +02006import logging
Dale Curtis5c32c722011-05-04 19:24:23 -07007import os
Sean O'Connor5346e4e2010-08-12 18:49:24 +02008import re
Richard Barnette0beb14b2018-05-15 18:07:52 +00009import time
Prashanth B32baa9b2014-03-13 13:23:01 -070010import urllib2
Richard Barnette0beb14b2018-05-15 18:07:52 +000011import urlparse
Sean O'Connor5346e4e2010-08-12 18:49:24 +020012
Chris Sosa65425082013-10-16 13:26:22 -070013from autotest_lib.client.bin import utils
Dale Curtis5c32c722011-05-04 19:24:23 -070014from autotest_lib.client.common_lib import error, global_config
Prashanth B32baa9b2014-03-13 13:23:01 -070015from autotest_lib.client.common_lib.cros import dev_server
Richard Barnette0beb14b2018-05-15 18:07:52 +000016from autotest_lib.server import autotest
Shelley Chen61d28982016-10-28 09:40:20 -070017from autotest_lib.server import utils as server_utils
Richard Barnette0beb14b2018-05-15 18:07:52 +000018from autotest_lib.server.cros.dynamic_suite import constants as ds_constants
19from autotest_lib.server.cros.dynamic_suite import tools
Luigi Semenzatoe76d9f82016-11-21 11:15:10 -080020from chromite.lib import retry_util
Dan Shif3a35f72016-01-25 11:18:14 -080021
Shelley Chen16b8df32016-10-27 16:24:21 -070022try:
23 from chromite.lib import metrics
Dan Shi5e2efb72017-02-07 11:40:23 -080024except ImportError:
25 metrics = utils.metrics_mock
Sean O'Connor5346e4e2010-08-12 18:49:24 +020026
Gwendal Grignou3e96cc22017-06-07 16:22:51 -070027try:
28 import devserver
Richard Barnette3e8b2282018-05-15 20:42:20 +000029 _STATEFUL_UPDATE_PATH = devserver.__path__[0]
Gwendal Grignou3e96cc22017-06-07 16:22:51 -070030except ImportError:
Richard Barnette3e8b2282018-05-15 20:42:20 +000031 _STATEFUL_UPDATE_PATH = '/usr/bin'
Gwendal Grignou3e96cc22017-06-07 16:22:51 -070032
Dale Curtis5c32c722011-05-04 19:24:23 -070033# Local stateful update path is relative to the CrOS source directory.
Sean O'Connor5346e4e2010-08-12 18:49:24 +020034UPDATER_IDLE = 'UPDATE_STATUS_IDLE'
Sean Oc053dfe2010-08-23 18:22:26 +020035UPDATER_NEED_REBOOT = 'UPDATE_STATUS_UPDATED_NEED_REBOOT'
beeps5e8c45a2013-12-17 22:05:11 -080036# A list of update engine client states that occur after an update is triggered.
37UPDATER_PROCESSING_UPDATE = ['UPDATE_STATUS_CHECKING_FORUPDATE',
38 'UPDATE_STATUS_UPDATE_AVAILABLE',
39 'UPDATE_STATUS_DOWNLOADING',
40 'UPDATE_STATUS_FINALIZING']
Sean O'Connor5346e4e2010-08-12 18:49:24 +020041
Richard Barnette0beb14b2018-05-15 18:07:52 +000042
Richard Barnette3e8b2282018-05-15 20:42:20 +000043_STATEFUL_UPDATE_SCRIPT = 'stateful_update'
44_REMOTE_STATEFUL_UPDATE_PATH = os.path.join(
45 '/usr/local/bin', _STATEFUL_UPDATE_SCRIPT)
46_REMOTE_TMP_STATEFUL_UPDATE = os.path.join(
47 '/tmp', _STATEFUL_UPDATE_SCRIPT)
48
49_UPDATER_BIN = '/usr/bin/update_engine_client'
50_UPDATER_LOGS = ['/var/log/messages', '/var/log/update_engine']
51
52_KERNEL_A = {'name': 'KERN-A', 'kernel': 2, 'root': 3}
53_KERNEL_B = {'name': 'KERN-B', 'kernel': 4, 'root': 5}
54
55# Time to wait for new kernel to be marked successful after
56# auto update.
57_KERNEL_UPDATE_TIMEOUT = 120
58
59
Richard Barnette0beb14b2018-05-15 18:07:52 +000060# PROVISION_FAILED - A flag file to indicate provision failures. The
61# file is created at the start of any AU procedure (see
62# `ChromiumOSUpdater.run_full_update()`). The file's location in
63# stateful means that on successul update it will be removed. Thus, if
64# this file exists, it indicates that we've tried and failed in a
65# previous attempt to update.
66PROVISION_FAILED = '/var/tmp/provision_failed'
67
68
Richard Barnette3e8b2282018-05-15 20:42:20 +000069# A flag file used to enable special handling in lab DUTs. Some
70# parts of the system in Chromium OS test images will behave in ways
71# convenient to the test lab when this file is present. Generally,
72# we create this immediately after any update completes.
73_LAB_MACHINE_FILE = '/mnt/stateful_partition/.labmachine'
74
75
Sean O'Connor5346e4e2010-08-12 18:49:24 +020076class ChromiumOSError(error.InstallError):
77 """Generic error for ChromiumOS-specific exceptions."""
Gilad Arnoldd6adeb82015-09-21 07:10:03 -070078
79
Chris Sosa77556d82012-04-05 15:23:14 -070080class RootFSUpdateError(ChromiumOSError):
81 """Raised when the RootFS fails to update."""
Chris Sosa77556d82012-04-05 15:23:14 -070082
83
84class StatefulUpdateError(ChromiumOSError):
85 """Raised when the stateful partition fails to update."""
Chris Sosa77556d82012-04-05 15:23:14 -070086
87
Richard Barnette3e8b2282018-05-15 20:42:20 +000088def _url_to_version(update_url):
Dan Shi0f466e82013-02-22 15:44:58 -080089 """Return the version based on update_url.
90
91 @param update_url: url to the image to update to.
92
93 """
Dale Curtisddfdb942011-07-14 13:59:24 -070094 # The Chrome OS version is generally the last element in the URL. The only
95 # exception is delta update URLs, which are rooted under the version; e.g.,
96 # http://.../update/.../0.14.755.0/au/0.14.754.0. In this case we want to
97 # strip off the au section of the path before reading the version.
Dan Shi5002cfc2013-04-29 10:45:05 -070098 return re.sub('/au/.*', '',
99 urlparse.urlparse(update_url).path).split('/')[-1].strip()
Sean O'Connor5346e4e2010-08-12 18:49:24 +0200100
101
Scott Zawalskieadbf702013-03-14 09:23:06 -0400102def url_to_image_name(update_url):
103 """Return the image name based on update_url.
104
105 From a URL like:
106 http://172.22.50.205:8082/update/lumpy-release/R27-3837.0.0
107 return lumpy-release/R27-3837.0.0
108
109 @param update_url: url to the image to update to.
110 @returns a string representing the image name in the update_url.
111
112 """
113 return '/'.join(urlparse.urlparse(update_url).path.split('/')[-2:])
114
115
Prashanth B32baa9b2014-03-13 13:23:01 -0700116def _get_devserver_build_from_update_url(update_url):
117 """Get the devserver and build from the update url.
118
119 @param update_url: The url for update.
120 Eg: http://devserver:port/update/build.
121
122 @return: A tuple of (devserver url, build) or None if the update_url
123 doesn't match the expected pattern.
124
125 @raises ValueError: If the update_url doesn't match the expected pattern.
126 @raises ValueError: If no global_config was found, or it doesn't contain an
127 image_url_pattern.
128 """
129 pattern = global_config.global_config.get_config_value(
130 'CROS', 'image_url_pattern', type=str, default='')
131 if not pattern:
132 raise ValueError('Cannot parse update_url, the global config needs '
133 'an image_url_pattern.')
134 re_pattern = pattern.replace('%s', '(\S+)')
135 parts = re.search(re_pattern, update_url)
136 if not parts or len(parts.groups()) < 2:
137 raise ValueError('%s is not an update url' % update_url)
138 return parts.groups()
139
140
Richard Barnette3e8b2282018-05-15 20:42:20 +0000141def _list_image_dir_contents(update_url):
Prashanth B32baa9b2014-03-13 13:23:01 -0700142 """Lists the contents of the devserver for a given build/update_url.
143
144 @param update_url: An update url. Eg: http://devserver:port/update/build.
145 """
146 if not update_url:
147 logging.warning('Need update_url to list contents of the devserver.')
148 return
149 error_msg = 'Cannot check contents of devserver, update url %s' % update_url
150 try:
151 devserver_url, build = _get_devserver_build_from_update_url(update_url)
152 except ValueError as e:
153 logging.warning('%s: %s', error_msg, e)
154 return
155 devserver = dev_server.ImageServer(devserver_url)
156 try:
157 devserver.list_image_dir(build)
158 # The devserver will retry on URLError to avoid flaky connections, but will
159 # eventually raise the URLError if it persists. All HTTPErrors get
160 # converted to DevServerExceptions.
161 except (dev_server.DevServerException, urllib2.URLError) as e:
162 logging.warning('%s: %s', error_msg, e)
163
164
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700165# TODO(garnold) This implements shared updater functionality needed for
166# supporting the autoupdate_EndToEnd server-side test. We should probably
167# migrate more of the existing ChromiumOSUpdater functionality to it as we
168# expand non-CrOS support in other tests.
Richard Barnette3e8b2282018-05-15 20:42:20 +0000169class ChromiumOSUpdater(object):
170 """Chromium OS specific DUT update functionality."""
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700171
Richard Barnette3e8b2282018-05-15 20:42:20 +0000172 def __init__(self, update_url, host=None, interactive=True):
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700173 """Initializes the object.
174
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700175 @param update_url: The URL we want the update to use.
176 @param host: A client.common_lib.hosts.Host implementation.
David Haddock76a4c882017-12-13 18:50:09 -0800177 @param interactive: Bool whether we are doing an interactive update.
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700178 """
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700179 self.update_url = update_url
180 self.host = host
David Haddock76a4c882017-12-13 18:50:09 -0800181 self.interactive = interactive
Richard Barnette3e8b2282018-05-15 20:42:20 +0000182 self.update_version = _url_to_version(update_url)
183
184
185 def _run(self, cmd, *args, **kwargs):
186 """Abbreviated form of self.host.run(...)"""
187 return self.host.run(cmd, *args, **kwargs)
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700188
189
190 def check_update_status(self):
191 """Returns the current update engine state.
192
193 We use the `update_engine_client -status' command and parse the line
194 indicating the update state, e.g. "CURRENT_OP=UPDATE_STATUS_IDLE".
195 """
Luigi Semenzatof15c8fc2017-03-03 14:12:40 -0800196 update_status = self.host.run(command='%s -status | grep CURRENT_OP' %
Richard Barnette3e8b2282018-05-15 20:42:20 +0000197 _UPDATER_BIN)
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700198 return update_status.stdout.strip().split('=')[-1]
199
200
Richard Barnette55d1af82018-05-22 23:40:14 +0000201 def _rootdev(self, options=''):
202 """Returns the stripped output of rootdev <options>.
203
204 @param options: options to run rootdev.
205
206 """
207 return self._run('rootdev %s' % options).stdout.strip()
208
209
210 def get_kernel_state(self):
211 """Returns the (<active>, <inactive>) kernel state as a pair."""
212 active_root = int(re.findall('\d+\Z', self._rootdev('-s'))[0])
213 if active_root == _KERNEL_A['root']:
214 return _KERNEL_A, _KERNEL_B
215 elif active_root == _KERNEL_B['root']:
216 return _KERNEL_B, _KERNEL_A
217 else:
218 raise ChromiumOSError('Encountered unknown root partition: %s' %
219 active_root)
220
221
222 def _cgpt(self, flag, kernel, dev='$(rootdev -s -d)'):
223 """Return numeric cgpt value for the specified flag, kernel, device. """
224 return int(self._run('cgpt show -n -i %d %s %s' % (
225 kernel['kernel'], flag, dev)).stdout.strip())
226
227
228 def _get_next_kernel(self):
229 """Return the kernel that has priority for the next boot."""
230 priority_a = self._cgpt('-P', _KERNEL_A)
231 priority_b = self._cgpt('-P', _KERNEL_B)
232 if priority_a > priority_b:
233 return _KERNEL_A
234 else:
235 return _KERNEL_B
236
237
238 def _get_kernel_success(self, kernel):
239 """Return boolean success flag for the specified kernel.
240
241 @param kernel: information of the given kernel, either _KERNEL_A
242 or _KERNEL_B.
243 """
244 return self._cgpt('-S', kernel) != 0
245
246
247 def _get_kernel_tries(self, kernel):
248 """Return tries count for the specified kernel.
249
250 @param kernel: information of the given kernel, either _KERNEL_A
251 or _KERNEL_B.
252 """
253 return self._cgpt('-T', kernel)
254
255
Richard Barnette3e8b2282018-05-15 20:42:20 +0000256 def _get_last_update_error(self):
Shuqian Zhaod9992722016-02-29 12:26:38 -0800257 """Get the last autoupdate error code."""
Richard Barnette3e8b2282018-05-15 20:42:20 +0000258 command_result = self._run(
259 '%s --last_attempt_error' % _UPDATER_BIN)
260 return command_result.stdout.strip().replace('\n', ', ')
Shuqian Zhaod9992722016-02-29 12:26:38 -0800261
262
Luigi Semenzatoe76d9f82016-11-21 11:15:10 -0800263 def _base_update_handler_no_retry(self, run_args):
Shuqian Zhaod9992722016-02-29 12:26:38 -0800264 """Base function to handle a remote update ssh call.
265
266 @param run_args: Dictionary of args passed to ssh_host.run function.
Shuqian Zhaod9992722016-02-29 12:26:38 -0800267
Luigi Semenzatoe76d9f82016-11-21 11:15:10 -0800268 @throws: intercepts and re-throws all exceptions
Shuqian Zhaod9992722016-02-29 12:26:38 -0800269 """
Shuqian Zhaod9992722016-02-29 12:26:38 -0800270 try:
271 self.host.run(**run_args)
Shuqian Zhaod9992722016-02-29 12:26:38 -0800272 except Exception as e:
Luigi Semenzatoe76d9f82016-11-21 11:15:10 -0800273 logging.debug('exception in update handler: %s', e)
274 raise e
Shuqian Zhaod9992722016-02-29 12:26:38 -0800275
Luigi Semenzatoe76d9f82016-11-21 11:15:10 -0800276
277 def _base_update_handler(self, run_args, err_msg_prefix=None):
278 """Handle a remote update ssh call, possibly with retries.
279
280 @param run_args: Dictionary of args passed to ssh_host.run function.
281 @param err_msg_prefix: Prefix of the exception error message.
282 """
283 def exception_handler(e):
284 """Examines exceptions and returns True if the update handler
285 should be retried.
286
287 @param e: the exception intercepted by the retry util.
288 """
289 return (isinstance(e, error.AutoservSSHTimeout) or
290 (isinstance(e, error.GenericHostRunError) and
291 hasattr(e, 'description') and
292 (re.search('ERROR_CODE=37', e.description) or
293 re.search('generic error .255.', e.description))))
294
295 try:
296 # Try the update twice (arg 2 is max_retry, not including the first
297 # call). Some exceptions may be caught by the retry handler.
298 retry_util.GenericRetry(exception_handler, 1,
299 self._base_update_handler_no_retry,
300 run_args)
301 except Exception as e:
302 message = err_msg_prefix + ': ' + str(e)
303 raise RootFSUpdateError(message)
Shuqian Zhaod9992722016-02-29 12:26:38 -0800304
305
Luigi Semenzatof15c8fc2017-03-03 14:12:40 -0800306 def _wait_for_update_service(self):
307 """Ensure that the update engine daemon is running, possibly
308 by waiting for it a bit in case the DUT just rebooted and the
309 service hasn't started yet.
310 """
311 def handler(e):
312 """Retry exception handler.
313
314 Assumes that the error is due to the update service not having
315 started yet.
316
317 @param e: the exception intercepted by the retry util.
318 """
319 if isinstance(e, error.AutoservRunError):
320 logging.debug('update service check exception: %s\n'
321 'retrying...', e)
322 return True
323 else:
324 return False
325
326 # Retry at most three times, every 5s.
327 status = retry_util.GenericRetry(handler, 3,
328 self.check_update_status,
329 sleep=5)
330
331 # Expect the update engine to be idle.
332 if status != UPDATER_IDLE:
333 raise ChromiumOSError('%s is not in an installable state' %
334 self.host.hostname)
335
336
Richard Barnette55d1af82018-05-22 23:40:14 +0000337 def _reset_update_engine(self):
338 """Resets the host to prepare for a clean update regardless of state."""
339 self._run('stop ui || true')
340 self._run('stop update-engine || true')
341 self._run('start update-engine')
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700342
Richard Barnette55d1af82018-05-22 23:40:14 +0000343 # Wait for update engine to be ready.
Luigi Semenzatof15c8fc2017-03-03 14:12:40 -0800344 self._wait_for_update_service()
345
Richard Barnette55d1af82018-05-22 23:40:14 +0000346
347 def _reset_stateful_partition(self):
348 """Clear any pending stateful update request."""
349 statefuldev_cmd = [self.get_stateful_update_script()]
350 statefuldev_cmd += ['--stateful_change=reset', '2>&1']
351 self._run(' '.join(statefuldev_cmd))
352
353
354 def _revert_boot_partition(self):
355 """Revert the boot partition."""
356 part = self._rootdev('-s')
357 logging.warning('Reverting update; Boot partition will be %s', part)
358 return self._run('/postinst %s 2>&1' % part)
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700359
360
Allen Lib5420a72017-06-20 14:14:07 -0700361 def _get_metric_fields(self):
362 """Return a dict of metric fields.
363
364 This is used for sending autoupdate metrics for this instance.
365 """
366 build_name = url_to_image_name(self.update_url)
367 try:
368 board, build_type, milestone, _ = server_utils.ParseBuildName(
369 build_name)
370 except server_utils.ParseBuildNameException:
371 logging.warning('Unable to parse build name %s for metrics. '
372 'Continuing anyway.', build_name)
373 board, build_type, milestone = ('', '', '')
374 return {
375 'dev_server': dev_server.get_hostname(self.update_url),
376 'board': board,
377 'build_type': build_type,
378 'milestone': milestone,
379 }
380
381
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700382 def _verify_update_completed(self):
383 """Verifies that an update has completed.
384
385 @raise RootFSUpdateError: if verification fails.
386 """
387 status = self.check_update_status()
388 if status != UPDATER_NEED_REBOOT:
Shuqian Zhaod9992722016-02-29 12:26:38 -0800389 error_msg = ''
390 if status == UPDATER_IDLE:
Richard Barnette3e8b2282018-05-15 20:42:20 +0000391 error_msg = 'Update error: %s' % self._get_last_update_error()
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700392 raise RootFSUpdateError('Update did not complete with correct '
Shuqian Zhaod9992722016-02-29 12:26:38 -0800393 'status. Expecting %s, actual %s. %s' %
394 (UPDATER_NEED_REBOOT, status, error_msg))
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700395
396
Richard Barnette55d1af82018-05-22 23:40:14 +0000397 def trigger_update(self):
398 """Triggers a background update.
399
400 @raise RootFSUpdateError or unknown Exception if anything went wrong.
401 """
402 # If this function is called immediately after reboot (which it is at
403 # this time), there is no guarantee that the update service is up and
404 # running yet, so wait for it.
405 self._wait_for_update_service()
406
407 autoupdate_cmd = ('%s --check_for_update --omaha_url=%s' %
408 (_UPDATER_BIN, self.update_url))
409 run_args = {'command': autoupdate_cmd}
410 err_prefix = 'Failed to trigger an update on %s. ' % self.host.hostname
411 logging.info('Triggering update via: %s', autoupdate_cmd)
412 metric_fields = {'success': False}
413 try:
414 self._base_update_handler(run_args, err_prefix)
415 metric_fields['success'] = True
416 finally:
417 c = metrics.Counter('chromeos/autotest/autoupdater/trigger')
418 metric_fields.update(self._get_metric_fields())
419 c.increment(fields=metric_fields)
420
421
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700422 def update_image(self):
423 """Updates the device image and verifies success."""
Shuqian Zhaofe4d62e2016-06-23 14:46:45 -0700424 autoupdate_cmd = ('%s --update --omaha_url=%s' %
Richard Barnette3e8b2282018-05-15 20:42:20 +0000425 (_UPDATER_BIN, self.update_url))
David Haddock76a4c882017-12-13 18:50:09 -0800426 if not self.interactive:
427 autoupdate_cmd = '%s --interactive=false' % autoupdate_cmd
Shuqian Zhaod9992722016-02-29 12:26:38 -0800428 run_args = {'command': autoupdate_cmd, 'timeout': 3600}
429 err_prefix = ('Failed to install device image using payload at %s '
430 'on %s. ' % (self.update_url, self.host.hostname))
431 logging.info('Updating image via: %s', autoupdate_cmd)
Allen Li1a5cc0a2017-06-20 14:08:59 -0700432 metric_fields = {'success': False}
Luigi Semenzatoe76d9f82016-11-21 11:15:10 -0800433 try:
Luigi Semenzatoe76d9f82016-11-21 11:15:10 -0800434 self._base_update_handler(run_args, err_prefix)
Allen Li1a5cc0a2017-06-20 14:08:59 -0700435 metric_fields['success'] = True
436 finally:
Allen Li1a5cc0a2017-06-20 14:08:59 -0700437 c = metrics.Counter('chromeos/autotest/autoupdater/update')
Allen Lib5420a72017-06-20 14:14:07 -0700438 metric_fields.update(self._get_metric_fields())
Allen Li1a5cc0a2017-06-20 14:08:59 -0700439 c.increment(fields=metric_fields)
Aviv Keshetf37b2d72016-06-01 19:27:59 -0700440 self._verify_update_completed()
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700441
442
Chris Sosa5e4246b2012-05-22 18:05:22 -0700443 def get_stateful_update_script(self):
Gwendal Grignou3e96cc22017-06-07 16:22:51 -0700444 """Returns the path to the stateful update script on the target.
Chris Sosa5e4246b2012-05-22 18:05:22 -0700445
Gwendal Grignou3e96cc22017-06-07 16:22:51 -0700446 When runnning test_that, stateful_update is in chroot /usr/sbin,
447 as installed by chromeos-base/devserver packages.
448 In the lab, it is installed with the python module devserver, by
449 build_externals.py command.
Chris Sosaa3ac2152012-05-23 22:23:13 -0700450
Gwendal Grignou3e96cc22017-06-07 16:22:51 -0700451 If we can find it, we hope it exists already on the DUT, we assert
452 otherwise.
453 """
Richard Barnette3e8b2282018-05-15 20:42:20 +0000454 stateful_update_file = os.path.join(_STATEFUL_UPDATE_PATH,
455 _STATEFUL_UPDATE_SCRIPT)
Gwendal Grignou3e96cc22017-06-07 16:22:51 -0700456 if os.path.exists(stateful_update_file):
Chris Sosa5e4246b2012-05-22 18:05:22 -0700457 self.host.send_file(
Richard Barnette3e8b2282018-05-15 20:42:20 +0000458 stateful_update_file, _REMOTE_TMP_STATEFUL_UPDATE,
Gilad Arnold0c0df732015-09-21 06:37:59 -0700459 delete_dest=True)
Richard Barnette3e8b2282018-05-15 20:42:20 +0000460 return _REMOTE_TMP_STATEFUL_UPDATE
Chris Sosa5e4246b2012-05-22 18:05:22 -0700461
Richard Barnette3e8b2282018-05-15 20:42:20 +0000462 if self.host.path_exists(_REMOTE_STATEFUL_UPDATE_PATH):
Gwendal Grignou3e96cc22017-06-07 16:22:51 -0700463 logging.warning('Could not chroot %s script, falling back on %s',
Richard Barnette3e8b2282018-05-15 20:42:20 +0000464 _STATEFUL_UPDATE_SCRIPT,
465 _REMOTE_STATEFUL_UPDATE_PATH)
466 return _REMOTE_STATEFUL_UPDATE_PATH
Gwendal Grignou3e96cc22017-06-07 16:22:51 -0700467 else:
Richard Barnette3e8b2282018-05-15 20:42:20 +0000468 raise ChromiumOSError('Could not locate %s' %
469 _STATEFUL_UPDATE_SCRIPT)
Chris Sosa5e4246b2012-05-22 18:05:22 -0700470
471
Chris Sosac1932172013-10-16 13:28:53 -0700472 def rollback_rootfs(self, powerwash):
473 """Triggers rollback and waits for it to complete.
474
475 @param powerwash: If true, powerwash as part of rollback.
476
477 @raise RootFSUpdateError if anything went wrong.
478
479 """
Dan Shi549fb822015-03-24 18:01:11 -0700480 version = self.host.get_release_version()
Chris Sosac8617522014-06-09 23:22:26 +0000481 # Introduced can_rollback in M36 (build 5772). # etc/lsb-release matches
482 # X.Y.Z. This version split just pulls the first part out.
483 try:
484 build_number = int(version.split('.')[0])
485 except ValueError:
486 logging.error('Could not parse build number.')
487 build_number = 0
488
489 if build_number >= 5772:
Richard Barnette3e8b2282018-05-15 20:42:20 +0000490 can_rollback_cmd = '%s --can_rollback' % _UPDATER_BIN
Chris Sosac8617522014-06-09 23:22:26 +0000491 logging.info('Checking for rollback.')
492 try:
493 self._run(can_rollback_cmd)
494 except error.AutoservRunError as e:
495 raise RootFSUpdateError("Rollback isn't possible on %s: %s" %
496 (self.host.hostname, str(e)))
497
Richard Barnette3e8b2282018-05-15 20:42:20 +0000498 rollback_cmd = '%s --rollback --follow' % _UPDATER_BIN
Chris Sosac1932172013-10-16 13:28:53 -0700499 if not powerwash:
Dan Shif3a35f72016-01-25 11:18:14 -0800500 rollback_cmd += ' --nopowerwash'
Chris Sosac1932172013-10-16 13:28:53 -0700501
Chris Sosac8617522014-06-09 23:22:26 +0000502 logging.info('Performing rollback.')
Chris Sosac1932172013-10-16 13:28:53 -0700503 try:
504 self._run(rollback_cmd)
Chris Sosac1932172013-10-16 13:28:53 -0700505 except error.AutoservRunError as e:
506 raise RootFSUpdateError('Rollback failed on %s: %s' %
507 (self.host.hostname, str(e)))
508
509 self._verify_update_completed()
510
Gilad Arnold0ed760c2012-11-05 23:42:53 -0800511
Chris Sosa72312602013-04-16 15:01:56 -0700512 def update_stateful(self, clobber=True):
513 """Updates the stateful partition.
514
515 @param clobber: If True, a clean stateful installation.
516 """
Chris Sosa77556d82012-04-05 15:23:14 -0700517 logging.info('Updating stateful partition...')
joychen03eaad92013-06-26 09:55:21 -0700518 statefuldev_url = self.update_url.replace('update',
519 'static')
Chris Sosaa3ac2152012-05-23 22:23:13 -0700520
Dale Curtis5c32c722011-05-04 19:24:23 -0700521 # Attempt stateful partition update; this must succeed so that the newly
522 # installed host is testable after update.
Chris Sosa72312602013-04-16 15:01:56 -0700523 statefuldev_cmd = [self.get_stateful_update_script(), statefuldev_url]
524 if clobber:
525 statefuldev_cmd.append('--stateful_change=clean')
526
527 statefuldev_cmd.append('2>&1')
Dale Curtis5c32c722011-05-04 19:24:23 -0700528 try:
Dan Shi205b8732016-01-25 10:56:22 -0800529 self._run(' '.join(statefuldev_cmd), timeout=1200)
Dale Curtis5c32c722011-05-04 19:24:23 -0700530 except error.AutoservRunError:
Gilad Arnold62cf3a42015-10-01 09:15:25 -0700531 update_error = StatefulUpdateError(
532 'Failed to perform stateful update on %s' %
533 self.host.hostname)
Chris Sosa77556d82012-04-05 15:23:14 -0700534 raise update_error
Dale Curtis5c32c722011-05-04 19:24:23 -0700535
Chris Sosaa3ac2152012-05-23 22:23:13 -0700536
Richard Barnette54d14f52018-05-18 16:39:49 +0000537 def verify_boot_expectations(self, expected_kernel, rollback_message):
Richard Barnette55d1af82018-05-22 23:40:14 +0000538 """Verifies that we fully booted given expected kernel state.
539
540 This method both verifies that we booted using the correct kernel
541 state and that the OS has marked the kernel as good.
542
Richard Barnette54d14f52018-05-18 16:39:49 +0000543 @param expected_kernel: kernel that we are verifying with,
Richard Barnette55d1af82018-05-22 23:40:14 +0000544 i.e. I expect to be booted onto partition 4 etc. See output of
545 get_kernel_state.
546 @param rollback_message: string to raise as a ChromiumOSError
547 if we booted with the wrong partition.
548
549 @raises ChromiumOSError: If we didn't.
550 """
551 # Figure out the newly active kernel.
Richard Barnette54d14f52018-05-18 16:39:49 +0000552 active_kernel = self.get_kernel_state()[0]
Richard Barnette55d1af82018-05-22 23:40:14 +0000553
554 # Check for rollback due to a bad build.
Richard Barnette54d14f52018-05-18 16:39:49 +0000555 if active_kernel != expected_kernel:
Richard Barnette55d1af82018-05-22 23:40:14 +0000556
557 # Kernel crash reports should be wiped between test runs, but
558 # may persist from earlier parts of the test, or from problems
559 # with provisioning.
560 #
561 # Kernel crash reports will NOT be present if the crash happened
562 # before encrypted stateful is mounted.
563 #
564 # TODO(dgarrett): Integrate with server/crashcollect.py at some
565 # point.
566 kernel_crashes = glob.glob('/var/spool/crash/kernel.*.kcrash')
567 if kernel_crashes:
568 rollback_message += ': kernel_crash'
569 logging.debug('Found %d kernel crash reports:',
570 len(kernel_crashes))
571 # The crash names contain timestamps that may be useful:
572 # kernel.20131207.005945.0.kcrash
573 for crash in kernel_crashes:
574 logging.debug(' %s', os.path.basename(crash))
575
576 # Print out some information to make it easier to debug
577 # the rollback.
578 logging.debug('Dumping partition table.')
579 self._run('cgpt show $(rootdev -s -d)')
580 logging.debug('Dumping crossystem for firmware debugging.')
581 self._run('crossystem --all')
582 raise ChromiumOSError(rollback_message)
583
584 # Make sure chromeos-setgoodkernel runs.
585 try:
586 utils.poll_for_condition(
Richard Barnette54d14f52018-05-18 16:39:49 +0000587 lambda: (self._get_kernel_tries(active_kernel) == 0
588 and self._get_kernel_success(active_kernel)),
Richard Barnette55d1af82018-05-22 23:40:14 +0000589 exception=ChromiumOSError(),
590 timeout=_KERNEL_UPDATE_TIMEOUT, sleep_interval=5)
591 except ChromiumOSError:
592 services_status = self._run('status system-services').stdout
593 if services_status != 'system-services start/running\n':
594 event = ('Chrome failed to reach login screen')
595 else:
596 event = ('update-engine failed to call '
597 'chromeos-setgoodkernel')
598 raise ChromiumOSError(
599 'After update and reboot, %s '
600 'within %d seconds' % (event, _KERNEL_UPDATE_TIMEOUT))
601
602
Richard Barnette54d14f52018-05-18 16:39:49 +0000603 def _install_update(self):
Richard Barnette0beb14b2018-05-15 18:07:52 +0000604 """Install the requested image on the DUT, but don't start it.
605
606 This downloads all content needed for the requested update, and
607 installs it in place on the DUT. This does not reboot the DUT,
608 so the update is merely pending when the function returns.
Dan Shi0f466e82013-02-22 15:44:58 -0800609 """
Dan Shi549fb822015-03-24 18:01:11 -0700610 booted_version = self.host.get_release_version()
Richard Barnette0173ea82018-05-04 21:13:57 +0000611 logging.info('Updating from version %s to %s.',
612 booted_version, self.update_version)
Dale Curtis53d55862011-05-16 12:17:59 -0700613
Dale Curtis5c32c722011-05-04 19:24:23 -0700614 # Check that Dev Server is accepting connections (from autoserv's host).
Sean O'Connor5346e4e2010-08-12 18:49:24 +0200615 # If we can't talk to it, the machine host probably can't either.
xixuanccf2e722016-06-10 16:42:38 -0700616 auserver_host = 'http://%s' % urlparse.urlparse(self.update_url)[1]
Sean O'Connor5346e4e2010-08-12 18:49:24 +0200617 try:
xixuanccf2e722016-06-10 16:42:38 -0700618 if not dev_server.ImageServer.devserver_healthy(auserver_host):
619 raise ChromiumOSError(
620 'Update server at %s not healthy' % auserver_host)
621 except Exception as e:
622 logging.debug('Error happens in connection to devserver: %r', e)
Dale Curtis5c32c722011-05-04 19:24:23 -0700623 raise ChromiumOSError(
624 'Update server at %s not available' % auserver_host)
Sean O'Connor5346e4e2010-08-12 18:49:24 +0200625
Chris Sosaa3ac2152012-05-23 22:23:13 -0700626 logging.info('Installing from %s to %s', self.update_url,
Chris Sosa77556d82012-04-05 15:23:14 -0700627 self.host.hostname)
628
Chris Sosa5e4246b2012-05-22 18:05:22 -0700629 # Reset update state.
Richard Barnette3e8b2282018-05-15 20:42:20 +0000630 self._reset_update_engine()
631 self._reset_stateful_partition()
Sean Oc053dfe2010-08-23 18:22:26 +0200632
Dale Curtis1e973182011-07-12 18:21:36 -0700633 try:
Aviv Keshet2610d3e2016-06-01 16:37:01 -0700634 try:
Richard Barnette54d14f52018-05-18 16:39:49 +0000635 self.update_image()
Aviv Keshet2610d3e2016-06-01 16:37:01 -0700636 self.update_stateful()
637 except:
Richard Barnette3e8b2282018-05-15 20:42:20 +0000638 self._revert_boot_partition()
639 self._reset_stateful_partition()
Aviv Keshet2610d3e2016-06-01 16:37:01 -0700640 raise
Sean Oc053dfe2010-08-23 18:22:26 +0200641
Dale Curtis1e973182011-07-12 18:21:36 -0700642 logging.info('Update complete.')
Dale Curtis1e973182011-07-12 18:21:36 -0700643 except:
644 # Collect update engine logs in the event of failure.
645 if self.host.job:
Aviv Keshet2610d3e2016-06-01 16:37:01 -0700646 logging.info('Collecting update engine logs due to failure...')
Dale Curtis1e973182011-07-12 18:21:36 -0700647 self.host.get_file(
Richard Barnette3e8b2282018-05-15 20:42:20 +0000648 _UPDATER_LOGS, self.host.job.sysinfo.sysinfodir,
Gilad Arnold0c0df732015-09-21 06:37:59 -0700649 preserve_perm=False)
Richard Barnette3e8b2282018-05-15 20:42:20 +0000650 _list_image_dir_contents(self.update_url)
Dale Curtis1e973182011-07-12 18:21:36 -0700651 raise
Dan Shi10e992b2013-08-30 11:02:59 -0700652 finally:
Shuqian Zhaod9992722016-02-29 12:26:38 -0800653 logging.info('Update engine log has downloaded in '
654 'sysinfo/update_engine dir. Check the lastest.')
Sean O'Connor5346e4e2010-08-12 18:49:24 +0200655
656
Richard Barnette0beb14b2018-05-15 18:07:52 +0000657 def _post_update_processing(self, expected_kernel):
658 """After the DUT is updated, confirm machine_install succeeded.
659
660 @param updater: ChromiumOSUpdater instance used to update the DUT.
661 @param expected_kernel: kernel expected to be active after reboot,
662 or `None` to skip rollback checking.
663
664 """
665 # Touch the lab machine file to leave a marker that
666 # distinguishes this image from other test images.
667 # Afterwards, we must re-run the autoreboot script because
668 # it depends on the _LAB_MACHINE_FILE.
669 autoreboot_cmd = ('FILE="%s" ; [ -f "$FILE" ] || '
670 '( touch "$FILE" ; start autoreboot )')
Richard Barnette3e8b2282018-05-15 20:42:20 +0000671 self._run(autoreboot_cmd % _LAB_MACHINE_FILE)
Richard Barnette0beb14b2018-05-15 18:07:52 +0000672 self.verify_boot_expectations(
673 expected_kernel, rollback_message=
674 'Build %s failed to boot on %s; system rolled back to previous '
675 'build' % (self.update_version, self.host.hostname))
676
677 logging.debug('Cleaning up old autotest directories.')
678 try:
679 installed_autodir = autotest.Autotest.get_installed_autodir(
680 self.host)
681 self._run('rm -rf ' + installed_autodir)
682 except autotest.AutodirNotFoundError:
683 logging.debug('No autotest installed directory found.')
684
685
Richard Barnette54d14f52018-05-18 16:39:49 +0000686 def run_update(self):
Richard Barnette0beb14b2018-05-15 18:07:52 +0000687 """Perform a full update of a DUT in the test lab.
688
689 This downloads and installs the root FS and stateful partition
690 content needed for the update specified in `self.host` and
691 `self.update_url`. The update is performed according to the
692 requirements for provisioning a DUT for testing the requested
693 build.
694
Richard Barnette0beb14b2018-05-15 18:07:52 +0000695 @returns A tuple of the form `(image_name, attributes)`, where
696 `image_name` is the name of the image installed, and
697 `attributes` is new attributes to be applied to the DUT.
698 """
699 logging.debug('Update URL is %s', self.update_url)
700
701 # Report provision stats.
702 server_name = dev_server.get_hostname(self.update_url)
703 (metrics.Counter('chromeos/autotest/provision/install')
704 .increment(fields={'devserver': server_name}))
705
706 # Create a file to indicate if provision fails. The file will be
707 # removed by any successful update.
708 self._run('touch %s' % PROVISION_FAILED)
709
Richard Barnette54d14f52018-05-18 16:39:49 +0000710 self.host.reboot(timeout=self.host.REBOOT_TIMEOUT)
711 self.host.prepare_for_update()
Richard Barnette0beb14b2018-05-15 18:07:52 +0000712
Richard Barnette54d14f52018-05-18 16:39:49 +0000713 self._install_update()
Richard Barnette0beb14b2018-05-15 18:07:52 +0000714
Richard Barnette54d14f52018-05-18 16:39:49 +0000715 # Give it some time in case of IO issues.
716 time.sleep(10)
Richard Barnette0beb14b2018-05-15 18:07:52 +0000717
Richard Barnette54d14f52018-05-18 16:39:49 +0000718 inactive_kernel = self.get_kernel_state()[1]
719 next_kernel = self._get_next_kernel()
720 if next_kernel != inactive_kernel:
721 raise ChromiumOSError(
722 'Update failed. The kernel for next boot is %s, '
723 'but %s was expected.' %
724 (next_kernel['name'], inactive_kernel['name']))
Richard Barnette0beb14b2018-05-15 18:07:52 +0000725
Richard Barnette54d14f52018-05-18 16:39:49 +0000726 # Update has returned successfully; reboot the host.
727 #
728 # Regarding the 'crossystem' command below: In some cases,
729 # the update flow puts the TPM into a state such that it
730 # fails verification. We don't know why. However, this
731 # call papers over the problem by clearing the TPM during
732 # the reboot.
733 #
734 # We ignore failures from 'crossystem'. Although failure
735 # here is unexpected, and could signal a bug, the point of
736 # the exercise is to paper over problems; allowing this to
737 # fail would defeat the purpose.
738 self._run('crossystem clear_tpm_owner_request=1',
739 ignore_status=True)
740 self.host.reboot(timeout=self.host.REBOOT_TIMEOUT)
Richard Barnette0beb14b2018-05-15 18:07:52 +0000741
742 self._post_update_processing(inactive_kernel)
743 image_name = url_to_image_name(self.update_url)
744 # update_url is different from devserver url needed to stage autotest
745 # packages, therefore, resolve a new devserver url here.
746 devserver_url = dev_server.ImageServer.resolve(
747 image_name, self.host.hostname).url()
748 repo_url = tools.get_package_url(devserver_url, image_name)
749 return image_name, {ds_constants.JOB_REPO_URL: repo_url}