blob: 74327c2d574aab4ba4dfba7d334598058629a418 [file] [log] [blame]
Chris Sosa5e4246b2012-05-22 18:05:22 -07001# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
Sean O'Connor5346e4e2010-08-12 18:49:24 +02002# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
Don Garrett56b1cc82013-12-06 17:49:20 -08005import glob
Sean O'Connor5346e4e2010-08-12 18:49:24 +02006import logging
Dale Curtis5c32c722011-05-04 19:24:23 -07007import os
Sean O'Connor5346e4e2010-08-12 18:49:24 +02008import re
Richard Barnette621a8e42018-06-25 17:34:11 -07009import time
Prashanth B32baa9b2014-03-13 13:23:01 -070010import urllib2
Richard Barnette0beb14b2018-05-15 18:07:52 +000011import urlparse
Sean O'Connor5346e4e2010-08-12 18:49:24 +020012
Chris Sosa65425082013-10-16 13:26:22 -070013from autotest_lib.client.bin import utils
Dale Curtis5c32c722011-05-04 19:24:23 -070014from autotest_lib.client.common_lib import error, global_config
Prashanth B32baa9b2014-03-13 13:23:01 -070015from autotest_lib.client.common_lib.cros import dev_server
Richard Barnette0beb14b2018-05-15 18:07:52 +000016from autotest_lib.server import autotest
Shelley Chen61d28982016-10-28 09:40:20 -070017from autotest_lib.server import utils as server_utils
Richard Barnette0beb14b2018-05-15 18:07:52 +000018from autotest_lib.server.cros.dynamic_suite import constants as ds_constants
19from autotest_lib.server.cros.dynamic_suite import tools
Luigi Semenzatoe76d9f82016-11-21 11:15:10 -080020from chromite.lib import retry_util
Dan Shif3a35f72016-01-25 11:18:14 -080021
Shelley Chen16b8df32016-10-27 16:24:21 -070022try:
23 from chromite.lib import metrics
Dan Shi5e2efb72017-02-07 11:40:23 -080024except ImportError:
25 metrics = utils.metrics_mock
Sean O'Connor5346e4e2010-08-12 18:49:24 +020026
Gwendal Grignou3e96cc22017-06-07 16:22:51 -070027
Richard Barnette621a8e42018-06-25 17:34:11 -070028def _metric_name(base_name):
29 return 'chromeos/autotest/provision/' + base_name
30
31
Dale Curtis5c32c722011-05-04 19:24:23 -070032# Local stateful update path is relative to the CrOS source directory.
Sean O'Connor5346e4e2010-08-12 18:49:24 +020033UPDATER_IDLE = 'UPDATE_STATUS_IDLE'
Sean Oc053dfe2010-08-23 18:22:26 +020034UPDATER_NEED_REBOOT = 'UPDATE_STATUS_UPDATED_NEED_REBOOT'
beeps5e8c45a2013-12-17 22:05:11 -080035# A list of update engine client states that occur after an update is triggered.
36UPDATER_PROCESSING_UPDATE = ['UPDATE_STATUS_CHECKING_FORUPDATE',
37 'UPDATE_STATUS_UPDATE_AVAILABLE',
38 'UPDATE_STATUS_DOWNLOADING',
39 'UPDATE_STATUS_FINALIZING']
Sean O'Connor5346e4e2010-08-12 18:49:24 +020040
Richard Barnette0beb14b2018-05-15 18:07:52 +000041
Richard Barnette3e8b2282018-05-15 20:42:20 +000042_STATEFUL_UPDATE_SCRIPT = 'stateful_update'
Richard Barnettee86b1ce2018-06-07 10:37:23 -070043_QUICK_PROVISION_SCRIPT = 'quick-provision'
Richard Barnette3e8b2282018-05-15 20:42:20 +000044
45_UPDATER_BIN = '/usr/bin/update_engine_client'
46_UPDATER_LOGS = ['/var/log/messages', '/var/log/update_engine']
47
48_KERNEL_A = {'name': 'KERN-A', 'kernel': 2, 'root': 3}
49_KERNEL_B = {'name': 'KERN-B', 'kernel': 4, 'root': 5}
50
51# Time to wait for new kernel to be marked successful after
52# auto update.
53_KERNEL_UPDATE_TIMEOUT = 120
54
55
Richard Barnette0beb14b2018-05-15 18:07:52 +000056# PROVISION_FAILED - A flag file to indicate provision failures. The
57# file is created at the start of any AU procedure (see
Richard Barnette9d43e562018-06-05 17:20:10 +000058# `ChromiumOSUpdater._prepare_host()`). The file's location in
Richard Barnette0beb14b2018-05-15 18:07:52 +000059# stateful means that on successul update it will be removed. Thus, if
60# this file exists, it indicates that we've tried and failed in a
61# previous attempt to update.
62PROVISION_FAILED = '/var/tmp/provision_failed'
63
64
Richard Barnette3e8b2282018-05-15 20:42:20 +000065# A flag file used to enable special handling in lab DUTs. Some
66# parts of the system in Chromium OS test images will behave in ways
67# convenient to the test lab when this file is present. Generally,
68# we create this immediately after any update completes.
69_LAB_MACHINE_FILE = '/mnt/stateful_partition/.labmachine'
70
71
Richard Barnette9d43e562018-06-05 17:20:10 +000072class RootFSUpdateError(error.TestFail):
Chris Sosa77556d82012-04-05 15:23:14 -070073 """Raised when the RootFS fails to update."""
Chris Sosa77556d82012-04-05 15:23:14 -070074
75
Richard Barnette9d43e562018-06-05 17:20:10 +000076class StatefulUpdateError(error.TestFail):
Chris Sosa77556d82012-04-05 15:23:14 -070077 """Raised when the stateful partition fails to update."""
Chris Sosa77556d82012-04-05 15:23:14 -070078
79
Richard Barnette9d43e562018-06-05 17:20:10 +000080class _AttributedUpdateError(error.TestFail):
81 """Update failure with an attributed cause."""
82
83 def __init__(self, attribution, msg):
84 super(_AttributedUpdateError, self).__init__(
85 '%s: %s' % (attribution, msg))
86
87
88class HostUpdateError(_AttributedUpdateError):
89 """Failure updating a DUT attributable to the DUT.
90
91 This class of exception should be raised when the most likely cause
92 of failure was a condition existing on the DUT prior to the update,
93 such as a hardware problem, or a bug in the software on the DUT.
94 """
95
96 def __init__(self, hostname, msg):
97 super(HostUpdateError, self).__init__(
98 'Error on %s prior to update' % hostname, msg)
99
Richard Barnette621a8e42018-06-25 17:34:11 -0700100 @property
101 def failure_summary(self):
102 #pylint: disable=missing-docstring
103 return 'DUT failed prior to update'
104
Richard Barnette9d43e562018-06-05 17:20:10 +0000105
106class DevServerError(_AttributedUpdateError):
107 """Failure updating a DUT attributable to the devserver.
108
109 This class of exception should be raised when the most likely cause
110 of failure was the devserver serving the target image for update.
111 """
112
113 def __init__(self, devserver, msg):
114 super(DevServerError, self).__init__(
115 'Devserver error on %s' % devserver, msg)
116
Richard Barnette621a8e42018-06-25 17:34:11 -0700117 @property
118 def failure_summary(self):
119 #pylint: disable=missing-docstring
120 return 'Devserver failed prior to update'
121
Richard Barnette9d43e562018-06-05 17:20:10 +0000122
123class ImageInstallError(_AttributedUpdateError):
124 """Failure updating a DUT when installing from the devserver.
125
126 This class of exception should be raised when the target DUT fails
127 to download and install the target image from the devserver, and
128 either the devserver or the DUT might be at fault.
129 """
130
131 def __init__(self, hostname, devserver, msg):
132 super(ImageInstallError, self).__init__(
133 'Download and install failed from %s onto %s'
134 % (devserver, hostname), msg)
135
Richard Barnette621a8e42018-06-25 17:34:11 -0700136 @property
137 def failure_summary(self):
138 #pylint: disable=missing-docstring
139 return 'Image failed to download and install'
140
Richard Barnette9d43e562018-06-05 17:20:10 +0000141
142class NewBuildUpdateError(_AttributedUpdateError):
143 """Failure updating a DUT attributable to the target build.
144
145 This class of exception should be raised when updating to a new
146 build fails, and the most likely cause of the failure is a bug in
147 the newly installed target build.
148 """
149
150 def __init__(self, update_version, msg):
151 super(NewBuildUpdateError, self).__init__(
152 'Failure in build %s' % update_version, msg)
153
Richard Barnette621a8e42018-06-25 17:34:11 -0700154 @property
155 def failure_summary(self):
156 #pylint: disable=missing-docstring
157 return 'Build failed to work after installing'
158
Richard Barnette9d43e562018-06-05 17:20:10 +0000159
Richard Barnette3e8b2282018-05-15 20:42:20 +0000160def _url_to_version(update_url):
Dan Shi0f466e82013-02-22 15:44:58 -0800161 """Return the version based on update_url.
162
163 @param update_url: url to the image to update to.
164
165 """
Dale Curtisddfdb942011-07-14 13:59:24 -0700166 # The Chrome OS version is generally the last element in the URL. The only
167 # exception is delta update URLs, which are rooted under the version; e.g.,
168 # http://.../update/.../0.14.755.0/au/0.14.754.0. In this case we want to
169 # strip off the au section of the path before reading the version.
Dan Shi5002cfc2013-04-29 10:45:05 -0700170 return re.sub('/au/.*', '',
171 urlparse.urlparse(update_url).path).split('/')[-1].strip()
Sean O'Connor5346e4e2010-08-12 18:49:24 +0200172
173
Scott Zawalskieadbf702013-03-14 09:23:06 -0400174def url_to_image_name(update_url):
175 """Return the image name based on update_url.
176
177 From a URL like:
178 http://172.22.50.205:8082/update/lumpy-release/R27-3837.0.0
179 return lumpy-release/R27-3837.0.0
180
181 @param update_url: url to the image to update to.
182 @returns a string representing the image name in the update_url.
183
184 """
185 return '/'.join(urlparse.urlparse(update_url).path.split('/')[-2:])
186
187
Prashanth B32baa9b2014-03-13 13:23:01 -0700188def _get_devserver_build_from_update_url(update_url):
189 """Get the devserver and build from the update url.
190
191 @param update_url: The url for update.
192 Eg: http://devserver:port/update/build.
193
194 @return: A tuple of (devserver url, build) or None if the update_url
195 doesn't match the expected pattern.
196
197 @raises ValueError: If the update_url doesn't match the expected pattern.
198 @raises ValueError: If no global_config was found, or it doesn't contain an
199 image_url_pattern.
200 """
201 pattern = global_config.global_config.get_config_value(
202 'CROS', 'image_url_pattern', type=str, default='')
203 if not pattern:
204 raise ValueError('Cannot parse update_url, the global config needs '
205 'an image_url_pattern.')
206 re_pattern = pattern.replace('%s', '(\S+)')
207 parts = re.search(re_pattern, update_url)
208 if not parts or len(parts.groups()) < 2:
209 raise ValueError('%s is not an update url' % update_url)
210 return parts.groups()
211
212
Richard Barnette3e8b2282018-05-15 20:42:20 +0000213def _list_image_dir_contents(update_url):
Prashanth B32baa9b2014-03-13 13:23:01 -0700214 """Lists the contents of the devserver for a given build/update_url.
215
216 @param update_url: An update url. Eg: http://devserver:port/update/build.
217 """
218 if not update_url:
219 logging.warning('Need update_url to list contents of the devserver.')
220 return
221 error_msg = 'Cannot check contents of devserver, update url %s' % update_url
222 try:
223 devserver_url, build = _get_devserver_build_from_update_url(update_url)
224 except ValueError as e:
225 logging.warning('%s: %s', error_msg, e)
226 return
227 devserver = dev_server.ImageServer(devserver_url)
228 try:
229 devserver.list_image_dir(build)
230 # The devserver will retry on URLError to avoid flaky connections, but will
231 # eventually raise the URLError if it persists. All HTTPErrors get
232 # converted to DevServerExceptions.
233 except (dev_server.DevServerException, urllib2.URLError) as e:
234 logging.warning('%s: %s', error_msg, e)
235
236
Richard Barnette621a8e42018-06-25 17:34:11 -0700237def _get_metric_fields(update_url):
238 """Return a dict of metric fields.
239
240 This is used for sending autoupdate metrics for the given update URL.
241
242 @param update_url Metrics fields will be calculated from this URL.
243 """
244 build_name = url_to_image_name(update_url)
245 try:
246 board, build_type, milestone, _ = server_utils.ParseBuildName(
247 build_name)
248 except server_utils.ParseBuildNameException:
249 logging.warning('Unable to parse build name %s for metrics. '
250 'Continuing anyway.', build_name)
251 board, build_type, milestone = ('', '', '')
252 return {
253 'dev_server': dev_server.get_resolved_hostname(update_url),
254 'board': board,
255 'build_type': build_type,
256 'milestone': milestone,
257 }
258
259
260def _emit_provision_metrics(update_url, dut_host_name,
261 failure_reason, duration):
262 """Send metrics for provision request."""
263 # The following is high cardinality, but sparse.
264 # Each DUT is of a single board type, and likely build type.
265 # The affinity also results in each DUT being attached to the same
266 # dev_server as well.
267 image_fields = _get_metric_fields(update_url)
268 fields = {
269 'board': image_fields['board'],
270 'build_type': image_fields['build_type'],
271 'dut_host_name': dut_host_name,
272 'dev_server': image_fields['dev_server'],
273 'success': not failure_reason,
274 }
275 build_name = url_to_image_name(update_url)
276
277 # reset_after=True is required for String gauges events to ensure that
278 # the metrics are not repeatedly emitted until the server restarts.
279
280 metrics.String(_metric_name('provision_build_by_devserver_dut'),
281 reset_after=True).set(build_name, fields=fields)
282 if failure_reason:
283 metrics.String(
284 _metric_name('provision_failure_reason_by_devserver_dut'),
285 reset_after=True).set(failure_reason, fields=fields)
286 metrics.SecondsDistribution(
287 _metric_name('provision_duration_by_devserver_dut')).add(
288 duration, fields=fields)
289
290
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700291# TODO(garnold) This implements shared updater functionality needed for
292# supporting the autoupdate_EndToEnd server-side test. We should probably
293# migrate more of the existing ChromiumOSUpdater functionality to it as we
294# expand non-CrOS support in other tests.
Richard Barnette3e8b2282018-05-15 20:42:20 +0000295class ChromiumOSUpdater(object):
296 """Chromium OS specific DUT update functionality."""
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700297
Richard Barnette3e8b2282018-05-15 20:42:20 +0000298 def __init__(self, update_url, host=None, interactive=True):
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700299 """Initializes the object.
300
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700301 @param update_url: The URL we want the update to use.
302 @param host: A client.common_lib.hosts.Host implementation.
David Haddock76a4c882017-12-13 18:50:09 -0800303 @param interactive: Bool whether we are doing an interactive update.
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700304 """
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700305 self.update_url = update_url
306 self.host = host
David Haddock76a4c882017-12-13 18:50:09 -0800307 self.interactive = interactive
Richard Barnette3e8b2282018-05-15 20:42:20 +0000308 self.update_version = _url_to_version(update_url)
309
310
311 def _run(self, cmd, *args, **kwargs):
312 """Abbreviated form of self.host.run(...)"""
313 return self.host.run(cmd, *args, **kwargs)
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700314
315
316 def check_update_status(self):
317 """Returns the current update engine state.
318
319 We use the `update_engine_client -status' command and parse the line
320 indicating the update state, e.g. "CURRENT_OP=UPDATE_STATUS_IDLE".
321 """
Luigi Semenzatof15c8fc2017-03-03 14:12:40 -0800322 update_status = self.host.run(command='%s -status | grep CURRENT_OP' %
Richard Barnette3e8b2282018-05-15 20:42:20 +0000323 _UPDATER_BIN)
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700324 return update_status.stdout.strip().split('=')[-1]
325
326
Richard Barnette55d1af82018-05-22 23:40:14 +0000327 def _rootdev(self, options=''):
328 """Returns the stripped output of rootdev <options>.
329
330 @param options: options to run rootdev.
331
332 """
333 return self._run('rootdev %s' % options).stdout.strip()
334
335
336 def get_kernel_state(self):
Richard Barnette9d43e562018-06-05 17:20:10 +0000337 """Returns the (<active>, <inactive>) kernel state as a pair.
338
339 @raise RootFSUpdateError if the DUT reports a root partition
340 number that isn't one of the known valid values.
341 """
Richard Barnette55d1af82018-05-22 23:40:14 +0000342 active_root = int(re.findall('\d+\Z', self._rootdev('-s'))[0])
343 if active_root == _KERNEL_A['root']:
344 return _KERNEL_A, _KERNEL_B
345 elif active_root == _KERNEL_B['root']:
346 return _KERNEL_B, _KERNEL_A
347 else:
Richard Barnette9d43e562018-06-05 17:20:10 +0000348 raise RootFSUpdateError(
349 'Encountered unknown root partition: %s' % active_root)
Richard Barnette55d1af82018-05-22 23:40:14 +0000350
351
Richard Barnette18fd5842018-05-25 18:21:14 +0000352 def _cgpt(self, flag, kernel):
353 """Return numeric cgpt value for the specified flag, kernel, device."""
354 return int(self._run('cgpt show -n -i %d %s $(rootdev -s -d)' % (
355 kernel['kernel'], flag)).stdout.strip())
Richard Barnette55d1af82018-05-22 23:40:14 +0000356
357
358 def _get_next_kernel(self):
359 """Return the kernel that has priority for the next boot."""
360 priority_a = self._cgpt('-P', _KERNEL_A)
361 priority_b = self._cgpt('-P', _KERNEL_B)
362 if priority_a > priority_b:
363 return _KERNEL_A
364 else:
365 return _KERNEL_B
366
367
368 def _get_kernel_success(self, kernel):
369 """Return boolean success flag for the specified kernel.
370
371 @param kernel: information of the given kernel, either _KERNEL_A
372 or _KERNEL_B.
373 """
374 return self._cgpt('-S', kernel) != 0
375
376
377 def _get_kernel_tries(self, kernel):
378 """Return tries count for the specified kernel.
379
380 @param kernel: information of the given kernel, either _KERNEL_A
381 or _KERNEL_B.
382 """
383 return self._cgpt('-T', kernel)
384
385
Richard Barnette3e8b2282018-05-15 20:42:20 +0000386 def _get_last_update_error(self):
Shuqian Zhaod9992722016-02-29 12:26:38 -0800387 """Get the last autoupdate error code."""
Richard Barnette3e8b2282018-05-15 20:42:20 +0000388 command_result = self._run(
389 '%s --last_attempt_error' % _UPDATER_BIN)
390 return command_result.stdout.strip().replace('\n', ', ')
Shuqian Zhaod9992722016-02-29 12:26:38 -0800391
392
Luigi Semenzatoe76d9f82016-11-21 11:15:10 -0800393 def _base_update_handler_no_retry(self, run_args):
Shuqian Zhaod9992722016-02-29 12:26:38 -0800394 """Base function to handle a remote update ssh call.
395
396 @param run_args: Dictionary of args passed to ssh_host.run function.
Shuqian Zhaod9992722016-02-29 12:26:38 -0800397
Luigi Semenzatoe76d9f82016-11-21 11:15:10 -0800398 @throws: intercepts and re-throws all exceptions
Shuqian Zhaod9992722016-02-29 12:26:38 -0800399 """
Shuqian Zhaod9992722016-02-29 12:26:38 -0800400 try:
401 self.host.run(**run_args)
Shuqian Zhaod9992722016-02-29 12:26:38 -0800402 except Exception as e:
Luigi Semenzatoe76d9f82016-11-21 11:15:10 -0800403 logging.debug('exception in update handler: %s', e)
404 raise e
Shuqian Zhaod9992722016-02-29 12:26:38 -0800405
Luigi Semenzatoe76d9f82016-11-21 11:15:10 -0800406
407 def _base_update_handler(self, run_args, err_msg_prefix=None):
408 """Handle a remote update ssh call, possibly with retries.
409
410 @param run_args: Dictionary of args passed to ssh_host.run function.
411 @param err_msg_prefix: Prefix of the exception error message.
412 """
413 def exception_handler(e):
414 """Examines exceptions and returns True if the update handler
415 should be retried.
416
417 @param e: the exception intercepted by the retry util.
418 """
419 return (isinstance(e, error.AutoservSSHTimeout) or
420 (isinstance(e, error.GenericHostRunError) and
421 hasattr(e, 'description') and
422 (re.search('ERROR_CODE=37', e.description) or
423 re.search('generic error .255.', e.description))))
424
425 try:
426 # Try the update twice (arg 2 is max_retry, not including the first
427 # call). Some exceptions may be caught by the retry handler.
428 retry_util.GenericRetry(exception_handler, 1,
429 self._base_update_handler_no_retry,
430 run_args)
431 except Exception as e:
432 message = err_msg_prefix + ': ' + str(e)
433 raise RootFSUpdateError(message)
Shuqian Zhaod9992722016-02-29 12:26:38 -0800434
435
Luigi Semenzatof15c8fc2017-03-03 14:12:40 -0800436 def _wait_for_update_service(self):
437 """Ensure that the update engine daemon is running, possibly
438 by waiting for it a bit in case the DUT just rebooted and the
439 service hasn't started yet.
440 """
441 def handler(e):
442 """Retry exception handler.
443
444 Assumes that the error is due to the update service not having
445 started yet.
446
447 @param e: the exception intercepted by the retry util.
448 """
449 if isinstance(e, error.AutoservRunError):
450 logging.debug('update service check exception: %s\n'
451 'retrying...', e)
452 return True
453 else:
454 return False
455
456 # Retry at most three times, every 5s.
457 status = retry_util.GenericRetry(handler, 3,
458 self.check_update_status,
459 sleep=5)
460
461 # Expect the update engine to be idle.
462 if status != UPDATER_IDLE:
Richard Barnette9d43e562018-06-05 17:20:10 +0000463 raise RootFSUpdateError(
464 'Update engine status is %s (%s was expected).'
465 % (status, UPDATER_IDLE))
Luigi Semenzatof15c8fc2017-03-03 14:12:40 -0800466
467
Richard Barnette55d1af82018-05-22 23:40:14 +0000468 def _reset_update_engine(self):
469 """Resets the host to prepare for a clean update regardless of state."""
470 self._run('stop ui || true')
471 self._run('stop update-engine || true')
472 self._run('start update-engine')
Luigi Semenzatof15c8fc2017-03-03 14:12:40 -0800473 self._wait_for_update_service()
474
Richard Barnette55d1af82018-05-22 23:40:14 +0000475
476 def _reset_stateful_partition(self):
477 """Clear any pending stateful update request."""
Richard Barnette18fd5842018-05-25 18:21:14 +0000478 self._run('%s --stateful_change=reset 2>&1'
Richard Barnettef00a2ee2018-06-08 11:51:38 -0700479 % self._get_stateful_update_script())
Richard Barnette55d1af82018-05-22 23:40:14 +0000480
481
482 def _revert_boot_partition(self):
483 """Revert the boot partition."""
484 part = self._rootdev('-s')
485 logging.warning('Reverting update; Boot partition will be %s', part)
486 return self._run('/postinst %s 2>&1' % part)
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700487
488
Richard Barnettee86b1ce2018-06-07 10:37:23 -0700489 def _verify_kernel_state(self):
490 """Verify that the next kernel to boot is correct for update.
491
492 This tests that the kernel state is correct for a successfully
493 downloaded and installed update. That is, the next kernel to
494 boot must be the currently inactive kernel.
495
496 @raise RootFSUpdateError if the DUT next kernel isn't the
497 expected next kernel.
498 """
499 inactive_kernel = self.get_kernel_state()[1]
500 next_kernel = self._get_next_kernel()
501 if next_kernel != inactive_kernel:
502 raise RootFSUpdateError(
503 'Update failed. The kernel for next boot is %s, '
504 'but %s was expected.'
505 % (next_kernel['name'], inactive_kernel['name']))
506 return inactive_kernel
507
508
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700509 def _verify_update_completed(self):
510 """Verifies that an update has completed.
511
Richard Barnette9d43e562018-06-05 17:20:10 +0000512 @raise RootFSUpdateError if the DUT doesn't indicate that
513 download is complete and the DUT is ready for reboot.
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700514 """
515 status = self.check_update_status()
516 if status != UPDATER_NEED_REBOOT:
Shuqian Zhaod9992722016-02-29 12:26:38 -0800517 error_msg = ''
518 if status == UPDATER_IDLE:
Richard Barnette3e8b2282018-05-15 20:42:20 +0000519 error_msg = 'Update error: %s' % self._get_last_update_error()
Richard Barnette9d43e562018-06-05 17:20:10 +0000520 raise RootFSUpdateError(
521 'Update engine status is %s (%s was expected). %s'
522 % (status, UPDATER_NEED_REBOOT, error_msg))
Richard Barnettee86b1ce2018-06-07 10:37:23 -0700523 return self._verify_kernel_state()
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700524
525
Richard Barnette55d1af82018-05-22 23:40:14 +0000526 def trigger_update(self):
Richard Barnette9d43e562018-06-05 17:20:10 +0000527 """Triggers a background update."""
528 # If this function is called immediately after reboot (which it
529 # can be), there is no guarantee that the update engine is up
530 # and running yet, so wait for it.
Richard Barnette55d1af82018-05-22 23:40:14 +0000531 self._wait_for_update_service()
532
533 autoupdate_cmd = ('%s --check_for_update --omaha_url=%s' %
534 (_UPDATER_BIN, self.update_url))
535 run_args = {'command': autoupdate_cmd}
536 err_prefix = 'Failed to trigger an update on %s. ' % self.host.hostname
537 logging.info('Triggering update via: %s', autoupdate_cmd)
538 metric_fields = {'success': False}
539 try:
540 self._base_update_handler(run_args, err_prefix)
541 metric_fields['success'] = True
542 finally:
543 c = metrics.Counter('chromeos/autotest/autoupdater/trigger')
Richard Barnette621a8e42018-06-25 17:34:11 -0700544 metric_fields.update(_get_metric_fields(self.update_url))
Richard Barnette55d1af82018-05-22 23:40:14 +0000545 c.increment(fields=metric_fields)
546
547
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700548 def update_image(self):
Richard Barnette18fd5842018-05-25 18:21:14 +0000549 """Updates the device root FS and kernel and verifies success."""
Shuqian Zhaofe4d62e2016-06-23 14:46:45 -0700550 autoupdate_cmd = ('%s --update --omaha_url=%s' %
Richard Barnette3e8b2282018-05-15 20:42:20 +0000551 (_UPDATER_BIN, self.update_url))
David Haddock76a4c882017-12-13 18:50:09 -0800552 if not self.interactive:
553 autoupdate_cmd = '%s --interactive=false' % autoupdate_cmd
Shuqian Zhaod9992722016-02-29 12:26:38 -0800554 run_args = {'command': autoupdate_cmd, 'timeout': 3600}
555 err_prefix = ('Failed to install device image using payload at %s '
556 'on %s. ' % (self.update_url, self.host.hostname))
557 logging.info('Updating image via: %s', autoupdate_cmd)
Allen Li1a5cc0a2017-06-20 14:08:59 -0700558 metric_fields = {'success': False}
Luigi Semenzatoe76d9f82016-11-21 11:15:10 -0800559 try:
Luigi Semenzatoe76d9f82016-11-21 11:15:10 -0800560 self._base_update_handler(run_args, err_prefix)
Allen Li1a5cc0a2017-06-20 14:08:59 -0700561 metric_fields['success'] = True
562 finally:
Allen Li1a5cc0a2017-06-20 14:08:59 -0700563 c = metrics.Counter('chromeos/autotest/autoupdater/update')
Richard Barnette621a8e42018-06-25 17:34:11 -0700564 metric_fields.update(_get_metric_fields(self.update_url))
Allen Li1a5cc0a2017-06-20 14:08:59 -0700565 c.increment(fields=metric_fields)
Richard Barnette4d211c92018-05-24 18:56:08 +0000566 return self._verify_update_completed()
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700567
568
Richard Barnettef00a2ee2018-06-08 11:51:38 -0700569 def _get_remote_script(self, script_name):
570 """Ensure that `script_name` is present on the DUT.
Chris Sosa5e4246b2012-05-22 18:05:22 -0700571
Richard Barnettef00a2ee2018-06-08 11:51:38 -0700572 The given script (e.g. `stateful_update`) may be present in the
573 stateful partition under /usr/local/bin, or we may have to
574 download it from the devserver.
Chris Sosaa3ac2152012-05-23 22:23:13 -0700575
Richard Barnettef00a2ee2018-06-08 11:51:38 -0700576 Determine whether the script is present or must be downloaded
577 and download if necessary. Then, return a command fragment
578 sufficient to run the script from whereever it now lives on the
579 DUT.
Richard Barnette9d43e562018-06-05 17:20:10 +0000580
Richard Barnettef00a2ee2018-06-08 11:51:38 -0700581 @param script_name The name of the script as expected in
582 /usr/local/bin and on the devserver.
583 @return A string with the command (minus arguments) that will
584 run the target script.
Gwendal Grignou3e96cc22017-06-07 16:22:51 -0700585 """
Richard Barnettef00a2ee2018-06-08 11:51:38 -0700586 remote_script = '/usr/local/bin/%s' % script_name
587 if self.host.path_exists(remote_script):
588 return remote_script
589 remote_tmp_script = '/tmp/%s' % script_name
590 server_name = urlparse.urlparse(self.update_url)[1]
591 script_url = 'http://%s/static/%s' % (server_name, script_name)
592 fetch_script = (
593 'curl -o %s %s && head -1 %s | grep "^#!" | sed "s/#!//"') % (
594 remote_tmp_script, script_url, remote_tmp_script)
595 script_interpreter = self._run(fetch_script,
596 ignore_status=True).stdout.strip()
597 if not script_interpreter:
598 return None
599 return '%s %s' % (script_interpreter, remote_tmp_script)
Chris Sosa5e4246b2012-05-22 18:05:22 -0700600
Richard Barnettef00a2ee2018-06-08 11:51:38 -0700601
602 def _get_stateful_update_script(self):
603 """Returns a command to run the stateful update script.
604
605 Find `stateful_update` on the target or install it, as
606 necessary. If installation fails, raise an exception.
607
608 @raise StatefulUpdateError if the script can't be found or
609 installed.
610 @return A string that can be joined with arguments to run the
611 `stateful_update` command on the DUT.
612 """
613 script_command = self._get_remote_script(_STATEFUL_UPDATE_SCRIPT)
614 if not script_command:
615 raise StatefulUpdateError('Could not install %s on DUT'
Richard Barnette9d43e562018-06-05 17:20:10 +0000616 % _STATEFUL_UPDATE_SCRIPT)
Richard Barnettef00a2ee2018-06-08 11:51:38 -0700617 return script_command
Chris Sosa5e4246b2012-05-22 18:05:22 -0700618
619
Chris Sosac1932172013-10-16 13:28:53 -0700620 def rollback_rootfs(self, powerwash):
621 """Triggers rollback and waits for it to complete.
622
623 @param powerwash: If true, powerwash as part of rollback.
624
625 @raise RootFSUpdateError if anything went wrong.
Chris Sosac1932172013-10-16 13:28:53 -0700626 """
Dan Shi549fb822015-03-24 18:01:11 -0700627 version = self.host.get_release_version()
Chris Sosac8617522014-06-09 23:22:26 +0000628 # Introduced can_rollback in M36 (build 5772). # etc/lsb-release matches
629 # X.Y.Z. This version split just pulls the first part out.
630 try:
631 build_number = int(version.split('.')[0])
632 except ValueError:
633 logging.error('Could not parse build number.')
634 build_number = 0
635
636 if build_number >= 5772:
Richard Barnette3e8b2282018-05-15 20:42:20 +0000637 can_rollback_cmd = '%s --can_rollback' % _UPDATER_BIN
Chris Sosac8617522014-06-09 23:22:26 +0000638 logging.info('Checking for rollback.')
639 try:
640 self._run(can_rollback_cmd)
641 except error.AutoservRunError as e:
642 raise RootFSUpdateError("Rollback isn't possible on %s: %s" %
643 (self.host.hostname, str(e)))
644
Richard Barnette3e8b2282018-05-15 20:42:20 +0000645 rollback_cmd = '%s --rollback --follow' % _UPDATER_BIN
Chris Sosac1932172013-10-16 13:28:53 -0700646 if not powerwash:
Dan Shif3a35f72016-01-25 11:18:14 -0800647 rollback_cmd += ' --nopowerwash'
Chris Sosac1932172013-10-16 13:28:53 -0700648
Chris Sosac8617522014-06-09 23:22:26 +0000649 logging.info('Performing rollback.')
Chris Sosac1932172013-10-16 13:28:53 -0700650 try:
651 self._run(rollback_cmd)
Chris Sosac1932172013-10-16 13:28:53 -0700652 except error.AutoservRunError as e:
653 raise RootFSUpdateError('Rollback failed on %s: %s' %
654 (self.host.hostname, str(e)))
655
656 self._verify_update_completed()
657
Gilad Arnold0ed760c2012-11-05 23:42:53 -0800658
Chris Sosa72312602013-04-16 15:01:56 -0700659 def update_stateful(self, clobber=True):
660 """Updates the stateful partition.
661
662 @param clobber: If True, a clean stateful installation.
Richard Barnette9d43e562018-06-05 17:20:10 +0000663
664 @raise StatefulUpdateError if the update script fails to
665 complete successfully.
Chris Sosa72312602013-04-16 15:01:56 -0700666 """
Chris Sosa77556d82012-04-05 15:23:14 -0700667 logging.info('Updating stateful partition...')
Richard Barnette18fd5842018-05-25 18:21:14 +0000668 statefuldev_url = self.update_url.replace('update', 'static')
Chris Sosaa3ac2152012-05-23 22:23:13 -0700669
Dale Curtis5c32c722011-05-04 19:24:23 -0700670 # Attempt stateful partition update; this must succeed so that the newly
671 # installed host is testable after update.
Richard Barnettef00a2ee2018-06-08 11:51:38 -0700672 statefuldev_cmd = [self._get_stateful_update_script(), statefuldev_url]
Chris Sosa72312602013-04-16 15:01:56 -0700673 if clobber:
674 statefuldev_cmd.append('--stateful_change=clean')
675
676 statefuldev_cmd.append('2>&1')
Dale Curtis5c32c722011-05-04 19:24:23 -0700677 try:
Dan Shi205b8732016-01-25 10:56:22 -0800678 self._run(' '.join(statefuldev_cmd), timeout=1200)
Dale Curtis5c32c722011-05-04 19:24:23 -0700679 except error.AutoservRunError:
Richard Barnette18fd5842018-05-25 18:21:14 +0000680 raise StatefulUpdateError(
Gilad Arnold62cf3a42015-10-01 09:15:25 -0700681 'Failed to perform stateful update on %s' %
682 self.host.hostname)
Dale Curtis5c32c722011-05-04 19:24:23 -0700683
Chris Sosaa3ac2152012-05-23 22:23:13 -0700684
Richard Barnette54d14f52018-05-18 16:39:49 +0000685 def verify_boot_expectations(self, expected_kernel, rollback_message):
Richard Barnette55d1af82018-05-22 23:40:14 +0000686 """Verifies that we fully booted given expected kernel state.
687
688 This method both verifies that we booted using the correct kernel
689 state and that the OS has marked the kernel as good.
690
Richard Barnette54d14f52018-05-18 16:39:49 +0000691 @param expected_kernel: kernel that we are verifying with,
Richard Barnette55d1af82018-05-22 23:40:14 +0000692 i.e. I expect to be booted onto partition 4 etc. See output of
693 get_kernel_state.
Richard Barnette9d43e562018-06-05 17:20:10 +0000694 @param rollback_message: string include in except message text
Richard Barnette55d1af82018-05-22 23:40:14 +0000695 if we booted with the wrong partition.
696
Richard Barnette9d43e562018-06-05 17:20:10 +0000697 @raise NewBuildUpdateError if any of the various checks fail.
Richard Barnette55d1af82018-05-22 23:40:14 +0000698 """
699 # Figure out the newly active kernel.
Richard Barnette54d14f52018-05-18 16:39:49 +0000700 active_kernel = self.get_kernel_state()[0]
Richard Barnette55d1af82018-05-22 23:40:14 +0000701
702 # Check for rollback due to a bad build.
Richard Barnette54d14f52018-05-18 16:39:49 +0000703 if active_kernel != expected_kernel:
Richard Barnette55d1af82018-05-22 23:40:14 +0000704
705 # Kernel crash reports should be wiped between test runs, but
706 # may persist from earlier parts of the test, or from problems
707 # with provisioning.
708 #
709 # Kernel crash reports will NOT be present if the crash happened
710 # before encrypted stateful is mounted.
711 #
712 # TODO(dgarrett): Integrate with server/crashcollect.py at some
713 # point.
714 kernel_crashes = glob.glob('/var/spool/crash/kernel.*.kcrash')
715 if kernel_crashes:
716 rollback_message += ': kernel_crash'
717 logging.debug('Found %d kernel crash reports:',
718 len(kernel_crashes))
719 # The crash names contain timestamps that may be useful:
720 # kernel.20131207.005945.0.kcrash
721 for crash in kernel_crashes:
722 logging.debug(' %s', os.path.basename(crash))
723
724 # Print out some information to make it easier to debug
725 # the rollback.
726 logging.debug('Dumping partition table.')
727 self._run('cgpt show $(rootdev -s -d)')
728 logging.debug('Dumping crossystem for firmware debugging.')
729 self._run('crossystem --all')
Richard Barnette9d43e562018-06-05 17:20:10 +0000730 raise NewBuildUpdateError(self.update_version, rollback_message)
Richard Barnette55d1af82018-05-22 23:40:14 +0000731
732 # Make sure chromeos-setgoodkernel runs.
733 try:
734 utils.poll_for_condition(
Richard Barnette54d14f52018-05-18 16:39:49 +0000735 lambda: (self._get_kernel_tries(active_kernel) == 0
736 and self._get_kernel_success(active_kernel)),
Richard Barnette9d43e562018-06-05 17:20:10 +0000737 exception=RootFSUpdateError(),
Richard Barnette55d1af82018-05-22 23:40:14 +0000738 timeout=_KERNEL_UPDATE_TIMEOUT, sleep_interval=5)
Richard Barnette9d43e562018-06-05 17:20:10 +0000739 except RootFSUpdateError:
Richard Barnette55d1af82018-05-22 23:40:14 +0000740 services_status = self._run('status system-services').stdout
741 if services_status != 'system-services start/running\n':
742 event = ('Chrome failed to reach login screen')
743 else:
744 event = ('update-engine failed to call '
745 'chromeos-setgoodkernel')
Richard Barnette9d43e562018-06-05 17:20:10 +0000746 raise NewBuildUpdateError(self.update_version, event)
Richard Barnette55d1af82018-05-22 23:40:14 +0000747
748
Richard Barnette14ee84c2018-05-18 20:23:42 +0000749 def _prepare_host(self):
750 """Make sure the target DUT is working and ready for update.
751
752 Initially, the target DUT's state is unknown. The DUT is
753 expected to be online, but we strive to be forgiving if Chrome
754 and/or the update engine aren't fully functional.
755 """
756 # Summary of work, and the rationale:
757 # 1. Reboot, because it's a good way to clear out problems.
758 # 2. Touch the PROVISION_FAILED file, to allow repair to detect
759 # failure later.
760 # 3. Run the hook for host class specific preparation.
761 # 4. Stop Chrome, because the system is designed to eventually
762 # reboot if Chrome is stuck in a crash loop.
763 # 5. Force `update-engine` to start, because if Chrome failed
764 # to start properly, the status of the `update-engine` job
765 # will be uncertain.
766 self._reset_stateful_partition()
767 self.host.reboot(timeout=self.host.REBOOT_TIMEOUT)
768 self._run('touch %s' % PROVISION_FAILED)
769 self.host.prepare_for_update()
Richard Barnettee86b1ce2018-06-07 10:37:23 -0700770 self._reset_update_engine()
Richard Barnette14ee84c2018-05-18 20:23:42 +0000771 logging.info('Updating from version %s to %s.',
772 self.host.get_release_version(),
773 self.update_version)
774
775
776 def _verify_devserver(self):
Richard Barnette9d43e562018-06-05 17:20:10 +0000777 """Check that our chosen devserver is still working.
778
779 @raise DevServerError if the devserver fails any sanity check.
780 """
Richard Barnette14ee84c2018-05-18 20:23:42 +0000781 server = 'http://%s' % urlparse.urlparse(self.update_url)[1]
782 try:
783 if not dev_server.ImageServer.devserver_healthy(server):
Richard Barnette9d43e562018-06-05 17:20:10 +0000784 raise DevServerError(
785 server, 'Devserver is not healthy')
Richard Barnette14ee84c2018-05-18 20:23:42 +0000786 except Exception as e:
Richard Barnette9d43e562018-06-05 17:20:10 +0000787 raise DevServerError(
788 server, 'Devserver is not up and available')
Richard Barnette14ee84c2018-05-18 20:23:42 +0000789
790
Richard Barnettee86b1ce2018-06-07 10:37:23 -0700791 def _install_via_update_engine(self):
792 """Install an updating using the production AU flow.
793
794 This uses the standard AU flow and the `stateful_update` script
795 to download and install a root FS, kernel and stateful
796 filesystem content.
797
798 @return The kernel expected to be booted next.
799 """
800 logging.info('Installing image using update_engine.')
801 expected_kernel = self.update_image()
802 self.update_stateful()
803 return expected_kernel
804
805
806 def _install_via_quick_provision(self):
807 """Install an updating using the `quick-provision` script.
808
809 This uses the `quick-provision` script to download and install
810 a root FS, kernel and stateful filesystem content.
811
812 @return The kernel expected to be booted next.
813 """
814 build_re = global_config.global_config.get_config_value(
815 'CROS', 'quick_provision_build_regex', type=str, default='')
816 image_name = url_to_image_name(self.update_url)
817 if not build_re or re.match(build_re, image_name) is None:
818 logging.info('Not eligible for quick-provision.')
819 return None
820 logging.info('Installing image using quick-provision.')
821 provision_command = self._get_remote_script(_QUICK_PROVISION_SCRIPT)
822 server_name = urlparse.urlparse(self.update_url)[1]
823 static_url = 'http://%s/static' % server_name
824 command = '%s --noreboot %s %s' % (
825 provision_command, image_name, static_url)
826 try:
827 self._run(command)
828 return self._verify_kernel_state()
829 except Exception:
830 # N.B. We handle only `Exception` here. Non-Exception
831 # classes (such as KeyboardInterrupt) are handled by our
832 # caller.
833 logging.exception('quick-provision script failed; '
834 'will fall back to update_engine.')
835 self._revert_boot_partition()
836 self._reset_stateful_partition()
837 self._reset_update_engine()
838 return None
839
840
Richard Barnette54d14f52018-05-18 16:39:49 +0000841 def _install_update(self):
Richard Barnette0beb14b2018-05-15 18:07:52 +0000842 """Install the requested image on the DUT, but don't start it.
843
Richard Barnettee86b1ce2018-06-07 10:37:23 -0700844 This downloads and installs a root FS, kernel and stateful
845 filesystem content. This does not reboot the DUT, so the update
846 is merely pending when the method returns.
847
848 @return The kernel expected to be booted next.
Dan Shi0f466e82013-02-22 15:44:58 -0800849 """
Richard Barnette14ee84c2018-05-18 20:23:42 +0000850 logging.info('Installing image at %s onto %s',
851 self.update_url, self.host.hostname)
Sean O'Connor5346e4e2010-08-12 18:49:24 +0200852 try:
Richard Barnettee86b1ce2018-06-07 10:37:23 -0700853 return (self._install_via_quick_provision()
854 or self._install_via_update_engine())
Dale Curtis1e973182011-07-12 18:21:36 -0700855 except:
Richard Barnettee86b1ce2018-06-07 10:37:23 -0700856 # N.B. This handling code includes non-Exception classes such
857 # as KeyboardInterrupt. We need to clean up, but we also must
858 # re-raise.
Richard Barnette14ee84c2018-05-18 20:23:42 +0000859 self._revert_boot_partition()
860 self._reset_stateful_partition()
Richard Barnettee86b1ce2018-06-07 10:37:23 -0700861 self._reset_update_engine()
Dale Curtis1e973182011-07-12 18:21:36 -0700862 # Collect update engine logs in the event of failure.
863 if self.host.job:
Aviv Keshet2610d3e2016-06-01 16:37:01 -0700864 logging.info('Collecting update engine logs due to failure...')
Dale Curtis1e973182011-07-12 18:21:36 -0700865 self.host.get_file(
Richard Barnette3e8b2282018-05-15 20:42:20 +0000866 _UPDATER_LOGS, self.host.job.sysinfo.sysinfodir,
Gilad Arnold0c0df732015-09-21 06:37:59 -0700867 preserve_perm=False)
Richard Barnette3e8b2282018-05-15 20:42:20 +0000868 _list_image_dir_contents(self.update_url)
Dale Curtis1e973182011-07-12 18:21:36 -0700869 raise
Sean O'Connor5346e4e2010-08-12 18:49:24 +0200870
871
Richard Barnette14ee84c2018-05-18 20:23:42 +0000872 def _complete_update(self, expected_kernel):
873 """Finish the update, and confirm that it succeeded.
Richard Barnette0beb14b2018-05-15 18:07:52 +0000874
Richard Barnette14ee84c2018-05-18 20:23:42 +0000875 Initial condition is that the target build has been downloaded
876 and installed on the DUT, but has not yet been booted. This
877 function is responsible for rebooting the DUT, and checking that
878 the new build is running successfully.
Richard Barnette0beb14b2018-05-15 18:07:52 +0000879
Richard Barnette14ee84c2018-05-18 20:23:42 +0000880 @param expected_kernel: kernel expected to be active after reboot.
Richard Barnette0beb14b2018-05-15 18:07:52 +0000881 """
Richard Barnette14ee84c2018-05-18 20:23:42 +0000882 # Regarding the 'crossystem' command below: In some cases,
883 # the update flow puts the TPM into a state such that it
884 # fails verification. We don't know why. However, this
885 # call papers over the problem by clearing the TPM during
886 # the reboot.
887 #
888 # We ignore failures from 'crossystem'. Although failure
889 # here is unexpected, and could signal a bug, the point of
890 # the exercise is to paper over problems; allowing this to
891 # fail would defeat the purpose.
892 self._run('crossystem clear_tpm_owner_request=1',
893 ignore_status=True)
894 self.host.reboot(timeout=self.host.REBOOT_TIMEOUT)
895
Richard Barnette0beb14b2018-05-15 18:07:52 +0000896 # Touch the lab machine file to leave a marker that
897 # distinguishes this image from other test images.
898 # Afterwards, we must re-run the autoreboot script because
899 # it depends on the _LAB_MACHINE_FILE.
900 autoreboot_cmd = ('FILE="%s" ; [ -f "$FILE" ] || '
901 '( touch "$FILE" ; start autoreboot )')
Richard Barnette3e8b2282018-05-15 20:42:20 +0000902 self._run(autoreboot_cmd % _LAB_MACHINE_FILE)
Richard Barnette0beb14b2018-05-15 18:07:52 +0000903 self.verify_boot_expectations(
904 expected_kernel, rollback_message=
905 'Build %s failed to boot on %s; system rolled back to previous '
906 'build' % (self.update_version, self.host.hostname))
907
908 logging.debug('Cleaning up old autotest directories.')
909 try:
910 installed_autodir = autotest.Autotest.get_installed_autodir(
911 self.host)
912 self._run('rm -rf ' + installed_autodir)
913 except autotest.AutodirNotFoundError:
914 logging.debug('No autotest installed directory found.')
915
916
Richard Barnette621a8e42018-06-25 17:34:11 -0700917 def _run_update_steps(self):
918 """Perform a full update of a DUT, with diagnosis for failures.
Richard Barnette0beb14b2018-05-15 18:07:52 +0000919
Richard Barnette621a8e42018-06-25 17:34:11 -0700920 Run the individual steps of the update. If a step fails, make
921 sure that the exception raised describes the failure with a
922 diagnosis based on the step that failed.
Richard Barnette0beb14b2018-05-15 18:07:52 +0000923
Richard Barnette621a8e42018-06-25 17:34:11 -0700924 @raise HostUpdateError if a failure is caused by a problem on
Richard Barnette9d43e562018-06-05 17:20:10 +0000925 the DUT prior to the update.
Richard Barnette621a8e42018-06-25 17:34:11 -0700926 @raise ImageInstallError if a failure occurs during download
Richard Barnette9d43e562018-06-05 17:20:10 +0000927 and install of the update and cannot be definitively
928 blamed on either the DUT or the devserver.
Richard Barnette621a8e42018-06-25 17:34:11 -0700929 @raise NewBuildUpdateError if a failure occurs because the
Richard Barnette9d43e562018-06-05 17:20:10 +0000930 new build fails to function correctly.
Richard Barnette0beb14b2018-05-15 18:07:52 +0000931 """
Richard Barnette14ee84c2018-05-18 20:23:42 +0000932 self._verify_devserver()
Richard Barnette9d43e562018-06-05 17:20:10 +0000933
934 try:
935 self._prepare_host()
936 except _AttributedUpdateError:
937 raise
938 except Exception as e:
939 logging.exception('Failure preparing host prior to update.')
940 raise HostUpdateError(self.host.hostname, str(e))
941
942 try:
943 expected_kernel = self._install_update()
944 except _AttributedUpdateError:
945 raise
946 except Exception as e:
947 logging.exception('Failure during download and install.')
Richard Barnette621a8e42018-06-25 17:34:11 -0700948 server_name = dev_server.get_resolved_hostname(self.update_url)
Richard Barnette9d43e562018-06-05 17:20:10 +0000949 raise ImageInstallError(self.host.hostname, server_name, str(e))
950
951 try:
952 self._complete_update(expected_kernel)
953 except _AttributedUpdateError:
954 raise
955 except Exception as e:
956 logging.exception('Failure from build after update.')
957 raise NewBuildUpdateError(self.update_version, str(e))
Richard Barnette0beb14b2018-05-15 18:07:52 +0000958
Richard Barnette621a8e42018-06-25 17:34:11 -0700959
960 def run_update(self):
961 """Perform a full update of a DUT in the test lab.
962
963 This downloads and installs the root FS and stateful partition
964 content needed for the update specified in `self.host` and
965 `self.update_url`. The update is performed according to the
966 requirements for provisioning a DUT for testing the requested
967 build.
968
969 At the end of the procedure, metrics are reported describing the
970 outcome of the operation.
971
972 @returns A tuple of the form `(image_name, attributes)`, where
973 `image_name` is the name of the image installed, and
974 `attributes` is new attributes to be applied to the DUT.
975 """
976 start_time = time.time()
977 failure_reason = None
978 server_name = dev_server.get_resolved_hostname(self.update_url)
979 metrics.Counter(_metric_name('install')).increment(
980 fields={'devserver': server_name})
981 try:
982 self._run_update_steps()
983 except _AttributedUpdateError as e:
984 failure_reason = e.failure_summary
985 raise
986 except Exception as e:
987 failure_reason = 'Unknown failure'
988 raise
989 finally:
990 end_time = time.time()
991 _emit_provision_metrics(
992 self.update_url, self.host.hostname,
993 failure_reason, end_time - start_time)
994
Richard Barnette0beb14b2018-05-15 18:07:52 +0000995 image_name = url_to_image_name(self.update_url)
996 # update_url is different from devserver url needed to stage autotest
997 # packages, therefore, resolve a new devserver url here.
998 devserver_url = dev_server.ImageServer.resolve(
999 image_name, self.host.hostname).url()
1000 repo_url = tools.get_package_url(devserver_url, image_name)
1001 return image_name, {ds_constants.JOB_REPO_URL: repo_url}