blob: 25c415a17f467d872ebd6ec68a7ed5fcd3c8f2c5 [file] [log] [blame]
Chris Sosa5e4246b2012-05-22 18:05:22 -07001# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
Sean O'Connor5346e4e2010-08-12 18:49:24 +02002# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
Don Garrett56b1cc82013-12-06 17:49:20 -08005import glob
Sean O'Connor5346e4e2010-08-12 18:49:24 +02006import logging
Dale Curtis5c32c722011-05-04 19:24:23 -07007import os
Sean O'Connor5346e4e2010-08-12 18:49:24 +02008import re
Richard Barnette0beb14b2018-05-15 18:07:52 +00009import time
Prashanth B32baa9b2014-03-13 13:23:01 -070010import urllib2
Richard Barnette0beb14b2018-05-15 18:07:52 +000011import urlparse
Sean O'Connor5346e4e2010-08-12 18:49:24 +020012
Chris Sosa65425082013-10-16 13:26:22 -070013from autotest_lib.client.bin import utils
Dale Curtis5c32c722011-05-04 19:24:23 -070014from autotest_lib.client.common_lib import error, global_config
Prashanth B32baa9b2014-03-13 13:23:01 -070015from autotest_lib.client.common_lib.cros import dev_server
Richard Barnette0beb14b2018-05-15 18:07:52 +000016from autotest_lib.server import autotest
Shelley Chen61d28982016-10-28 09:40:20 -070017from autotest_lib.server import utils as server_utils
Richard Barnette0beb14b2018-05-15 18:07:52 +000018from autotest_lib.server.cros.dynamic_suite import constants as ds_constants
19from autotest_lib.server.cros.dynamic_suite import tools
Luigi Semenzatoe76d9f82016-11-21 11:15:10 -080020from chromite.lib import retry_util
Dan Shif3a35f72016-01-25 11:18:14 -080021
Shelley Chen16b8df32016-10-27 16:24:21 -070022try:
23 from chromite.lib import metrics
Dan Shi5e2efb72017-02-07 11:40:23 -080024except ImportError:
25 metrics = utils.metrics_mock
Sean O'Connor5346e4e2010-08-12 18:49:24 +020026
Gwendal Grignou3e96cc22017-06-07 16:22:51 -070027try:
28 import devserver
Richard Barnette3e8b2282018-05-15 20:42:20 +000029 _STATEFUL_UPDATE_PATH = devserver.__path__[0]
Gwendal Grignou3e96cc22017-06-07 16:22:51 -070030except ImportError:
Richard Barnette3e8b2282018-05-15 20:42:20 +000031 _STATEFUL_UPDATE_PATH = '/usr/bin'
Gwendal Grignou3e96cc22017-06-07 16:22:51 -070032
Dale Curtis5c32c722011-05-04 19:24:23 -070033# Local stateful update path is relative to the CrOS source directory.
Sean O'Connor5346e4e2010-08-12 18:49:24 +020034UPDATER_IDLE = 'UPDATE_STATUS_IDLE'
Sean Oc053dfe2010-08-23 18:22:26 +020035UPDATER_NEED_REBOOT = 'UPDATE_STATUS_UPDATED_NEED_REBOOT'
beeps5e8c45a2013-12-17 22:05:11 -080036# A list of update engine client states that occur after an update is triggered.
37UPDATER_PROCESSING_UPDATE = ['UPDATE_STATUS_CHECKING_FORUPDATE',
38 'UPDATE_STATUS_UPDATE_AVAILABLE',
39 'UPDATE_STATUS_DOWNLOADING',
40 'UPDATE_STATUS_FINALIZING']
Sean O'Connor5346e4e2010-08-12 18:49:24 +020041
Richard Barnette0beb14b2018-05-15 18:07:52 +000042
Richard Barnette3e8b2282018-05-15 20:42:20 +000043_STATEFUL_UPDATE_SCRIPT = 'stateful_update'
44_REMOTE_STATEFUL_UPDATE_PATH = os.path.join(
45 '/usr/local/bin', _STATEFUL_UPDATE_SCRIPT)
46_REMOTE_TMP_STATEFUL_UPDATE = os.path.join(
47 '/tmp', _STATEFUL_UPDATE_SCRIPT)
48
49_UPDATER_BIN = '/usr/bin/update_engine_client'
50_UPDATER_LOGS = ['/var/log/messages', '/var/log/update_engine']
51
52_KERNEL_A = {'name': 'KERN-A', 'kernel': 2, 'root': 3}
53_KERNEL_B = {'name': 'KERN-B', 'kernel': 4, 'root': 5}
54
55# Time to wait for new kernel to be marked successful after
56# auto update.
57_KERNEL_UPDATE_TIMEOUT = 120
58
59
Richard Barnette0beb14b2018-05-15 18:07:52 +000060# PROVISION_FAILED - A flag file to indicate provision failures. The
61# file is created at the start of any AU procedure (see
62# `ChromiumOSUpdater.run_full_update()`). The file's location in
63# stateful means that on successul update it will be removed. Thus, if
64# this file exists, it indicates that we've tried and failed in a
65# previous attempt to update.
66PROVISION_FAILED = '/var/tmp/provision_failed'
67
68
Richard Barnette3e8b2282018-05-15 20:42:20 +000069# A flag file used to enable special handling in lab DUTs. Some
70# parts of the system in Chromium OS test images will behave in ways
71# convenient to the test lab when this file is present. Generally,
72# we create this immediately after any update completes.
73_LAB_MACHINE_FILE = '/mnt/stateful_partition/.labmachine'
74
75
Sean O'Connor5346e4e2010-08-12 18:49:24 +020076class ChromiumOSError(error.InstallError):
77 """Generic error for ChromiumOS-specific exceptions."""
Gilad Arnoldd6adeb82015-09-21 07:10:03 -070078
79
Chris Sosa77556d82012-04-05 15:23:14 -070080class RootFSUpdateError(ChromiumOSError):
81 """Raised when the RootFS fails to update."""
Chris Sosa77556d82012-04-05 15:23:14 -070082
83
84class StatefulUpdateError(ChromiumOSError):
85 """Raised when the stateful partition fails to update."""
Chris Sosa77556d82012-04-05 15:23:14 -070086
87
Richard Barnette3e8b2282018-05-15 20:42:20 +000088def _url_to_version(update_url):
Dan Shi0f466e82013-02-22 15:44:58 -080089 """Return the version based on update_url.
90
91 @param update_url: url to the image to update to.
92
93 """
Dale Curtisddfdb942011-07-14 13:59:24 -070094 # The Chrome OS version is generally the last element in the URL. The only
95 # exception is delta update URLs, which are rooted under the version; e.g.,
96 # http://.../update/.../0.14.755.0/au/0.14.754.0. In this case we want to
97 # strip off the au section of the path before reading the version.
Dan Shi5002cfc2013-04-29 10:45:05 -070098 return re.sub('/au/.*', '',
99 urlparse.urlparse(update_url).path).split('/')[-1].strip()
Sean O'Connor5346e4e2010-08-12 18:49:24 +0200100
101
Scott Zawalskieadbf702013-03-14 09:23:06 -0400102def url_to_image_name(update_url):
103 """Return the image name based on update_url.
104
105 From a URL like:
106 http://172.22.50.205:8082/update/lumpy-release/R27-3837.0.0
107 return lumpy-release/R27-3837.0.0
108
109 @param update_url: url to the image to update to.
110 @returns a string representing the image name in the update_url.
111
112 """
113 return '/'.join(urlparse.urlparse(update_url).path.split('/')[-2:])
114
115
Prashanth B32baa9b2014-03-13 13:23:01 -0700116def _get_devserver_build_from_update_url(update_url):
117 """Get the devserver and build from the update url.
118
119 @param update_url: The url for update.
120 Eg: http://devserver:port/update/build.
121
122 @return: A tuple of (devserver url, build) or None if the update_url
123 doesn't match the expected pattern.
124
125 @raises ValueError: If the update_url doesn't match the expected pattern.
126 @raises ValueError: If no global_config was found, or it doesn't contain an
127 image_url_pattern.
128 """
129 pattern = global_config.global_config.get_config_value(
130 'CROS', 'image_url_pattern', type=str, default='')
131 if not pattern:
132 raise ValueError('Cannot parse update_url, the global config needs '
133 'an image_url_pattern.')
134 re_pattern = pattern.replace('%s', '(\S+)')
135 parts = re.search(re_pattern, update_url)
136 if not parts or len(parts.groups()) < 2:
137 raise ValueError('%s is not an update url' % update_url)
138 return parts.groups()
139
140
Richard Barnette3e8b2282018-05-15 20:42:20 +0000141def _list_image_dir_contents(update_url):
Prashanth B32baa9b2014-03-13 13:23:01 -0700142 """Lists the contents of the devserver for a given build/update_url.
143
144 @param update_url: An update url. Eg: http://devserver:port/update/build.
145 """
146 if not update_url:
147 logging.warning('Need update_url to list contents of the devserver.')
148 return
149 error_msg = 'Cannot check contents of devserver, update url %s' % update_url
150 try:
151 devserver_url, build = _get_devserver_build_from_update_url(update_url)
152 except ValueError as e:
153 logging.warning('%s: %s', error_msg, e)
154 return
155 devserver = dev_server.ImageServer(devserver_url)
156 try:
157 devserver.list_image_dir(build)
158 # The devserver will retry on URLError to avoid flaky connections, but will
159 # eventually raise the URLError if it persists. All HTTPErrors get
160 # converted to DevServerExceptions.
161 except (dev_server.DevServerException, urllib2.URLError) as e:
162 logging.warning('%s: %s', error_msg, e)
163
164
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700165# TODO(garnold) This implements shared updater functionality needed for
166# supporting the autoupdate_EndToEnd server-side test. We should probably
167# migrate more of the existing ChromiumOSUpdater functionality to it as we
168# expand non-CrOS support in other tests.
Richard Barnette3e8b2282018-05-15 20:42:20 +0000169class ChromiumOSUpdater(object):
170 """Chromium OS specific DUT update functionality."""
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700171
Richard Barnette3e8b2282018-05-15 20:42:20 +0000172 def __init__(self, update_url, host=None, interactive=True):
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700173 """Initializes the object.
174
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700175 @param update_url: The URL we want the update to use.
176 @param host: A client.common_lib.hosts.Host implementation.
David Haddock76a4c882017-12-13 18:50:09 -0800177 @param interactive: Bool whether we are doing an interactive update.
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700178 """
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700179 self.update_url = update_url
180 self.host = host
David Haddock76a4c882017-12-13 18:50:09 -0800181 self.interactive = interactive
Richard Barnette3e8b2282018-05-15 20:42:20 +0000182 self.update_version = _url_to_version(update_url)
183
184
185 def _run(self, cmd, *args, **kwargs):
186 """Abbreviated form of self.host.run(...)"""
187 return self.host.run(cmd, *args, **kwargs)
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700188
189
190 def check_update_status(self):
191 """Returns the current update engine state.
192
193 We use the `update_engine_client -status' command and parse the line
194 indicating the update state, e.g. "CURRENT_OP=UPDATE_STATUS_IDLE".
195 """
Luigi Semenzatof15c8fc2017-03-03 14:12:40 -0800196 update_status = self.host.run(command='%s -status | grep CURRENT_OP' %
Richard Barnette3e8b2282018-05-15 20:42:20 +0000197 _UPDATER_BIN)
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700198 return update_status.stdout.strip().split('=')[-1]
199
200
Richard Barnette3e8b2282018-05-15 20:42:20 +0000201 def _get_last_update_error(self):
Shuqian Zhaod9992722016-02-29 12:26:38 -0800202 """Get the last autoupdate error code."""
Richard Barnette3e8b2282018-05-15 20:42:20 +0000203 command_result = self._run(
204 '%s --last_attempt_error' % _UPDATER_BIN)
205 return command_result.stdout.strip().replace('\n', ', ')
Shuqian Zhaod9992722016-02-29 12:26:38 -0800206
207
Luigi Semenzatoe76d9f82016-11-21 11:15:10 -0800208 def _base_update_handler_no_retry(self, run_args):
Shuqian Zhaod9992722016-02-29 12:26:38 -0800209 """Base function to handle a remote update ssh call.
210
211 @param run_args: Dictionary of args passed to ssh_host.run function.
Shuqian Zhaod9992722016-02-29 12:26:38 -0800212
Luigi Semenzatoe76d9f82016-11-21 11:15:10 -0800213 @throws: intercepts and re-throws all exceptions
Shuqian Zhaod9992722016-02-29 12:26:38 -0800214 """
Shuqian Zhaod9992722016-02-29 12:26:38 -0800215 try:
216 self.host.run(**run_args)
Shuqian Zhaod9992722016-02-29 12:26:38 -0800217 except Exception as e:
Luigi Semenzatoe76d9f82016-11-21 11:15:10 -0800218 logging.debug('exception in update handler: %s', e)
219 raise e
Shuqian Zhaod9992722016-02-29 12:26:38 -0800220
Luigi Semenzatoe76d9f82016-11-21 11:15:10 -0800221
222 def _base_update_handler(self, run_args, err_msg_prefix=None):
223 """Handle a remote update ssh call, possibly with retries.
224
225 @param run_args: Dictionary of args passed to ssh_host.run function.
226 @param err_msg_prefix: Prefix of the exception error message.
227 """
228 def exception_handler(e):
229 """Examines exceptions and returns True if the update handler
230 should be retried.
231
232 @param e: the exception intercepted by the retry util.
233 """
234 return (isinstance(e, error.AutoservSSHTimeout) or
235 (isinstance(e, error.GenericHostRunError) and
236 hasattr(e, 'description') and
237 (re.search('ERROR_CODE=37', e.description) or
238 re.search('generic error .255.', e.description))))
239
240 try:
241 # Try the update twice (arg 2 is max_retry, not including the first
242 # call). Some exceptions may be caught by the retry handler.
243 retry_util.GenericRetry(exception_handler, 1,
244 self._base_update_handler_no_retry,
245 run_args)
246 except Exception as e:
247 message = err_msg_prefix + ': ' + str(e)
248 raise RootFSUpdateError(message)
Shuqian Zhaod9992722016-02-29 12:26:38 -0800249
250
Luigi Semenzatof15c8fc2017-03-03 14:12:40 -0800251 def _wait_for_update_service(self):
252 """Ensure that the update engine daemon is running, possibly
253 by waiting for it a bit in case the DUT just rebooted and the
254 service hasn't started yet.
255 """
256 def handler(e):
257 """Retry exception handler.
258
259 Assumes that the error is due to the update service not having
260 started yet.
261
262 @param e: the exception intercepted by the retry util.
263 """
264 if isinstance(e, error.AutoservRunError):
265 logging.debug('update service check exception: %s\n'
266 'retrying...', e)
267 return True
268 else:
269 return False
270
271 # Retry at most three times, every 5s.
272 status = retry_util.GenericRetry(handler, 3,
273 self.check_update_status,
274 sleep=5)
275
276 # Expect the update engine to be idle.
277 if status != UPDATER_IDLE:
278 raise ChromiumOSError('%s is not in an installable state' %
279 self.host.hostname)
280
281
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700282 def trigger_update(self):
283 """Triggers a background update.
284
Shuqian Zhaod9992722016-02-29 12:26:38 -0800285 @raise RootFSUpdateError or unknown Exception if anything went wrong.
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700286 """
Luigi Semenzatof15c8fc2017-03-03 14:12:40 -0800287 # If this function is called immediately after reboot (which it is at
288 # this time), there is no guarantee that the update service is up and
289 # running yet, so wait for it.
290 self._wait_for_update_service()
291
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700292 autoupdate_cmd = ('%s --check_for_update --omaha_url=%s' %
Richard Barnette3e8b2282018-05-15 20:42:20 +0000293 (_UPDATER_BIN, self.update_url))
Shuqian Zhaod9992722016-02-29 12:26:38 -0800294 run_args = {'command': autoupdate_cmd}
295 err_prefix = 'Failed to trigger an update on %s. ' % self.host.hostname
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700296 logging.info('Triggering update via: %s', autoupdate_cmd)
Allen Li1a5cc0a2017-06-20 14:08:59 -0700297 metric_fields = {'success': False}
Luigi Semenzatoe76d9f82016-11-21 11:15:10 -0800298 try:
Luigi Semenzatoe76d9f82016-11-21 11:15:10 -0800299 self._base_update_handler(run_args, err_prefix)
Allen Li1a5cc0a2017-06-20 14:08:59 -0700300 metric_fields['success'] = True
301 finally:
Allen Li1a5cc0a2017-06-20 14:08:59 -0700302 c = metrics.Counter('chromeos/autotest/autoupdater/trigger')
Allen Lib5420a72017-06-20 14:14:07 -0700303 metric_fields.update(self._get_metric_fields())
Allen Li1a5cc0a2017-06-20 14:08:59 -0700304 c.increment(fields=metric_fields)
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700305
306
Allen Lib5420a72017-06-20 14:14:07 -0700307 def _get_metric_fields(self):
308 """Return a dict of metric fields.
309
310 This is used for sending autoupdate metrics for this instance.
311 """
312 build_name = url_to_image_name(self.update_url)
313 try:
314 board, build_type, milestone, _ = server_utils.ParseBuildName(
315 build_name)
316 except server_utils.ParseBuildNameException:
317 logging.warning('Unable to parse build name %s for metrics. '
318 'Continuing anyway.', build_name)
319 board, build_type, milestone = ('', '', '')
320 return {
321 'dev_server': dev_server.get_hostname(self.update_url),
322 'board': board,
323 'build_type': build_type,
324 'milestone': milestone,
325 }
326
327
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700328 def _verify_update_completed(self):
329 """Verifies that an update has completed.
330
331 @raise RootFSUpdateError: if verification fails.
332 """
333 status = self.check_update_status()
334 if status != UPDATER_NEED_REBOOT:
Shuqian Zhaod9992722016-02-29 12:26:38 -0800335 error_msg = ''
336 if status == UPDATER_IDLE:
Richard Barnette3e8b2282018-05-15 20:42:20 +0000337 error_msg = 'Update error: %s' % self._get_last_update_error()
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700338 raise RootFSUpdateError('Update did not complete with correct '
Shuqian Zhaod9992722016-02-29 12:26:38 -0800339 'status. Expecting %s, actual %s. %s' %
340 (UPDATER_NEED_REBOOT, status, error_msg))
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700341
342
343 def update_image(self):
344 """Updates the device image and verifies success."""
Shuqian Zhaofe4d62e2016-06-23 14:46:45 -0700345 autoupdate_cmd = ('%s --update --omaha_url=%s' %
Richard Barnette3e8b2282018-05-15 20:42:20 +0000346 (_UPDATER_BIN, self.update_url))
David Haddock76a4c882017-12-13 18:50:09 -0800347 if not self.interactive:
348 autoupdate_cmd = '%s --interactive=false' % autoupdate_cmd
Shuqian Zhaod9992722016-02-29 12:26:38 -0800349 run_args = {'command': autoupdate_cmd, 'timeout': 3600}
350 err_prefix = ('Failed to install device image using payload at %s '
351 'on %s. ' % (self.update_url, self.host.hostname))
352 logging.info('Updating image via: %s', autoupdate_cmd)
Allen Li1a5cc0a2017-06-20 14:08:59 -0700353 metric_fields = {'success': False}
Luigi Semenzatoe76d9f82016-11-21 11:15:10 -0800354 try:
Luigi Semenzatoe76d9f82016-11-21 11:15:10 -0800355 self._base_update_handler(run_args, err_prefix)
Allen Li1a5cc0a2017-06-20 14:08:59 -0700356 metric_fields['success'] = True
357 finally:
Allen Li1a5cc0a2017-06-20 14:08:59 -0700358 c = metrics.Counter('chromeos/autotest/autoupdater/update')
Allen Lib5420a72017-06-20 14:14:07 -0700359 metric_fields.update(self._get_metric_fields())
Allen Li1a5cc0a2017-06-20 14:08:59 -0700360 c.increment(fields=metric_fields)
Aviv Keshetf37b2d72016-06-01 19:27:59 -0700361 self._verify_update_completed()
Gilad Arnoldd6adeb82015-09-21 07:10:03 -0700362
363
Richard Barnette3e8b2282018-05-15 20:42:20 +0000364 def _reset_update_engine(self):
Chris Sosae92399e2015-04-24 11:32:59 -0700365 """Resets the host to prepare for a clean update regardless of state."""
Chris Sosae92399e2015-04-24 11:32:59 -0700366 self._run('stop ui || true')
367 self._run('stop update-engine || true')
368 self._run('start update-engine')
Dale Curtis5c32c722011-05-04 19:24:23 -0700369
Shuqian Zhaofe4d62e2016-06-23 14:46:45 -0700370 # Wait for update engine to be ready.
Luigi Semenzatof15c8fc2017-03-03 14:12:40 -0800371 self._wait_for_update_service()
Sean Oc053dfe2010-08-23 18:22:26 +0200372
373
Richard Barnette3e8b2282018-05-15 20:42:20 +0000374 def _rootdev(self, options=''):
Dan Shi0f466e82013-02-22 15:44:58 -0800375 """Returns the stripped output of rootdev <options>.
376
377 @param options: options to run rootdev.
378
379 """
Dale Curtisa94c19c2011-05-02 15:05:17 -0700380 return self._run('rootdev %s' % options).stdout.strip()
381
382
383 def get_kernel_state(self):
384 """Returns the (<active>, <inactive>) kernel state as a pair."""
Richard Barnette3e8b2282018-05-15 20:42:20 +0000385 active_root = int(re.findall('\d+\Z', self._rootdev('-s'))[0])
386 if active_root == _KERNEL_A['root']:
387 return _KERNEL_A, _KERNEL_B
388 elif active_root == _KERNEL_B['root']:
389 return _KERNEL_B, _KERNEL_A
Dale Curtisa94c19c2011-05-02 15:05:17 -0700390 else:
Dale Curtis5c32c722011-05-04 19:24:23 -0700391 raise ChromiumOSError('Encountered unknown root partition: %s' %
Dale Curtisa94c19c2011-05-02 15:05:17 -0700392 active_root)
393
394
395 def _cgpt(self, flag, kernel, dev='$(rootdev -s -d)'):
396 """Return numeric cgpt value for the specified flag, kernel, device. """
397 return int(self._run('cgpt show -n -i %d %s %s' % (
398 kernel['kernel'], flag, dev)).stdout.strip())
399
400
Richard Barnette3e8b2282018-05-15 20:42:20 +0000401 def _get_kernel_priority(self, kernel):
Dan Shi0f466e82013-02-22 15:44:58 -0800402 """Return numeric priority for the specified kernel.
403
Richard Barnette3e8b2282018-05-15 20:42:20 +0000404 @param kernel: information of the given kernel, either _KERNEL_A
405 or _KERNEL_B.
Dan Shi0f466e82013-02-22 15:44:58 -0800406 """
Dale Curtisa94c19c2011-05-02 15:05:17 -0700407 return self._cgpt('-P', kernel)
408
409
Richard Barnette3e8b2282018-05-15 20:42:20 +0000410 def _get_kernel_success(self, kernel):
Dan Shi0f466e82013-02-22 15:44:58 -0800411 """Return boolean success flag for the specified kernel.
412
Richard Barnette3e8b2282018-05-15 20:42:20 +0000413 @param kernel: information of the given kernel, either _KERNEL_A
414 or _KERNEL_B.
Dan Shi0f466e82013-02-22 15:44:58 -0800415 """
Dale Curtisa94c19c2011-05-02 15:05:17 -0700416 return self._cgpt('-S', kernel) != 0
417
418
Richard Barnette3e8b2282018-05-15 20:42:20 +0000419 def _get_kernel_tries(self, kernel):
Dan Shi0f466e82013-02-22 15:44:58 -0800420 """Return tries count for the specified kernel.
421
Richard Barnette3e8b2282018-05-15 20:42:20 +0000422 @param kernel: information of the given kernel, either _KERNEL_A
423 or _KERNEL_B.
Dan Shi0f466e82013-02-22 15:44:58 -0800424 """
Dale Curtisa94c19c2011-05-02 15:05:17 -0700425 return self._cgpt('-T', kernel)
Sean O267c00b2010-08-31 15:54:55 +0200426
427
Chris Sosa5e4246b2012-05-22 18:05:22 -0700428 def get_stateful_update_script(self):
Gwendal Grignou3e96cc22017-06-07 16:22:51 -0700429 """Returns the path to the stateful update script on the target.
Chris Sosa5e4246b2012-05-22 18:05:22 -0700430
Gwendal Grignou3e96cc22017-06-07 16:22:51 -0700431 When runnning test_that, stateful_update is in chroot /usr/sbin,
432 as installed by chromeos-base/devserver packages.
433 In the lab, it is installed with the python module devserver, by
434 build_externals.py command.
Chris Sosaa3ac2152012-05-23 22:23:13 -0700435
Gwendal Grignou3e96cc22017-06-07 16:22:51 -0700436 If we can find it, we hope it exists already on the DUT, we assert
437 otherwise.
438 """
Richard Barnette3e8b2282018-05-15 20:42:20 +0000439 stateful_update_file = os.path.join(_STATEFUL_UPDATE_PATH,
440 _STATEFUL_UPDATE_SCRIPT)
Gwendal Grignou3e96cc22017-06-07 16:22:51 -0700441 if os.path.exists(stateful_update_file):
Chris Sosa5e4246b2012-05-22 18:05:22 -0700442 self.host.send_file(
Richard Barnette3e8b2282018-05-15 20:42:20 +0000443 stateful_update_file, _REMOTE_TMP_STATEFUL_UPDATE,
Gilad Arnold0c0df732015-09-21 06:37:59 -0700444 delete_dest=True)
Richard Barnette3e8b2282018-05-15 20:42:20 +0000445 return _REMOTE_TMP_STATEFUL_UPDATE
Chris Sosa5e4246b2012-05-22 18:05:22 -0700446
Richard Barnette3e8b2282018-05-15 20:42:20 +0000447 if self.host.path_exists(_REMOTE_STATEFUL_UPDATE_PATH):
Gwendal Grignou3e96cc22017-06-07 16:22:51 -0700448 logging.warning('Could not chroot %s script, falling back on %s',
Richard Barnette3e8b2282018-05-15 20:42:20 +0000449 _STATEFUL_UPDATE_SCRIPT,
450 _REMOTE_STATEFUL_UPDATE_PATH)
451 return _REMOTE_STATEFUL_UPDATE_PATH
Gwendal Grignou3e96cc22017-06-07 16:22:51 -0700452 else:
Richard Barnette3e8b2282018-05-15 20:42:20 +0000453 raise ChromiumOSError('Could not locate %s' %
454 _STATEFUL_UPDATE_SCRIPT)
Chris Sosa5e4246b2012-05-22 18:05:22 -0700455
456
Richard Barnette3e8b2282018-05-15 20:42:20 +0000457 def _reset_stateful_partition(self):
Dan Shi0f466e82013-02-22 15:44:58 -0800458 """Clear any pending stateful update request."""
Chris Sosa5e4246b2012-05-22 18:05:22 -0700459 statefuldev_cmd = [self.get_stateful_update_script()]
460 statefuldev_cmd += ['--stateful_change=reset', '2>&1']
Chris Sosa66d74072013-09-19 11:21:29 -0700461 self._run(' '.join(statefuldev_cmd))
Chris Sosa5e4246b2012-05-22 18:05:22 -0700462
463
Richard Barnette3e8b2282018-05-15 20:42:20 +0000464 def _revert_boot_partition(self):
Dan Shi0f466e82013-02-22 15:44:58 -0800465 """Revert the boot partition."""
Richard Barnette3e8b2282018-05-15 20:42:20 +0000466 part = self._rootdev('-s')
Ilja H. Friedel04be2bd2014-05-07 21:29:59 -0700467 logging.warning('Reverting update; Boot partition will be %s', part)
Sean O267c00b2010-08-31 15:54:55 +0200468 return self._run('/postinst %s 2>&1' % part)
469
470
Chris Sosac1932172013-10-16 13:28:53 -0700471 def rollback_rootfs(self, powerwash):
472 """Triggers rollback and waits for it to complete.
473
474 @param powerwash: If true, powerwash as part of rollback.
475
476 @raise RootFSUpdateError if anything went wrong.
477
478 """
Dan Shi549fb822015-03-24 18:01:11 -0700479 version = self.host.get_release_version()
Chris Sosac8617522014-06-09 23:22:26 +0000480 # Introduced can_rollback in M36 (build 5772). # etc/lsb-release matches
481 # X.Y.Z. This version split just pulls the first part out.
482 try:
483 build_number = int(version.split('.')[0])
484 except ValueError:
485 logging.error('Could not parse build number.')
486 build_number = 0
487
488 if build_number >= 5772:
Richard Barnette3e8b2282018-05-15 20:42:20 +0000489 can_rollback_cmd = '%s --can_rollback' % _UPDATER_BIN
Chris Sosac8617522014-06-09 23:22:26 +0000490 logging.info('Checking for rollback.')
491 try:
492 self._run(can_rollback_cmd)
493 except error.AutoservRunError as e:
494 raise RootFSUpdateError("Rollback isn't possible on %s: %s" %
495 (self.host.hostname, str(e)))
496
Richard Barnette3e8b2282018-05-15 20:42:20 +0000497 rollback_cmd = '%s --rollback --follow' % _UPDATER_BIN
Chris Sosac1932172013-10-16 13:28:53 -0700498 if not powerwash:
Dan Shif3a35f72016-01-25 11:18:14 -0800499 rollback_cmd += ' --nopowerwash'
Chris Sosac1932172013-10-16 13:28:53 -0700500
Chris Sosac8617522014-06-09 23:22:26 +0000501 logging.info('Performing rollback.')
Chris Sosac1932172013-10-16 13:28:53 -0700502 try:
503 self._run(rollback_cmd)
Chris Sosac1932172013-10-16 13:28:53 -0700504 except error.AutoservRunError as e:
505 raise RootFSUpdateError('Rollback failed on %s: %s' %
506 (self.host.hostname, str(e)))
507
508 self._verify_update_completed()
509
Gilad Arnold0ed760c2012-11-05 23:42:53 -0800510
Chris Sosa72312602013-04-16 15:01:56 -0700511 def update_stateful(self, clobber=True):
512 """Updates the stateful partition.
513
514 @param clobber: If True, a clean stateful installation.
515 """
Chris Sosa77556d82012-04-05 15:23:14 -0700516 logging.info('Updating stateful partition...')
joychen03eaad92013-06-26 09:55:21 -0700517 statefuldev_url = self.update_url.replace('update',
518 'static')
Chris Sosaa3ac2152012-05-23 22:23:13 -0700519
Dale Curtis5c32c722011-05-04 19:24:23 -0700520 # Attempt stateful partition update; this must succeed so that the newly
521 # installed host is testable after update.
Chris Sosa72312602013-04-16 15:01:56 -0700522 statefuldev_cmd = [self.get_stateful_update_script(), statefuldev_url]
523 if clobber:
524 statefuldev_cmd.append('--stateful_change=clean')
525
526 statefuldev_cmd.append('2>&1')
Dale Curtis5c32c722011-05-04 19:24:23 -0700527 try:
Dan Shi205b8732016-01-25 10:56:22 -0800528 self._run(' '.join(statefuldev_cmd), timeout=1200)
Dale Curtis5c32c722011-05-04 19:24:23 -0700529 except error.AutoservRunError:
Gilad Arnold62cf3a42015-10-01 09:15:25 -0700530 update_error = StatefulUpdateError(
531 'Failed to perform stateful update on %s' %
532 self.host.hostname)
Chris Sosa77556d82012-04-05 15:23:14 -0700533 raise update_error
Dale Curtis5c32c722011-05-04 19:24:23 -0700534
Chris Sosaa3ac2152012-05-23 22:23:13 -0700535
Richard Barnette0beb14b2018-05-15 18:07:52 +0000536 def _install_update(self, update_root=True):
537 """Install the requested image on the DUT, but don't start it.
538
539 This downloads all content needed for the requested update, and
540 installs it in place on the DUT. This does not reboot the DUT,
541 so the update is merely pending when the function returns.
542
543 @param update_root: When true, force a rootfs update; otherwise
544 update the stateful partition only.
Dan Shi0f466e82013-02-22 15:44:58 -0800545 """
Dan Shi549fb822015-03-24 18:01:11 -0700546 booted_version = self.host.get_release_version()
Richard Barnette0173ea82018-05-04 21:13:57 +0000547 logging.info('Updating from version %s to %s.',
548 booted_version, self.update_version)
Dale Curtis53d55862011-05-16 12:17:59 -0700549
Dale Curtis5c32c722011-05-04 19:24:23 -0700550 # Check that Dev Server is accepting connections (from autoserv's host).
Sean O'Connor5346e4e2010-08-12 18:49:24 +0200551 # If we can't talk to it, the machine host probably can't either.
xixuanccf2e722016-06-10 16:42:38 -0700552 auserver_host = 'http://%s' % urlparse.urlparse(self.update_url)[1]
Sean O'Connor5346e4e2010-08-12 18:49:24 +0200553 try:
xixuanccf2e722016-06-10 16:42:38 -0700554 if not dev_server.ImageServer.devserver_healthy(auserver_host):
555 raise ChromiumOSError(
556 'Update server at %s not healthy' % auserver_host)
557 except Exception as e:
558 logging.debug('Error happens in connection to devserver: %r', e)
Dale Curtis5c32c722011-05-04 19:24:23 -0700559 raise ChromiumOSError(
560 'Update server at %s not available' % auserver_host)
Sean O'Connor5346e4e2010-08-12 18:49:24 +0200561
Chris Sosaa3ac2152012-05-23 22:23:13 -0700562 logging.info('Installing from %s to %s', self.update_url,
Chris Sosa77556d82012-04-05 15:23:14 -0700563 self.host.hostname)
564
Chris Sosa5e4246b2012-05-22 18:05:22 -0700565 # Reset update state.
Richard Barnette3e8b2282018-05-15 20:42:20 +0000566 self._reset_update_engine()
567 self._reset_stateful_partition()
Sean Oc053dfe2010-08-23 18:22:26 +0200568
Dale Curtis1e973182011-07-12 18:21:36 -0700569 try:
Aviv Keshet2610d3e2016-06-01 16:37:01 -0700570 try:
571 if not update_root:
572 logging.info('Root update is skipped.')
573 else:
Richard Barnette3e8b2282018-05-15 20:42:20 +0000574 self.update_image()
Sean O'Connor5346e4e2010-08-12 18:49:24 +0200575
Aviv Keshet2610d3e2016-06-01 16:37:01 -0700576 self.update_stateful()
577 except:
Richard Barnette3e8b2282018-05-15 20:42:20 +0000578 self._revert_boot_partition()
579 self._reset_stateful_partition()
Aviv Keshet2610d3e2016-06-01 16:37:01 -0700580 raise
Sean Oc053dfe2010-08-23 18:22:26 +0200581
Dale Curtis1e973182011-07-12 18:21:36 -0700582 logging.info('Update complete.')
Dale Curtis1e973182011-07-12 18:21:36 -0700583 except:
584 # Collect update engine logs in the event of failure.
585 if self.host.job:
Aviv Keshet2610d3e2016-06-01 16:37:01 -0700586 logging.info('Collecting update engine logs due to failure...')
Dale Curtis1e973182011-07-12 18:21:36 -0700587 self.host.get_file(
Richard Barnette3e8b2282018-05-15 20:42:20 +0000588 _UPDATER_LOGS, self.host.job.sysinfo.sysinfodir,
Gilad Arnold0c0df732015-09-21 06:37:59 -0700589 preserve_perm=False)
Richard Barnette3e8b2282018-05-15 20:42:20 +0000590 _list_image_dir_contents(self.update_url)
Dale Curtis1e973182011-07-12 18:21:36 -0700591 raise
Dan Shi10e992b2013-08-30 11:02:59 -0700592 finally:
Shuqian Zhaod9992722016-02-29 12:26:38 -0800593 logging.info('Update engine log has downloaded in '
594 'sysinfo/update_engine dir. Check the lastest.')
Sean O'Connor5346e4e2010-08-12 18:49:24 +0200595
596
Richard Barnette0beb14b2018-05-15 18:07:52 +0000597 def _try_stateful_update(self):
598 """Try to use stateful update to initialize DUT.
599
600 When DUT is already running the same version that machine_install
601 tries to install, stateful update is a much faster way to clean up
602 the DUT for testing, compared to a full reimage. It is implemeted
603 by calling autoupdater._run_full_update, but skipping updating root,
604 as updating the kernel is time consuming and not necessary.
605
606 @param update_url: url of the image.
607 @param updater: ChromiumOSUpdater instance used to update the DUT.
608 @returns: True if the DUT was updated with stateful update.
609
610 """
611 self.host.prepare_for_update()
612
613 # TODO(jrbarnette): Yes, I hate this re.match() test case.
614 # It's better than the alternative: see crbug.com/360944.
615 image_name = url_to_image_name(self.update_url)
616 release_pattern = r'^.*-release/R[0-9]+-[0-9]+\.[0-9]+\.0$'
617 if not re.match(release_pattern, image_name):
618 return False
Richard Barnette3e8b2282018-05-15 20:42:20 +0000619 if not self._check_version():
Richard Barnette0beb14b2018-05-15 18:07:52 +0000620 return False
621 # Following folders should be rebuilt after stateful update.
622 # A test file is used to confirm each folder gets rebuilt after
623 # the stateful update.
624 folders_to_check = ['/var', '/home', '/mnt/stateful_partition']
625 test_file = '.test_file_to_be_deleted'
626 paths = [os.path.join(folder, test_file) for folder in folders_to_check]
627 self._run('touch %s' % ' '.join(paths))
628
629 self._install_update(update_root=False)
630
631 # Reboot to complete stateful update.
632 self.host.reboot(timeout=self.host.REBOOT_TIMEOUT, wait=True)
633
634 # After stateful update and a reboot, all of the test_files shouldn't
635 # exist any more. Otherwise the stateful update is failed.
636 return not any(
637 self.host.path_exists(os.path.join(folder, test_file))
638 for folder in folders_to_check)
639
640
641 def _post_update_processing(self, expected_kernel):
642 """After the DUT is updated, confirm machine_install succeeded.
643
644 @param updater: ChromiumOSUpdater instance used to update the DUT.
645 @param expected_kernel: kernel expected to be active after reboot,
646 or `None` to skip rollback checking.
647
648 """
649 # Touch the lab machine file to leave a marker that
650 # distinguishes this image from other test images.
651 # Afterwards, we must re-run the autoreboot script because
652 # it depends on the _LAB_MACHINE_FILE.
653 autoreboot_cmd = ('FILE="%s" ; [ -f "$FILE" ] || '
654 '( touch "$FILE" ; start autoreboot )')
Richard Barnette3e8b2282018-05-15 20:42:20 +0000655 self._run(autoreboot_cmd % _LAB_MACHINE_FILE)
Richard Barnette0beb14b2018-05-15 18:07:52 +0000656 self.verify_boot_expectations(
657 expected_kernel, rollback_message=
658 'Build %s failed to boot on %s; system rolled back to previous '
659 'build' % (self.update_version, self.host.hostname))
660
661 logging.debug('Cleaning up old autotest directories.')
662 try:
663 installed_autodir = autotest.Autotest.get_installed_autodir(
664 self.host)
665 self._run('rm -rf ' + installed_autodir)
666 except autotest.AutodirNotFoundError:
667 logging.debug('No autotest installed directory found.')
668
669
670 def run_update(self, force_full_update):
671 """Perform a full update of a DUT in the test lab.
672
673 This downloads and installs the root FS and stateful partition
674 content needed for the update specified in `self.host` and
675 `self.update_url`. The update is performed according to the
676 requirements for provisioning a DUT for testing the requested
677 build.
678
679 @param force_full_update: When true, update the root file
680 system to the new build, even if the target DUT already has
681 that build installed.
682 @returns A tuple of the form `(image_name, attributes)`, where
683 `image_name` is the name of the image installed, and
684 `attributes` is new attributes to be applied to the DUT.
685 """
686 logging.debug('Update URL is %s', self.update_url)
687
688 # Report provision stats.
689 server_name = dev_server.get_hostname(self.update_url)
690 (metrics.Counter('chromeos/autotest/provision/install')
691 .increment(fields={'devserver': server_name}))
692
693 # Create a file to indicate if provision fails. The file will be
694 # removed by any successful update.
695 self._run('touch %s' % PROVISION_FAILED)
696
697 update_complete = False
698 if not force_full_update:
699 try:
700 # If the DUT is already running the same build, try stateful
701 # update first as it's much quicker than a full re-image.
702 update_complete = self._try_stateful_update()
703 except Exception as e:
704 logging.exception(e)
705
706 inactive_kernel = None
707 if update_complete:
708 logging.info('Install complete without full update')
709 else:
710 logging.info('DUT requires full update.')
711 self.host.reboot(timeout=self.host.REBOOT_TIMEOUT, wait=True)
712 self.host.prepare_for_update()
713
714 self._install_update()
715
716 # Give it some time in case of IO issues.
717 time.sleep(10)
718
719 # Figure out active and inactive kernel.
720 active_kernel, inactive_kernel = self.get_kernel_state()
721
722 # Ensure inactive kernel has higher priority than active.
Richard Barnette3e8b2282018-05-15 20:42:20 +0000723 if (self._get_kernel_priority(inactive_kernel)
724 < self._get_kernel_priority(active_kernel)):
Richard Barnette0beb14b2018-05-15 18:07:52 +0000725 raise ChromiumOSError(
726 'Update failed. The priority of the inactive kernel'
727 ' partition is less than that of the active kernel'
728 ' partition.')
729
730 # Update has returned successfully; reboot the host.
731 #
732 # Regarding the 'crossystem' command below: In some cases,
733 # the update flow puts the TPM into a state such that it
734 # fails verification. We don't know why. However, this
735 # call papers over the problem by clearing the TPM during
736 # the reboot.
737 #
738 # We ignore failures from 'crossystem'. Although failure
739 # here is unexpected, and could signal a bug, the point of
740 # the exercise is to paper over problems; allowing this to
741 # fail would defeat the purpose.
742 self._run('crossystem clear_tpm_owner_request=1',
743 ignore_status=True)
744 self.host.reboot(timeout=self.host.REBOOT_TIMEOUT, wait=True)
745
746 self._post_update_processing(inactive_kernel)
747 image_name = url_to_image_name(self.update_url)
748 # update_url is different from devserver url needed to stage autotest
749 # packages, therefore, resolve a new devserver url here.
750 devserver_url = dev_server.ImageServer.resolve(
751 image_name, self.host.hostname).url()
752 repo_url = tools.get_package_url(devserver_url, image_name)
753 return image_name, {ds_constants.JOB_REPO_URL: repo_url}
754
755
Richard Barnette3e8b2282018-05-15 20:42:20 +0000756 def _check_version(self):
Dan Shi0f466e82013-02-22 15:44:58 -0800757 """Check the image running in DUT has the desired version.
758
759 @returns: True if the DUT's image version matches the version that
760 the autoupdater tries to update to.
761
762 """
Dan Shi549fb822015-03-24 18:01:11 -0700763 booted_version = self.host.get_release_version()
Richard Barnette0173ea82018-05-04 21:13:57 +0000764 return self.update_version.endswith(booted_version)
Dan Shib95bb862013-03-22 16:29:28 -0700765
766
Chris Sosa65425082013-10-16 13:26:22 -0700767 def verify_boot_expectations(self, expected_kernel_state, rollback_message):
768 """Verifies that we fully booted given expected kernel state.
769
770 This method both verifies that we booted using the correct kernel
771 state and that the OS has marked the kernel as good.
772
773 @param expected_kernel_state: kernel state that we are verifying with
774 i.e. I expect to be booted onto partition 4 etc. See output of
775 get_kernel_state.
776 @param rollback_message: string to raise as a ChromiumOSError
777 if we booted with the wrong partition.
778
779 @raises ChromiumOSError: If we didn't.
780 """
781 # Figure out the newly active kernel.
782 active_kernel_state = self.get_kernel_state()[0]
783
784 # Check for rollback due to a bad build.
785 if (expected_kernel_state and
786 active_kernel_state != expected_kernel_state):
Don Garrett56b1cc82013-12-06 17:49:20 -0800787
788 # Kernel crash reports should be wiped between test runs, but
789 # may persist from earlier parts of the test, or from problems
790 # with provisioning.
791 #
792 # Kernel crash reports will NOT be present if the crash happened
793 # before encrypted stateful is mounted.
794 #
795 # TODO(dgarrett): Integrate with server/crashcollect.py at some
796 # point.
797 kernel_crashes = glob.glob('/var/spool/crash/kernel.*.kcrash')
798 if kernel_crashes:
799 rollback_message += ': kernel_crash'
800 logging.debug('Found %d kernel crash reports:',
801 len(kernel_crashes))
802 # The crash names contain timestamps that may be useful:
803 # kernel.20131207.005945.0.kcrash
804 for crash in kernel_crashes:
Dan Shi0942b1d2015-03-31 11:07:00 -0700805 logging.debug(' %s', os.path.basename(crash))
Don Garrett56b1cc82013-12-06 17:49:20 -0800806
Chris Sosa65425082013-10-16 13:26:22 -0700807 # Print out some information to make it easier to debug
808 # the rollback.
809 logging.debug('Dumping partition table.')
810 self._run('cgpt show $(rootdev -s -d)')
811 logging.debug('Dumping crossystem for firmware debugging.')
812 self._run('crossystem --all')
813 raise ChromiumOSError(rollback_message)
814
815 # Make sure chromeos-setgoodkernel runs.
816 try:
817 utils.poll_for_condition(
Richard Barnette3e8b2282018-05-15 20:42:20 +0000818 lambda: (self._get_kernel_tries(active_kernel_state) == 0
819 and self._get_kernel_success(active_kernel_state)),
Chris Sosa65425082013-10-16 13:26:22 -0700820 exception=ChromiumOSError(),
Richard Barnette3e8b2282018-05-15 20:42:20 +0000821 timeout=_KERNEL_UPDATE_TIMEOUT, sleep_interval=5)
Chris Sosa65425082013-10-16 13:26:22 -0700822 except ChromiumOSError:
823 services_status = self._run('status system-services').stdout
824 if services_status != 'system-services start/running\n':
825 event = ('Chrome failed to reach login screen')
826 else:
827 event = ('update-engine failed to call '
828 'chromeos-setgoodkernel')
829 raise ChromiumOSError(
830 'After update and reboot, %s '
Richard Barnette3e8b2282018-05-15 20:42:20 +0000831 'within %d seconds' % (event, _KERNEL_UPDATE_TIMEOUT))