blob: fc4154d5bc5587b64edef09f31bf172447d6712c [file] [log] [blame]
J. Richard Barnette24adbf42012-04-11 15:04:53 -07001# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
Dale Curtisaa5eedb2011-08-23 16:18:52 -07002# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
Aviv Keshet74c89a92013-02-04 15:18:30 -08005import functools
J. Richard Barnette1d78b012012-05-15 13:56:30 -07006import logging
Dan Shi0f466e82013-02-22 15:44:58 -08007import os
Simran Basid5e5e272012-09-24 15:23:59 -07008import re
Christopher Wileyd78249a2013-03-01 13:05:31 -08009import socket
J. Richard Barnette1d78b012012-05-15 13:56:30 -070010import subprocess
J. Richard Barnette134ec2c2012-04-25 12:59:37 -070011import time
J. Richard Barnette1d78b012012-05-15 13:56:30 -070012import xmlrpclib
J. Richard Barnette134ec2c2012-04-25 12:59:37 -070013
J. Richard Barnette45e93de2012-04-11 17:24:15 -070014from autotest_lib.client.bin import utils
Richard Barnette0c73ffc2012-11-19 15:21:18 -080015from autotest_lib.client.common_lib import error
16from autotest_lib.client.common_lib import global_config
J. Richard Barnette45e93de2012-04-11 17:24:15 -070017from autotest_lib.client.common_lib.cros import autoupdater
Richard Barnette03a0c132012-11-05 12:40:35 -080018from autotest_lib.client.common_lib.cros import dev_server
Christopher Wileyd78249a2013-03-01 13:05:31 -080019from autotest_lib.client.common_lib.cros import retry
Richard Barnette82c35912012-11-20 10:09:10 -080020from autotest_lib.client.cros import constants
J. Richard Barnette45e93de2012-04-11 17:24:15 -070021from autotest_lib.server import autoserv_parser
Chris Sosaf4d43ff2012-10-30 11:21:05 -070022from autotest_lib.server import autotest
J. Richard Barnette45e93de2012-04-11 17:24:15 -070023from autotest_lib.server import site_host_attributes
Scott Zawalski89c44dd2013-02-26 09:28:02 -050024from autotest_lib.server.cros.dynamic_suite import constants as ds_constants
25from autotest_lib.server.cros.dynamic_suite import tools
J. Richard Barnette75487572013-03-08 12:47:50 -080026from autotest_lib.server.cros.servo import servo
J. Richard Barnette45e93de2012-04-11 17:24:15 -070027from autotest_lib.server.hosts import remote
Simran Basidcff4252012-11-20 16:13:20 -080028from autotest_lib.site_utils.rpm_control_system import rpm_client
Simran Basid5e5e272012-09-24 15:23:59 -070029
Richard Barnette82c35912012-11-20 10:09:10 -080030# Importing frontend.afe.models requires a full Autotest
31# installation (with the Django modules), not just the source
32# repository. Most developers won't have the full installation, so
33# the imports below will fail for them.
34#
35# The fix is to catch import exceptions, and set `models` to `None`
36# on failure. This has the side effect that
37# SiteHost._get_board_from_afe() will fail: That will manifest as
38# failures during Repair jobs leaving the DUT as "Repair Failed".
39# In practice, you can't test Repair jobs without a full
40# installation, so that kind of failure isn't expected.
41try:
J. Richard Barnette7214e0b2013-02-06 15:20:49 -080042 # pylint: disable=W0611
Richard Barnette82c35912012-11-20 10:09:10 -080043 from autotest_lib.frontend import setup_django_environment
44 from autotest_lib.frontend.afe import models
45except:
46 models = None
47
Simran Basid5e5e272012-09-24 15:23:59 -070048
J. Richard Barnettebe5ebcc2013-02-11 16:03:15 -080049def _make_servo_hostname(hostname):
50 host_parts = hostname.split('.')
51 host_parts[0] = host_parts[0] + '-servo'
52 return '.'.join(host_parts)
53
54
55def _get_lab_servo(target_hostname):
56 """Instantiate a Servo for |target_hostname| in the lab.
57
58 Assuming that |target_hostname| is a device in the CrOS test
59 lab, create and return a Servo object pointed at the servo
60 attached to that DUT. The servo in the test lab is assumed
61 to already have servod up and running on it.
62
63 @param target_hostname: device whose servo we want to target.
64 @return an appropriately configured Servo instance.
65 """
66 servo_host = _make_servo_hostname(target_hostname)
67 if utils.host_is_in_lab_zone(servo_host):
68 try:
J. Richard Barnetted5f807a2013-02-11 16:51:00 -080069 return servo.Servo(servo_host=servo_host)
J. Richard Barnettebe5ebcc2013-02-11 16:03:15 -080070 except: # pylint: disable=W0702
71 # TODO(jrbarnette): Long-term, if we can't get to
72 # a servo in the lab, we want to fail, so we should
73 # pass any exceptions along. Short-term, we're not
74 # ready to rely on servo, so we ignore failures.
75 pass
76 return None
77
78
Dale Curtiscb7bfaf2011-06-07 16:21:57 -070079def make_ssh_command(user='root', port=22, opts='', hosts_file=None,
80 connect_timeout=None, alive_interval=None):
81 """Override default make_ssh_command to use options tuned for Chrome OS.
82
83 Tuning changes:
Chris Sosaf7fcd6e2011-09-27 17:30:47 -070084 - ConnectTimeout=30; maximum of 30 seconds allowed for an SSH connection
85 failure. Consistency with remote_access.sh.
Dale Curtiscb7bfaf2011-06-07 16:21:57 -070086
Dale Curtisaa5eedb2011-08-23 16:18:52 -070087 - ServerAliveInterval=180; which causes SSH to ping connection every
88 180 seconds. In conjunction with ServerAliveCountMax ensures that if the
89 connection dies, Autotest will bail out quickly. Originally tried 60 secs,
90 but saw frequent job ABORTS where the test completed successfully.
Dale Curtiscb7bfaf2011-06-07 16:21:57 -070091
Chris Sosaf7fcd6e2011-09-27 17:30:47 -070092 - ServerAliveCountMax=3; consistency with remote_access.sh.
93
94 - ConnectAttempts=4; reduce flakiness in connection errors; consistency
95 with remote_access.sh.
Dale Curtiscb7bfaf2011-06-07 16:21:57 -070096
97 - UserKnownHostsFile=/dev/null; we don't care about the keys. Host keys
98 change with every new installation, don't waste memory/space saving them.
Chris Sosaf7fcd6e2011-09-27 17:30:47 -070099
100 - SSH protocol forced to 2; needed for ServerAliveInterval.
J. Richard Barnette7214e0b2013-02-06 15:20:49 -0800101
102 @param user User name to use for the ssh connection.
103 @param port Port on the target host to use for ssh connection.
104 @param opts Additional options to the ssh command.
105 @param hosts_file Ignored.
106 @param connect_timeout Ignored.
107 @param alive_interval Ignored.
Dale Curtiscb7bfaf2011-06-07 16:21:57 -0700108 """
109 base_command = ('/usr/bin/ssh -a -x %s -o StrictHostKeyChecking=no'
110 ' -o UserKnownHostsFile=/dev/null -o BatchMode=yes'
Chris Sosaf7fcd6e2011-09-27 17:30:47 -0700111 ' -o ConnectTimeout=30 -o ServerAliveInterval=180'
112 ' -o ServerAliveCountMax=3 -o ConnectionAttempts=4'
113 ' -o Protocol=2 -l %s -p %d')
Dale Curtiscb7bfaf2011-06-07 16:21:57 -0700114 return base_command % (opts, user, port)
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700115
116
J. Richard Barnette7214e0b2013-02-06 15:20:49 -0800117
Aviv Keshet74c89a92013-02-04 15:18:30 -0800118def add_label_detector(label_function_list, label_list=None, label=None):
119 """Decorator used to group functions together into the provided list.
120 @param label_function_list: List of label detecting functions to add
121 decorated function to.
122 @param label_list: List of detectable labels to add detectable labels to.
123 (Default: None)
124 @param label: Label string that is detectable by this detection function
125 (Default: None)
J. Richard Barnette7214e0b2013-02-06 15:20:49 -0800126 """
Simran Basic6f1f7a2012-10-16 10:47:46 -0700127 def add_func(func):
Aviv Keshet74c89a92013-02-04 15:18:30 -0800128 """
129 @param func: The function to be added as a detector.
130 """
131 label_function_list.append(func)
132 if label and label_list is not None:
133 label_list.append(label)
Simran Basic6f1f7a2012-10-16 10:47:46 -0700134 return func
135 return add_func
136
137
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700138class SiteHost(remote.RemoteHost):
139 """Chromium OS specific subclass of Host."""
140
141 _parser = autoserv_parser.autoserv_parser
142
Richard Barnette0c73ffc2012-11-19 15:21:18 -0800143 # Time to wait for new kernel to be marked successful after
144 # auto update.
Chris Masone163cead2012-05-16 11:49:48 -0700145 _KERNEL_UPDATE_TIMEOUT = 120
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700146
Richard Barnette03a0c132012-11-05 12:40:35 -0800147 # Timeout values (in seconds) associated with various Chrome OS
148 # state changes.
J. Richard Barnette134ec2c2012-04-25 12:59:37 -0700149 #
Richard Barnette0c73ffc2012-11-19 15:21:18 -0800150 # In general, a good rule of thumb is that the timeout can be up
151 # to twice the typical measured value on the slowest platform.
152 # The times here have not necessarily been empirically tested to
153 # meet this criterion.
J. Richard Barnetteeb69d722012-06-18 17:29:44 -0700154 #
155 # SLEEP_TIMEOUT: Time to allow for suspend to memory.
Richard Barnette0c73ffc2012-11-19 15:21:18 -0800156 # RESUME_TIMEOUT: Time to allow for resume after suspend, plus
157 # time to restart the netwowrk.
J. Richard Barnetteeb69d722012-06-18 17:29:44 -0700158 # BOOT_TIMEOUT: Time to allow for boot from power off. Among
Richard Barnette0c73ffc2012-11-19 15:21:18 -0800159 # other things, this must account for the 30 second dev-mode
J. Richard Barnetted4649c62013-03-06 17:42:27 -0800160 # screen delay and time to start the network.
J. Richard Barnetteeb69d722012-06-18 17:29:44 -0700161 # USB_BOOT_TIMEOUT: Time to allow for boot from a USB device,
Richard Barnette0c73ffc2012-11-19 15:21:18 -0800162 # including the 30 second dev-mode delay and time to start the
J. Richard Barnetted4649c62013-03-06 17:42:27 -0800163 # network.
Richard Barnette0c73ffc2012-11-19 15:21:18 -0800164 # SHUTDOWN_TIMEOUT: Time to allow for shut down.
J. Richard Barnetteeb69d722012-06-18 17:29:44 -0700165 # REBOOT_TIMEOUT: Combination of shutdown and reboot times.
Richard Barnette03a0c132012-11-05 12:40:35 -0800166 # _INSTALL_TIMEOUT: Time to allow for chromeos-install.
J. Richard Barnetteeb69d722012-06-18 17:29:44 -0700167
168 SLEEP_TIMEOUT = 2
J. Richard Barnetted4649c62013-03-06 17:42:27 -0800169 RESUME_TIMEOUT = 10
J. Richard Barnetteeb69d722012-06-18 17:29:44 -0700170 BOOT_TIMEOUT = 45
171 USB_BOOT_TIMEOUT = 150
172 SHUTDOWN_TIMEOUT = 5
173 REBOOT_TIMEOUT = SHUTDOWN_TIMEOUT + BOOT_TIMEOUT
Richard Barnette03a0c132012-11-05 12:40:35 -0800174 _INSTALL_TIMEOUT = 240
175
Ismail Noorbasha07fdb612013-02-14 14:13:31 -0800176 # _USB_POWER_TIMEOUT: Time to allow for USB to power toggle ON and OFF.
177 # _POWER_CYCLE_TIMEOUT: Time to allow for manual power cycle.
178 _USB_POWER_TIMEOUT = 5
179 _POWER_CYCLE_TIMEOUT = 10
180
Richard Barnette03a0c132012-11-05 12:40:35 -0800181 _DEFAULT_SERVO_URL_FORMAT = ('/static/servo-images/'
182 '%(board)s_test_image.bin')
183
J. Richard Barnettec14897e2013-03-06 15:56:55 -0800184 # TODO(jrbarnette): Servo repair is restricted to specific
185 # boards, because the existing servo client code doesn't account
186 # for board-specific differences in handling for 'cold_reset'.
187 # http://crosbug.com/36973
188 _SERVO_REPAIR_WHITELIST = ('x86-alex', 'lumpy')
Richard Barnette0c73ffc2012-11-19 15:21:18 -0800189
190
Richard Barnette82c35912012-11-20 10:09:10 -0800191 _RPM_RECOVERY_BOARDS = global_config.global_config.get_config_value('CROS',
192 'rpm_recovery_boards', type=str).split(',')
193
194 _MAX_POWER_CYCLE_ATTEMPTS = 6
195 _LAB_MACHINE_FILE = '/mnt/stateful_partition/.labmachine'
196 _RPM_HOSTNAME_REGEX = ('chromeos[0-9]+(-row[0-9]+)?-rack[0-9]+[a-z]*-'
197 'host[0-9]+')
198 _LIGHTSENSOR_FILES = ['in_illuminance0_input',
199 'in_illuminance0_raw',
200 'illuminance0_input']
201 _LIGHTSENSOR_SEARCH_DIR = '/sys/bus/iio/devices'
202 _LABEL_FUNCTIONS = []
Aviv Keshet74c89a92013-02-04 15:18:30 -0800203 _DETECTABLE_LABELS = []
204 label_decorator = functools.partial(add_label_detector, _LABEL_FUNCTIONS,
205 _DETECTABLE_LABELS)
J. Richard Barnette134ec2c2012-04-25 12:59:37 -0700206
J. Richard Barnetteb6de7e32013-02-14 13:28:04 -0800207 # Constants used in ping_wait_up() and ping_wait_down().
208 #
209 # _PING_WAIT_COUNT is the approximate number of polling
210 # cycles to use when waiting for a host state change.
211 #
212 # _PING_STATUS_DOWN and _PING_STATUS_UP are names used
213 # for arguments to the internal _ping_wait_for_status()
214 # method.
215 _PING_WAIT_COUNT = 40
216 _PING_STATUS_DOWN = False
217 _PING_STATUS_UP = True
218
Ismail Noorbasha07fdb612013-02-14 14:13:31 -0800219 # Allowed values for the power_method argument.
220
221 # POWER_CONTROL_RPM: Passed as default arg for power_off/on/cycle() methods.
222 # POWER_CONTROL_SERVO: Used in set_power() and power_cycle() methods.
223 # POWER_CONTROL_MANUAL: Used in set_power() and power_cycle() methods.
224 POWER_CONTROL_RPM = 'RPM'
225 POWER_CONTROL_SERVO = 'servoj10'
226 POWER_CONTROL_MANUAL = 'manual'
227
228 POWER_CONTROL_VALID_ARGS = (POWER_CONTROL_RPM,
229 POWER_CONTROL_SERVO,
230 POWER_CONTROL_MANUAL)
Richard Barnette0c73ffc2012-11-19 15:21:18 -0800231
J. Richard Barnette964fba02012-10-24 17:34:29 -0700232 @staticmethod
J. Richard Barnette7214e0b2013-02-06 15:20:49 -0800233 def get_servo_arguments(args_dict):
234 """Extract servo options from `args_dict` and return the result.
235
236 Take the provided dictionary of argument options and return
237 a subset that represent standard arguments needed to
238 construct a servo object for a host. The intent is to
239 provide standard argument processing from run_remote_tests
240 for tests that require a servo to operate.
241
242 Recommended usage:
243 ~~~~~~~~
244 args_dict = utils.args_to_dict(args)
245 servo_args = hosts.SiteHost.get_servo_arguments(args_dict)
246 host = hosts.create_host(machine, servo_args=servo_args)
247 ~~~~~~~~
248
249 @param args_dict Dictionary from which to extract the servo
250 arguments.
251 """
J. Richard Barnette964fba02012-10-24 17:34:29 -0700252 servo_args = {}
253 for arg in ('servo_host', 'servo_port'):
J. Richard Barnette7214e0b2013-02-06 15:20:49 -0800254 if arg in args_dict:
255 servo_args[arg] = args_dict[arg]
J. Richard Barnette964fba02012-10-24 17:34:29 -0700256 return servo_args
J. Richard Barnette134ec2c2012-04-25 12:59:37 -0700257
J. Richard Barnette964fba02012-10-24 17:34:29 -0700258
259 def _initialize(self, hostname, servo_args=None, *args, **dargs):
J. Richard Barnette67ccb872012-04-19 16:34:56 -0700260 """Initialize superclasses, and |self.servo|.
261
262 For creating the host servo object, there are three
263 possibilities: First, if the host is a lab system known to
264 have a servo board, we connect to that servo unconditionally.
265 Second, if we're called from a control file that requires
J. Richard Barnette55fb8062012-05-23 10:29:31 -0700266 servo features for testing, it will pass settings for
267 `servo_host`, `servo_port`, or both. If neither of these
268 cases apply, `self.servo` will be `None`.
J. Richard Barnette67ccb872012-04-19 16:34:56 -0700269
270 """
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700271 super(SiteHost, self)._initialize(hostname=hostname,
272 *args, **dargs)
J. Richard Barnettef0859852012-08-20 14:55:50 -0700273 # self.env is a dictionary of environment variable settings
274 # to be exported for commands run on the host.
275 # LIBC_FATAL_STDERR_ can be useful for diagnosing certain
276 # errors that might happen.
277 self.env['LIBC_FATAL_STDERR_'] = '1'
J. Richard Barnette1d78b012012-05-15 13:56:30 -0700278 self._xmlrpc_proxy_map = {}
J. Richard Barnettebe5ebcc2013-02-11 16:03:15 -0800279 self.servo = _get_lab_servo(hostname)
J. Richard Barnettead7da482012-10-30 16:46:52 -0700280 if not self.servo and servo_args is not None:
J. Richard Barnette964fba02012-10-24 17:34:29 -0700281 self.servo = servo.Servo(**servo_args)
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700282
283
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500284 def get_repair_image_name(self):
285 """Generate a image_name from variables in the global config.
286
287 @returns a str of $board-version/$BUILD.
288
289 """
290 stable_version = global_config.global_config.get_config_value(
291 'CROS', 'stable_cros_version')
292 build_pattern = global_config.global_config.get_config_value(
293 'CROS', 'stable_build_pattern')
294 board = self._get_board_from_afe()
295 if board is None:
296 raise error.AutoservError('DUT has no board attribute, '
297 'cannot be repaired.')
298 return build_pattern % (board, stable_version)
299
300
301 def clear_cros_version_labels_and_job_repo_url(self):
302 """Clear cros_version labels and host attribute job_repo_url."""
303 host_model = models.Host.objects.get(hostname=self.hostname)
304 for label in host_model.labels.iterator():
305 if not label.name.startswith(ds_constants.VERSION_PREFIX):
306 continue
Dan Shi0f466e82013-02-22 15:44:58 -0800307
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500308 label.host_set.remove(host_model)
309
310 host_model.set_or_delete_attribute('job_repo_url', None)
311
312
Dan Shi0f466e82013-02-22 15:44:58 -0800313 def _try_stateful_update(self, update_url, force_update, updater):
314 """Try to use stateful update to initialize DUT.
315
316 When DUT is already running the same version that machine_install
317 tries to install, stateful update is a much faster way to clean up
318 the DUT for testing, compared to a full reimage. It is implemeted
319 by calling autoupdater.run_update, but skipping updating root, as
320 updating the kernel is time consuming and not necessary.
321
322 @param update_url: url of the image.
323 @param force_update: Set to True to update the image even if the DUT
324 is running the same version.
325 @param updater: ChromiumOSUpdater instance used to update the DUT.
326 @returns: True if the DUT was updated with stateful update.
327
328 """
329 if not updater.check_version():
330 return False
331 if not force_update:
332 logging.info('Canceling stateful update because the new and '
333 'old versions are the same.')
334 return False
335 # Following folders should be rebuilt after stateful update.
336 # A test file is used to confirm each folder gets rebuilt after
337 # the stateful update.
338 folders_to_check = ['/var', '/home', '/mnt/stateful_partition']
339 test_file = '.test_file_to_be_deleted'
340 for folder in folders_to_check:
341 touch_path = os.path.join(folder, test_file)
342 self.run('touch %s' % touch_path)
343
344 if not updater.run_update(force_update=True, update_root=False):
345 return False
346
347 # Reboot to complete stateful update.
348 self.reboot(timeout=60, wait=True)
349 check_file_cmd = 'test -f %s; echo $?'
350 for folder in folders_to_check:
351 test_file_path = os.path.join(folder, test_file)
352 result = self.run(check_file_cmd % test_file_path,
353 ignore_status=True)
354 if result.exit_status == 1:
355 return False
356 return True
357
358
359 def _post_update_processing(self, updater, inactive_kernel=None):
360 """After the DUT is updated, confirm machine_install succeeded.
361
362 @param updater: ChromiumOSUpdater instance used to update the DUT.
363 @param inactive_kernel: kernel state of inactive kernel before reboot.
364
365 """
366
367 # Touch the lab machine file to leave a marker that distinguishes
368 # this image from other test images.
369 self.run('touch %s' % self._LAB_MACHINE_FILE)
370
371 # Kick off the autoreboot script as the _LAB_MACHINE_FILE was
372 # missing on the first boot.
373 self.run('start autoreboot')
374
375 # Following the reboot, verify the correct version.
376 if not updater.check_version():
377 # Print out crossystem to make it easier to debug the rollback.
378 logging.debug('Dumping partition table.')
379 self.host.run('cgpt show $(rootdev -s -d)')
380 logging.debug('Dumping crossystem for firmware debugging.')
381 self.host.run('crossystem --all')
382 logging.error('Expected Chromium OS version: %s. '
383 'Found Chromium OS %s',
384 self.update_version, updater.get_build_id())
385 raise ChromiumOSError('Updater failed on host %s' %
386 self.host.hostname)
387
388 # Figure out newly active kernel.
389 new_active_kernel, _ = updater.get_kernel_state()
390
391 # Ensure that previously inactive kernel is now the active kernel.
392 if inactive_kernel and new_active_kernel != inactive_kernel:
393 raise autoupdater.ChromiumOSError(
394 'Update failed. New kernel partition is not active after'
395 ' boot.')
396
397 host_attributes = site_host_attributes.HostAttributes(self.hostname)
398 if host_attributes.has_chromeos_firmware:
399 # Wait until tries == 0 and success, or until timeout.
400 utils.poll_for_condition(
401 lambda: (updater.get_kernel_tries(new_active_kernel) == 0
402 and updater.get_kernel_success(new_active_kernel)),
403 exception=autoupdater.ChromiumOSError(
404 'Update failed. Timed out waiting for system to mark'
405 ' new kernel as successful.'),
406 timeout=self._KERNEL_UPDATE_TIMEOUT, sleep_interval=5)
407
408
Chris Sosaa3ac2152012-05-23 22:23:13 -0700409 def machine_install(self, update_url=None, force_update=False,
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500410 local_devserver=False, repair=False):
411 """Install the DUT.
412
Dan Shi0f466e82013-02-22 15:44:58 -0800413 Use stateful update if the DUT is already running the same build.
414 Stateful update does not update kernel and tends to run much faster
415 than a full reimage. If the DUT is running a different build, or it
416 failed to do a stateful update, full update, including kernel update,
417 will be applied to the DUT.
418
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500419 @param update_url: The url to use for the update
420 pattern: http://$devserver:###/update/$build
421 If update_url is None and repair is True we will install the
422 stable image listed in global_config under
423 CROS.stable_cros_version.
424 @param force_update: Force an update even if the version installed
425 is the same. Default:False
426 @param local_devserver: Used by run_remote_test to allow people to
427 use their local devserver. Default: False
428 @param repair: Whether or not we are in repair mode. This adds special
429 cases for repairing a machine like starting update_engine.
430 Setting repair to True sets force_update to True as well.
431 default: False
432 @raises autoupdater.ChromiumOSError
433
434 """
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700435 if not update_url and self._parser.options.image:
436 update_url = self._parser.options.image
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500437 elif not update_url and not repair:
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700438 raise autoupdater.ChromiumOSError(
439 'Update failed. No update URL provided.')
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500440 elif not update_url and repair:
441 image_name = self.get_repair_image_name()
442 devserver = dev_server.ImageServer.resolve(image_name)
443 logging.info('Staging repair build: %s', image_name)
444 devserver.trigger_download(image_name, synchronous=False)
445 self.clear_cros_version_labels_and_job_repo_url()
446 update_url = tools.image_url_pattern() % (devserver.url(),
447 image_name)
448
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500449 if repair:
Dan Shi0f466e82013-02-22 15:44:58 -0800450 # In case the system is in a bad state, we always reboot the machine
451 # before machine_install.
452 self.reboot(timeout=60, wait=True)
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500453 self.run('stop update-engine; start update-engine')
454 force_update = True
Dan Shi0f466e82013-02-22 15:44:58 -0800455
Chris Sosaa3ac2152012-05-23 22:23:13 -0700456 updater = autoupdater.ChromiumOSUpdater(update_url, host=self,
Dan Shi0f466e82013-02-22 15:44:58 -0800457 local_devserver=local_devserver)
458 updated = False
459 # If the DUT is already running the same build, try stateful update
460 # first. Stateful update does not update kernel and tends to run much
461 # faster than a full reimage.
462 try:
463 updated = self._try_stateful_update(update_url, force_update,
464 updater)
465 if updated:
466 logging.info('DUT is updated with stateful update.')
467 except Exception as e:
468 logging.exception(e)
469 logging.warn('Failed to stateful update DUT, force to update.')
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700470
Dan Shi0f466e82013-02-22 15:44:58 -0800471 inactive_kernel = None
472 # Do a full update if stateful update is not applicable or failed.
473 if not updated:
474 # In case the system is in a bad state, we always reboot the
475 # machine before machine_install.
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700476 self.reboot(timeout=60, wait=True)
Dan Shi0f466e82013-02-22 15:44:58 -0800477 if updater.run_update(force_update):
478 updated = True
479 # Figure out active and inactive kernel.
480 active_kernel, inactive_kernel = updater.get_kernel_state()
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700481
Dan Shi0f466e82013-02-22 15:44:58 -0800482 # Ensure inactive kernel has higher priority than active.
483 if (updater.get_kernel_priority(inactive_kernel)
484 < updater.get_kernel_priority(active_kernel)):
485 raise autoupdater.ChromiumOSError(
486 'Update failed. The priority of the inactive kernel'
487 ' partition is less than that of the active kernel'
488 ' partition.')
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700489
Dan Shi0f466e82013-02-22 15:44:58 -0800490 update_engine_log = '/var/log/update_engine.log'
491 logging.info('Dumping %s', update_engine_log)
492 self.run('cat %s' % update_engine_log)
493 # Updater has returned successfully; reboot the host.
494 self.reboot(timeout=60, wait=True)
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700495
Dan Shi0f466e82013-02-22 15:44:58 -0800496 if updated:
497 self._post_update_processing(updater, inactive_kernel)
Simran Basi13fa1ba2013-03-04 10:56:47 -0800498
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700499 # Clean up any old autotest directories which may be lying around.
500 for path in global_config.global_config.get_config_value(
501 'AUTOSERV', 'client_autodir_paths', type=list):
502 self.run('rm -rf ' + path)
503
504
Simran Basi833814b2013-01-29 13:13:43 -0800505 def _get_label_from_afe(self, label_prefix):
506 """Retrieve a host's specific label from the AFE.
507
508 Looks for a host label that has the form <label_prefix>:<value>
509 and returns the "<value>" part of the label. None is returned
510 if there is not a label matching the pattern
511
512 @returns the label that matches the prefix or 'None'
513 """
514 host_model = models.Host.objects.get(hostname=self.hostname)
515 host_label = host_model.labels.get(name__startswith=label_prefix)
516 if not host_label:
517 return None
518 return host_label.name.split(label_prefix, 1)[1]
519
520
Richard Barnette82c35912012-11-20 10:09:10 -0800521 def _get_board_from_afe(self):
522 """Retrieve this host's board from its labels in the AFE.
523
524 Looks for a host label of the form "board:<board>", and
525 returns the "<board>" part of the label. `None` is returned
526 if there is not a single, unique label matching the pattern.
527
528 @returns board from label, or `None`.
529 """
Simran Basi833814b2013-01-29 13:13:43 -0800530 return self._get_label_from_afe(ds_constants.BOARD_PREFIX)
531
532
533 def get_build(self):
534 """Retrieve the current build for this Host from the AFE.
535
536 Looks through this host's labels in the AFE to determine its build.
537
538 @returns The current build or None if it could not find it or if there
539 were multiple build labels assigned to this host.
540 """
541 return self._get_label_from_afe(ds_constants.VERSION_PREFIX)
Richard Barnette82c35912012-11-20 10:09:10 -0800542
543
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500544 def _install_repair(self):
545 """Attempt to repair this host using upate-engine.
546
547 If the host is up, try installing the DUT with a stable
548 "repair" version of Chrome OS as defined in the global_config
549 under CROS.stable_cros_version.
550
551 @returns True if successful, False if update_engine failed.
552
553 """
554 if not self.is_up():
555 return False
556
557 logging.info('Attempting to reimage machine to repair image.')
558 try:
559 self.machine_install(repair=True)
560 except autoupdater.ChromiumOSError:
561 logging.info('Repair via install failed.')
562 return False
563
564 return True
565
566
Richard Barnette03a0c132012-11-05 12:40:35 -0800567 def _servo_repair(self, board):
568 """Attempt to repair this host using an attached Servo.
569
570 Re-install the OS on the DUT by 1) installing a test image
571 on a USB storage device attached to the Servo board,
572 2) booting that image in recovery mode, and then
573 3) installing the image.
574
575 """
576 server = dev_server.ImageServer.devserver_url_for_servo(board)
577 image = server + (self._DEFAULT_SERVO_URL_FORMAT %
578 { 'board': board })
579 self.servo.install_recovery_image(image)
580 if not self.wait_up(timeout=self.USB_BOOT_TIMEOUT):
581 raise error.AutoservError('DUT failed to boot from USB'
582 ' after %d seconds' %
583 self.USB_BOOT_TIMEOUT)
584 self.run('chromeos-install --yes',
585 timeout=self._INSTALL_TIMEOUT)
586 self.servo.power_long_press()
587 self.servo.set('usb_mux_sel1', 'servo_sees_usbkey')
588 self.servo.power_short_press()
589 if not self.wait_up(timeout=self.BOOT_TIMEOUT):
590 raise error.AutoservError('DUT failed to reboot installed '
591 'test image after %d seconds' %
592 self.BOOT_TIMEOUT)
593
594
Richard Barnette82c35912012-11-20 10:09:10 -0800595 def _powercycle_to_repair(self):
596 """Utilize the RPM Infrastructure to bring the host back up.
597
598 If the host is not up/repaired after the first powercycle we utilize
599 auto fallback to the last good install by powercycling and rebooting the
600 host 6 times.
601 """
602 logging.info('Attempting repair via RPM powercycle.')
603 failed_cycles = 0
604 self.power_cycle()
605 while not self.wait_up(timeout=self.BOOT_TIMEOUT):
606 failed_cycles += 1
607 if failed_cycles >= self._MAX_POWER_CYCLE_ATTEMPTS:
608 raise error.AutoservError('Powercycled host %s %d times; '
609 'device did not come back online.' %
610 (self.hostname, failed_cycles))
611 self.power_cycle()
612 if failed_cycles == 0:
613 logging.info('Powercycling was successful first time.')
614 else:
615 logging.info('Powercycling was successful after %d failures.',
616 failed_cycles)
617
618
619 def repair_full(self):
620 """Repair a host for repair level NO_PROTECTION.
621
622 This overrides the base class function for repair; it does
623 not call back to the parent class, but instead offers a
624 simplified implementation based on the capabilities in the
625 Chrome OS test lab.
626
J. Richard Barnettefde55fc2013-03-15 17:47:01 -0700627 If `self.verify()` fails, the following procedures are
628 attempted:
629 1. Try to re-install to a known stable image using
630 auto-update.
631 2. If there's a servo for the DUT, try to re-install via
632 the servo.
633 3. If the DUT can be power-cycled via RPM, try to repair
Richard Barnette82c35912012-11-20 10:09:10 -0800634 by power-cycling.
635
636 As with the parent method, the last operation performed on
637 the DUT must be to call `self.verify()`; if that call fails,
638 the exception it raises is passed back to the caller.
J. Richard Barnettefde55fc2013-03-15 17:47:01 -0700639
Richard Barnette82c35912012-11-20 10:09:10 -0800640 """
641 try:
642 self.verify()
643 except:
644 host_board = self._get_board_from_afe()
Richard Barnette03a0c132012-11-05 12:40:35 -0800645 if host_board is None:
646 logging.error('host %s has no board; failing repair',
647 self.hostname)
Richard Barnette82c35912012-11-20 10:09:10 -0800648 raise
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500649
J. Richard Barnettefde55fc2013-03-15 17:47:01 -0700650 if not self._install_repair():
651 # TODO(scottz): All repair pathways should be
652 # executed until we've exhausted all options. Below
653 # we favor servo over powercycle when we really
654 # should be falling back to power if servo fails.
655 if (self.servo and
656 host_board in self._SERVO_REPAIR_WHITELIST):
657 self._servo_repair(host_board)
658 elif (self.has_power() and
659 host_board in self._RPM_RECOVERY_BOARDS):
660 self._powercycle_to_repair()
661 else:
662 logging.error('host %s has no servo and no RPM control; '
663 'failing repair', self.hostname)
664 raise
Richard Barnette82c35912012-11-20 10:09:10 -0800665 self.verify()
666
667
J. Richard Barnette1d78b012012-05-15 13:56:30 -0700668 def close(self):
669 super(SiteHost, self).close()
670 self.xmlrpc_disconnect_all()
671
672
Chris Sosaf4d43ff2012-10-30 11:21:05 -0700673 def cleanup(self):
Chris Sosaf4d43ff2012-10-30 11:21:05 -0700674 client_at = autotest.Autotest(self)
Richard Barnette82c35912012-11-20 10:09:10 -0800675 self.run('rm -f %s' % constants.CLEANUP_LOGS_PAUSED_FILE)
Scott Zawalskiddbc31e2012-11-15 11:29:01 -0500676 try:
677 client_at.run_static_method('autotest_lib.client.cros.cros_ui',
678 '_clear_login_prompt_state')
679 self.run('restart ui')
680 client_at.run_static_method('autotest_lib.client.cros.cros_ui',
681 '_wait_for_login_prompt')
Alex Millerf4517962013-02-25 15:03:02 -0800682 except (error.AutotestRunError, error.AutoservRunError):
Scott Zawalskiddbc31e2012-11-15 11:29:01 -0500683 logging.warn('Unable to restart ui, rebooting device.')
684 # Since restarting the UI fails fall back to normal Autotest
685 # cleanup routines, i.e. reboot the machine.
686 super(SiteHost, self).cleanup()
Chris Sosaf4d43ff2012-10-30 11:21:05 -0700687
688
Simran Basi154f5582012-10-23 16:27:11 -0700689 # TODO (sbasi) crosbug.com/35656
690 # Renamed the sitehost cleanup method so we don't go down this pathway.
691 # def cleanup(self):
692 def cleanup_poweron(self):
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700693 """Special cleanup method to make sure hosts always get power back."""
Chris Sosa9479fcd2012-10-09 13:44:22 -0700694 super(SiteHost, self).cleanup()
Simran Basid5e5e272012-09-24 15:23:59 -0700695 if self.has_power():
Simran Basifd23fb22012-10-22 17:56:22 -0700696 try:
697 self.power_on()
Chris Sosafab08082013-01-04 15:21:20 -0800698 except rpm_client.RemotePowerException:
Simran Basifd23fb22012-10-22 17:56:22 -0700699 # If cleanup has completed but there was an issue with the RPM
700 # Infrastructure, log an error message rather than fail cleanup
701 logging.error('Failed to turn Power On for this host after '
702 'cleanup through the RPM Infrastructure.')
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700703
704
Yu-Ju Honga2be94a2012-07-31 09:48:52 -0700705 def reboot(self, **dargs):
706 """
707 This function reboots the site host. The more generic
708 RemoteHost.reboot() performs sync and sleeps for 5
709 seconds. This is not necessary for Chrome OS devices as the
710 sync should be finished in a short time during the reboot
711 command.
712 """
Tom Wai-Hong Tamf5cd1d42012-08-13 12:04:08 +0800713 if 'reboot_cmd' not in dargs:
714 dargs['reboot_cmd'] = ('((reboot & sleep 10; reboot -f &)'
715 ' </dev/null >/dev/null 2>&1 &)')
Yu-Ju Honga2be94a2012-07-31 09:48:52 -0700716 # Enable fastsync to avoid running extra sync commands before reboot.
Tom Wai-Hong Tamf5cd1d42012-08-13 12:04:08 +0800717 if 'fastsync' not in dargs:
718 dargs['fastsync'] = True
Yu-Ju Honga2be94a2012-07-31 09:48:52 -0700719 super(SiteHost, self).reboot(**dargs)
720
721
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700722 def verify_software(self):
Richard Barnetteb2bc13c2013-01-08 17:32:51 -0800723 """Verify working software on a Chrome OS system.
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700724
Richard Barnetteb2bc13c2013-01-08 17:32:51 -0800725 Tests for the following conditions:
726 1. All conditions tested by the parent version of this
727 function.
728 2. Sufficient space in /mnt/stateful_partition.
729 3. update_engine answers a simple status request over DBus.
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700730
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700731 """
732 super(SiteHost, self).verify_software()
733 self.check_diskspace(
734 '/mnt/stateful_partition',
735 global_config.global_config.get_config_value(
736 'SERVER', 'gb_diskspace_required', type=int,
737 default=20))
Richard Barnetteb2bc13c2013-01-08 17:32:51 -0800738 self.run('update_engine_client --status')
Scott Zawalskifbca4a92013-03-04 15:56:42 -0500739 # Makes sure python is present, loads and can use built in functions.
740 # We have seen cases where importing cPickle fails with undefined
741 # symbols in cPickle.so.
742 self.run('python -c "import cPickle"')
J. Richard Barnette134ec2c2012-04-25 12:59:37 -0700743
744
Christopher Wileyd78249a2013-03-01 13:05:31 -0800745 def xmlrpc_connect(self, command, port, command_name=None,
746 ready_test_name=None, timeout_seconds=10):
J. Richard Barnette1d78b012012-05-15 13:56:30 -0700747 """Connect to an XMLRPC server on the host.
748
749 The `command` argument should be a simple shell command that
750 starts an XMLRPC server on the given `port`. The command
751 must not daemonize, and must terminate cleanly on SIGTERM.
752 The command is started in the background on the host, and a
753 local XMLRPC client for the server is created and returned
754 to the caller.
755
756 Note that the process of creating an XMLRPC client makes no
757 attempt to connect to the remote server; the caller is
758 responsible for determining whether the server is running
759 correctly, and is ready to serve requests.
760
Christopher Wileyd78249a2013-03-01 13:05:31 -0800761 Optionally, the caller can pass ready_test_name, a string
762 containing the name of a method to call on the proxy. This
763 method should take no parameters and return successfully only
764 when the server is ready to process client requests. When
765 ready_test_name is set, xmlrpc_connect will block until the
766 proxy is ready, and throw a TestError if the server isn't
767 ready by timeout_seconds.
768
J. Richard Barnette1d78b012012-05-15 13:56:30 -0700769 @param command Shell command to start the server.
770 @param port Port number on which the server is expected to
771 be serving.
J. Richard Barnette7214e0b2013-02-06 15:20:49 -0800772 @param command_name String to use as input to `pkill` to
773 terminate the XMLRPC server on the host.
Christopher Wileyd78249a2013-03-01 13:05:31 -0800774 @param ready_test_name String containing the name of a
775 method defined on the XMLRPC server.
776 @param timeout_seconds Number of seconds to wait
777 for the server to become 'ready.' Will throw a
778 TestFail error if server is not ready in time.
779
J. Richard Barnette1d78b012012-05-15 13:56:30 -0700780 """
781 self.xmlrpc_disconnect(port)
782
783 # Chrome OS on the target closes down most external ports
784 # for security. We could open the port, but doing that
785 # would conflict with security tests that check that only
786 # expected ports are open. So, to get to the port on the
787 # target we use an ssh tunnel.
788 local_port = utils.get_unused_port()
789 tunnel_options = '-n -N -q -L %d:localhost:%d' % (local_port, port)
790 ssh_cmd = make_ssh_command(opts=tunnel_options)
791 tunnel_cmd = '%s %s' % (ssh_cmd, self.hostname)
792 logging.debug('Full tunnel command: %s', tunnel_cmd)
793 tunnel_proc = subprocess.Popen(tunnel_cmd, shell=True, close_fds=True)
794 logging.debug('Started XMLRPC tunnel, local = %d'
795 ' remote = %d, pid = %d',
796 local_port, port, tunnel_proc.pid)
797
798 # Start the server on the host. Redirection in the command
799 # below is necessary, because 'ssh' won't terminate until
800 # background child processes close stdin, stdout, and
801 # stderr.
802 remote_cmd = '( %s ) </dev/null >/dev/null 2>&1 & echo $!' % command
803 remote_pid = self.run(remote_cmd).stdout.rstrip('\n')
804 logging.debug('Started XMLRPC server on host %s, pid = %s',
805 self.hostname, remote_pid)
806
J. Richard Barnette7214e0b2013-02-06 15:20:49 -0800807 self._xmlrpc_proxy_map[port] = (command_name, tunnel_proc)
J. Richard Barnette1d78b012012-05-15 13:56:30 -0700808 rpc_url = 'http://localhost:%d' % local_port
Christopher Wileyd78249a2013-03-01 13:05:31 -0800809 proxy = xmlrpclib.ServerProxy(rpc_url, allow_none=True)
810 if ready_test_name is not None:
J. Richard Barnette13eb7c02013-03-07 12:06:29 -0800811 # retry.retry logs each attempt; calculate delay_sec to
812 # keep log spam to a dull roar.
Christopher Wileyd78249a2013-03-01 13:05:31 -0800813 @retry.retry((socket.error, xmlrpclib.ProtocolError),
814 timeout_min=timeout_seconds/60.0,
J. Richard Barnette13eb7c02013-03-07 12:06:29 -0800815 delay_sec=min(max(timeout_seconds/20.0, 0.1), 1))
Christopher Wileyd78249a2013-03-01 13:05:31 -0800816 def ready_test():
817 """ Call proxy.ready_test_name(). """
818 getattr(proxy, ready_test_name)()
819 successful = False
820 try:
821 logging.info('Waiting %d seconds for XMLRPC server '
822 'to start.', timeout_seconds)
823 ready_test()
824 successful = True
825 except retry.TimeoutException:
826 raise error.TestError('Unable to start XMLRPC server after '
827 '%d seconds.' % timeout_seconds)
828 finally:
829 if not successful:
830 logging.error('Failed to start XMLRPC server.')
831 self.xmlrpc_disconnect(port)
832 logging.info('XMLRPC server started successfully.')
833 return proxy
J. Richard Barnette1d78b012012-05-15 13:56:30 -0700834
835 def xmlrpc_disconnect(self, port):
836 """Disconnect from an XMLRPC server on the host.
837
838 Terminates the remote XMLRPC server previously started for
839 the given `port`. Also closes the local ssh tunnel created
840 for the connection to the host. This function does not
841 directly alter the state of a previously returned XMLRPC
842 client object; however disconnection will cause all
843 subsequent calls to methods on the object to fail.
844
845 This function does nothing if requested to disconnect a port
846 that was not previously connected via `self.xmlrpc_connect()`
847
848 @param port Port number passed to a previous call to
849 `xmlrpc_connect()`
850 """
851 if port not in self._xmlrpc_proxy_map:
852 return
853 entry = self._xmlrpc_proxy_map[port]
854 remote_name = entry[0]
855 tunnel_proc = entry[1]
856 if remote_name:
857 # We use 'pkill' to find our target process rather than
858 # a PID, because the host may have rebooted since
859 # connecting, and we don't want to kill an innocent
860 # process with the same PID.
861 #
862 # 'pkill' helpfully exits with status 1 if no target
863 # process is found, for which run() will throw an
Simran Basid5e5e272012-09-24 15:23:59 -0700864 # exception. We don't want that, so we the ignore
J. Richard Barnette1d78b012012-05-15 13:56:30 -0700865 # status.
866 self.run("pkill -f '%s'" % remote_name, ignore_status=True)
867
868 if tunnel_proc.poll() is None:
869 tunnel_proc.terminate()
870 logging.debug('Terminated tunnel, pid %d', tunnel_proc.pid)
871 else:
872 logging.debug('Tunnel pid %d terminated early, status %d',
873 tunnel_proc.pid, tunnel_proc.returncode)
874 del self._xmlrpc_proxy_map[port]
875
876
877 def xmlrpc_disconnect_all(self):
878 """Disconnect all known XMLRPC proxy ports."""
879 for port in self._xmlrpc_proxy_map.keys():
880 self.xmlrpc_disconnect(port)
881
882
J. Richard Barnetteb6de7e32013-02-14 13:28:04 -0800883 def _ping_check_status(self, status):
884 """Ping the host once, and return whether it has a given status.
J. Richard Barnette134ec2c2012-04-25 12:59:37 -0700885
J. Richard Barnetteb6de7e32013-02-14 13:28:04 -0800886 @param status Check the ping status against this value.
887 @return True iff `status` and the result of ping are the same
888 (i.e. both True or both False).
889
890 """
891 ping_val = utils.ping(self.hostname, tries=1, deadline=1)
892 return not (status ^ (ping_val == 0))
893
894 def _ping_wait_for_status(self, status, timeout):
895 """Wait for the host to have a given status (UP or DOWN).
896
897 Status is checked by polling. Polling will not last longer
898 than the number of seconds in `timeout`. The polling
899 interval will be long enough that only approximately
900 _PING_WAIT_COUNT polling cycles will be executed, subject
901 to a maximum interval of about one minute.
902
903 @param status Waiting will stop immediately if `ping` of the
904 host returns this status.
905 @param timeout Poll for at most this many seconds.
906 @return True iff the host status from `ping` matched the
907 requested status at the time of return.
908
909 """
910 # _ping_check_status() takes about 1 second, hence the
911 # "- 1" in the formula below.
912 poll_interval = min(int(timeout / self._PING_WAIT_COUNT), 60) - 1
913 end_time = time.time() + timeout
914 while time.time() <= end_time:
915 if self._ping_check_status(status):
916 return True
917 if poll_interval > 0:
918 time.sleep(poll_interval)
919
920 # The last thing we did was sleep(poll_interval), so it may
921 # have been too long since the last `ping`. Check one more
922 # time, just to be sure.
923 return self._ping_check_status(status)
924
925 def ping_wait_up(self, timeout):
926 """Wait for the host to respond to `ping`.
927
928 N.B. This method is not a reliable substitute for
929 `wait_up()`, because a host that responds to ping will not
930 necessarily respond to ssh. This method should only be used
931 if the target DUT can be considered functional even if it
932 can't be reached via ssh.
933
934 @param timeout Minimum time to allow before declaring the
935 host to be non-responsive.
936 @return True iff the host answered to ping before the timeout.
937
938 """
939 return self._ping_wait_for_status(self._PING_STATUS_UP, timeout)
J. Richard Barnette134ec2c2012-04-25 12:59:37 -0700940
Andrew Bresticker678c0c72013-01-22 10:44:09 -0800941 def ping_wait_down(self, timeout):
J. Richard Barnette134ec2c2012-04-25 12:59:37 -0700942 """Wait until the host no longer responds to `ping`.
943
J. Richard Barnetteb6de7e32013-02-14 13:28:04 -0800944 This function can be used as a slightly faster version of
945 `wait_down()`, by avoiding potentially long ssh timeouts.
946
947 @param timeout Minimum time to allow for the host to become
948 non-responsive.
949 @return True iff the host quit answering ping before the
950 timeout.
951
J. Richard Barnette134ec2c2012-04-25 12:59:37 -0700952 """
J. Richard Barnetteb6de7e32013-02-14 13:28:04 -0800953 return self._ping_wait_for_status(self._PING_STATUS_DOWN, timeout)
J. Richard Barnette134ec2c2012-04-25 12:59:37 -0700954
955 def test_wait_for_sleep(self):
956 """Wait for the client to enter low-power sleep mode.
957
958 The test for "is asleep" can't distinguish a system that is
959 powered off; to confirm that the unit was asleep, it is
960 necessary to force resume, and then call
961 `test_wait_for_resume()`.
962
963 This function is expected to be called from a test as part
964 of a sequence like the following:
965
966 ~~~~~~~~
967 boot_id = host.get_boot_id()
968 # trigger sleep on the host
969 host.test_wait_for_sleep()
970 # trigger resume on the host
971 host.test_wait_for_resume(boot_id)
972 ~~~~~~~~
973
974 @exception TestFail The host did not go to sleep within
975 the allowed time.
976 """
Andrew Bresticker678c0c72013-01-22 10:44:09 -0800977 if not self.ping_wait_down(timeout=self.SLEEP_TIMEOUT):
J. Richard Barnette134ec2c2012-04-25 12:59:37 -0700978 raise error.TestFail(
979 'client failed to sleep after %d seconds' %
J. Richard Barnetteeb69d722012-06-18 17:29:44 -0700980 self.SLEEP_TIMEOUT)
J. Richard Barnette134ec2c2012-04-25 12:59:37 -0700981
982
983 def test_wait_for_resume(self, old_boot_id):
984 """Wait for the client to resume from low-power sleep mode.
985
986 The `old_boot_id` parameter should be the value from
987 `get_boot_id()` obtained prior to entering sleep mode. A
988 `TestFail` exception is raised if the boot id changes.
989
990 See @ref test_wait_for_sleep for more on this function's
991 usage.
992
J. Richard Barnette7214e0b2013-02-06 15:20:49 -0800993 @param old_boot_id A boot id value obtained before the
J. Richard Barnette134ec2c2012-04-25 12:59:37 -0700994 target host went to sleep.
995
996 @exception TestFail The host did not respond within the
997 allowed time.
998 @exception TestFail The host responded, but the boot id test
999 indicated a reboot rather than a sleep
1000 cycle.
1001 """
J. Richard Barnetteeb69d722012-06-18 17:29:44 -07001002 if not self.wait_up(timeout=self.RESUME_TIMEOUT):
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001003 raise error.TestFail(
1004 'client failed to resume from sleep after %d seconds' %
J. Richard Barnetteeb69d722012-06-18 17:29:44 -07001005 self.RESUME_TIMEOUT)
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001006 else:
1007 new_boot_id = self.get_boot_id()
1008 if new_boot_id != old_boot_id:
1009 raise error.TestFail(
1010 'client rebooted, but sleep was expected'
1011 ' (old boot %s, new boot %s)'
1012 % (old_boot_id, new_boot_id))
1013
1014
1015 def test_wait_for_shutdown(self):
1016 """Wait for the client to shut down.
1017
1018 The test for "has shut down" can't distinguish a system that
1019 is merely asleep; to confirm that the unit was down, it is
1020 necessary to force boot, and then call test_wait_for_boot().
1021
1022 This function is expected to be called from a test as part
1023 of a sequence like the following:
1024
1025 ~~~~~~~~
1026 boot_id = host.get_boot_id()
1027 # trigger shutdown on the host
1028 host.test_wait_for_shutdown()
1029 # trigger boot on the host
1030 host.test_wait_for_boot(boot_id)
1031 ~~~~~~~~
1032
1033 @exception TestFail The host did not shut down within the
1034 allowed time.
1035 """
Andrew Bresticker678c0c72013-01-22 10:44:09 -08001036 if not self.ping_wait_down(timeout=self.SHUTDOWN_TIMEOUT):
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001037 raise error.TestFail(
1038 'client failed to shut down after %d seconds' %
J. Richard Barnetteeb69d722012-06-18 17:29:44 -07001039 self.SHUTDOWN_TIMEOUT)
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001040
1041
1042 def test_wait_for_boot(self, old_boot_id=None):
1043 """Wait for the client to boot from cold power.
1044
1045 The `old_boot_id` parameter should be the value from
1046 `get_boot_id()` obtained prior to shutting down. A
1047 `TestFail` exception is raised if the boot id does not
1048 change. The boot id test is omitted if `old_boot_id` is not
1049 specified.
1050
1051 See @ref test_wait_for_shutdown for more on this function's
1052 usage.
1053
J. Richard Barnette7214e0b2013-02-06 15:20:49 -08001054 @param old_boot_id A boot id value obtained before the
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001055 shut down.
1056
1057 @exception TestFail The host did not respond within the
1058 allowed time.
1059 @exception TestFail The host responded, but the boot id test
1060 indicated that there was no reboot.
1061 """
J. Richard Barnetteeb69d722012-06-18 17:29:44 -07001062 if not self.wait_up(timeout=self.REBOOT_TIMEOUT):
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001063 raise error.TestFail(
1064 'client failed to reboot after %d seconds' %
J. Richard Barnetteeb69d722012-06-18 17:29:44 -07001065 self.REBOOT_TIMEOUT)
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001066 elif old_boot_id:
1067 if self.get_boot_id() == old_boot_id:
1068 raise error.TestFail(
1069 'client is back up, but did not reboot'
1070 ' (boot %s)' % old_boot_id)
Simran Basid5e5e272012-09-24 15:23:59 -07001071
1072
1073 @staticmethod
1074 def check_for_rpm_support(hostname):
1075 """For a given hostname, return whether or not it is powered by an RPM.
1076
1077 @return None if this host does not follows the defined naming format
1078 for RPM powered DUT's in the lab. If it does follow the format,
1079 it returns a regular expression MatchObject instead.
1080 """
Richard Barnette82c35912012-11-20 10:09:10 -08001081 return re.match(SiteHost._RPM_HOSTNAME_REGEX, hostname)
Simran Basid5e5e272012-09-24 15:23:59 -07001082
1083
1084 def has_power(self):
1085 """For this host, return whether or not it is powered by an RPM.
1086
1087 @return True if this host is in the CROS lab and follows the defined
1088 naming format.
1089 """
1090 return SiteHost.check_for_rpm_support(self.hostname)
1091
1092
Ismail Noorbasha07fdb612013-02-14 14:13:31 -08001093 def _set_power(self, state, power_method):
1094 """Sets the power to the host via RPM, Servo or manual.
1095
1096 @param state Specifies which power state to set to DUT
1097 @param power_method Specifies which method of power control to
1098 use. By default "RPM" will be used. Valid values
1099 are the strings "RPM", "manual", "servoj10".
1100
1101 """
1102 ACCEPTABLE_STATES = ['ON', 'OFF']
1103
1104 if state.upper() not in ACCEPTABLE_STATES:
1105 raise error.TestError('State must be one of: %s.'
1106 % (ACCEPTABLE_STATES,))
1107
1108 if power_method == self.POWER_CONTROL_SERVO:
1109 logging.info('Setting servo port J10 to %s', state)
1110 self.servo.set('prtctl3_pwren', state.lower())
1111 time.sleep(self._USB_POWER_TIMEOUT)
1112 elif power_method == self.POWER_CONTROL_MANUAL:
1113 logging.info('You have %d seconds to set the AC power to %s.',
1114 self._POWER_CYCLE_TIMEOUT, state)
1115 time.sleep(self._POWER_CYCLE_TIMEOUT)
1116 else:
1117 if not self.has_power():
1118 raise error.TestFail('DUT does not have RPM connected.')
1119 rpm_client.set_power(self.hostname, state.upper())
Simran Basid5e5e272012-09-24 15:23:59 -07001120
1121
Ismail Noorbasha07fdb612013-02-14 14:13:31 -08001122 def power_off(self, power_method=POWER_CONTROL_RPM):
1123 """Turn off power to this host via RPM, Servo or manual.
1124
1125 @param power_method Specifies which method of power control to
1126 use. By default "RPM" will be used. Valid values
1127 are the strings "RPM", "manual", "servoj10".
1128
1129 """
1130 self._set_power('OFF', power_method)
Simran Basid5e5e272012-09-24 15:23:59 -07001131
1132
Ismail Noorbasha07fdb612013-02-14 14:13:31 -08001133 def power_on(self, power_method=POWER_CONTROL_RPM):
1134 """Turn on power to this host via RPM, Servo or manual.
1135
1136 @param power_method Specifies which method of power control to
1137 use. By default "RPM" will be used. Valid values
1138 are the strings "RPM", "manual", "servoj10".
1139
1140 """
1141 self._set_power('ON', power_method)
1142
1143
1144 def power_cycle(self, power_method=POWER_CONTROL_RPM):
1145 """Cycle power to this host by turning it OFF, then ON.
1146
1147 @param power_method Specifies which method of power control to
1148 use. By default "RPM" will be used. Valid values
1149 are the strings "RPM", "manual", "servoj10".
1150
1151 """
1152 if power_method in (self.POWER_CONTROL_SERVO,
1153 self.POWER_CONTROL_MANUAL):
1154 self.power_off(power_method=power_method)
1155 time.sleep(self._POWER_CYCLE_TIMEOUT)
1156 self.power_on(power_method=power_method)
1157 else:
1158 rpm_client.set_power(self.hostname, 'CYCLE')
Simran Basic6f1f7a2012-10-16 10:47:46 -07001159
1160
1161 def get_platform(self):
1162 """Determine the correct platform label for this host.
1163
1164 @returns a string representing this host's platform.
1165 """
1166 crossystem = utils.Crossystem(self)
1167 crossystem.init()
1168 # Extract fwid value and use the leading part as the platform id.
1169 # fwid generally follow the format of {platform}.{firmware version}
1170 # Example: Alex.X.YYY.Z or Google_Alex.X.YYY.Z
1171 platform = crossystem.fwid().split('.')[0].lower()
1172 # Newer platforms start with 'Google_' while the older ones do not.
1173 return platform.replace('google_', '')
1174
1175
Aviv Keshet74c89a92013-02-04 15:18:30 -08001176 @label_decorator()
Simran Basic6f1f7a2012-10-16 10:47:46 -07001177 def get_board(self):
1178 """Determine the correct board label for this host.
1179
1180 @returns a string representing this host's board.
1181 """
1182 release_info = utils.parse_cmd_output('cat /etc/lsb-release',
1183 run_method=self.run)
1184 board = release_info['CHROMEOS_RELEASE_BOARD']
1185 # Devices in the lab generally have the correct board name but our own
1186 # development devices have {board_name}-signed-{key_type}. The board
1187 # name may also begin with 'x86-' which we need to keep.
Simran Basi833814b2013-01-29 13:13:43 -08001188 board_format_string = ds_constants.BOARD_PREFIX + '%s'
Simran Basic6f1f7a2012-10-16 10:47:46 -07001189 if 'x86' not in board:
Simran Basi833814b2013-01-29 13:13:43 -08001190 return board_format_string % board.split('-')[0]
1191 return board_format_string % '-'.join(board.split('-')[0:2])
Simran Basic6f1f7a2012-10-16 10:47:46 -07001192
1193
Aviv Keshet74c89a92013-02-04 15:18:30 -08001194 @label_decorator('lightsensor')
Simran Basic6f1f7a2012-10-16 10:47:46 -07001195 def has_lightsensor(self):
1196 """Determine the correct board label for this host.
1197
1198 @returns the string 'lightsensor' if this host has a lightsensor or
1199 None if it does not.
1200 """
1201 search_cmd = "find -L %s -maxdepth 4 | egrep '%s'" % (
Richard Barnette82c35912012-11-20 10:09:10 -08001202 self._LIGHTSENSOR_SEARCH_DIR, '|'.join(self._LIGHTSENSOR_FILES))
Simran Basic6f1f7a2012-10-16 10:47:46 -07001203 try:
1204 # Run the search cmd following the symlinks. Stderr_tee is set to
1205 # None as there can be a symlink loop, but the command will still
1206 # execute correctly with a few messages printed to stderr.
1207 self.run(search_cmd, stdout_tee=None, stderr_tee=None)
1208 return 'lightsensor'
1209 except error.AutoservRunError:
1210 # egrep exited with a return code of 1 meaning none of the possible
1211 # lightsensor files existed.
1212 return None
1213
1214
Aviv Keshet74c89a92013-02-04 15:18:30 -08001215 @label_decorator('bluetooth')
Simran Basic6f1f7a2012-10-16 10:47:46 -07001216 def has_bluetooth(self):
1217 """Determine the correct board label for this host.
1218
1219 @returns the string 'bluetooth' if this host has bluetooth or
1220 None if it does not.
1221 """
1222 try:
1223 self.run('test -d /sys/class/bluetooth/hci0')
1224 # test exited with a return code of 0.
1225 return 'bluetooth'
1226 except error.AutoservRunError:
1227 # test exited with a return code 1 meaning the directory did not
1228 # exist.
1229 return None
1230
1231
1232 def get_labels(self):
1233 """Return a list of labels for this given host.
1234
1235 This is the main way to retrieve all the automatic labels for a host
1236 as it will run through all the currently implemented label functions.
1237 """
1238 labels = []
Richard Barnette82c35912012-11-20 10:09:10 -08001239 for label_function in self._LABEL_FUNCTIONS:
Simran Basic6f1f7a2012-10-16 10:47:46 -07001240 label = label_function(self)
1241 if label:
1242 labels.append(label)
1243 return labels