blob: 5bc4ddb93b476054adc74de9568918c215b179d9 [file] [log] [blame]
J. Richard Barnette24adbf42012-04-11 15:04:53 -07001# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
Dale Curtisaa5eedb2011-08-23 16:18:52 -07002# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
Aviv Keshet74c89a92013-02-04 15:18:30 -08005import functools
J. Richard Barnette1d78b012012-05-15 13:56:30 -07006import logging
Dan Shi0f466e82013-02-22 15:44:58 -08007import os
Simran Basid5e5e272012-09-24 15:23:59 -07008import re
Christopher Wileyd78249a2013-03-01 13:05:31 -08009import socket
J. Richard Barnette1d78b012012-05-15 13:56:30 -070010import subprocess
J. Richard Barnette134ec2c2012-04-25 12:59:37 -070011import time
J. Richard Barnette1d78b012012-05-15 13:56:30 -070012import xmlrpclib
J. Richard Barnette134ec2c2012-04-25 12:59:37 -070013
J. Richard Barnette45e93de2012-04-11 17:24:15 -070014from autotest_lib.client.bin import utils
Richard Barnette0c73ffc2012-11-19 15:21:18 -080015from autotest_lib.client.common_lib import error
16from autotest_lib.client.common_lib import global_config
J. Richard Barnette45e93de2012-04-11 17:24:15 -070017from autotest_lib.client.common_lib.cros import autoupdater
Richard Barnette03a0c132012-11-05 12:40:35 -080018from autotest_lib.client.common_lib.cros import dev_server
Christopher Wileyd78249a2013-03-01 13:05:31 -080019from autotest_lib.client.common_lib.cros import retry
Richard Barnette82c35912012-11-20 10:09:10 -080020from autotest_lib.client.cros import constants
J. Richard Barnette45e93de2012-04-11 17:24:15 -070021from autotest_lib.server import autoserv_parser
Chris Sosaf4d43ff2012-10-30 11:21:05 -070022from autotest_lib.server import autotest
J. Richard Barnette45e93de2012-04-11 17:24:15 -070023from autotest_lib.server import site_host_attributes
Scott Zawalski89c44dd2013-02-26 09:28:02 -050024from autotest_lib.server.cros.dynamic_suite import constants as ds_constants
25from autotest_lib.server.cros.dynamic_suite import tools
J. Richard Barnette75487572013-03-08 12:47:50 -080026from autotest_lib.server.cros.servo import servo
J. Richard Barnette45e93de2012-04-11 17:24:15 -070027from autotest_lib.server.hosts import remote
Simran Basidcff4252012-11-20 16:13:20 -080028from autotest_lib.site_utils.rpm_control_system import rpm_client
Simran Basid5e5e272012-09-24 15:23:59 -070029
Richard Barnette82c35912012-11-20 10:09:10 -080030# Importing frontend.afe.models requires a full Autotest
31# installation (with the Django modules), not just the source
32# repository. Most developers won't have the full installation, so
33# the imports below will fail for them.
34#
35# The fix is to catch import exceptions, and set `models` to `None`
36# on failure. This has the side effect that
37# SiteHost._get_board_from_afe() will fail: That will manifest as
38# failures during Repair jobs leaving the DUT as "Repair Failed".
39# In practice, you can't test Repair jobs without a full
40# installation, so that kind of failure isn't expected.
41try:
J. Richard Barnette7214e0b2013-02-06 15:20:49 -080042 # pylint: disable=W0611
Richard Barnette82c35912012-11-20 10:09:10 -080043 from autotest_lib.frontend import setup_django_environment
44 from autotest_lib.frontend.afe import models
45except:
46 models = None
47
Simran Basid5e5e272012-09-24 15:23:59 -070048
J. Richard Barnettebe5ebcc2013-02-11 16:03:15 -080049def _make_servo_hostname(hostname):
50 host_parts = hostname.split('.')
51 host_parts[0] = host_parts[0] + '-servo'
52 return '.'.join(host_parts)
53
54
55def _get_lab_servo(target_hostname):
56 """Instantiate a Servo for |target_hostname| in the lab.
57
58 Assuming that |target_hostname| is a device in the CrOS test
59 lab, create and return a Servo object pointed at the servo
60 attached to that DUT. The servo in the test lab is assumed
61 to already have servod up and running on it.
62
63 @param target_hostname: device whose servo we want to target.
64 @return an appropriately configured Servo instance.
65 """
66 servo_host = _make_servo_hostname(target_hostname)
67 if utils.host_is_in_lab_zone(servo_host):
68 try:
J. Richard Barnetted5f807a2013-02-11 16:51:00 -080069 return servo.Servo(servo_host=servo_host)
J. Richard Barnettebe5ebcc2013-02-11 16:03:15 -080070 except: # pylint: disable=W0702
71 # TODO(jrbarnette): Long-term, if we can't get to
72 # a servo in the lab, we want to fail, so we should
73 # pass any exceptions along. Short-term, we're not
74 # ready to rely on servo, so we ignore failures.
75 pass
76 return None
77
78
Dale Curtiscb7bfaf2011-06-07 16:21:57 -070079def make_ssh_command(user='root', port=22, opts='', hosts_file=None,
80 connect_timeout=None, alive_interval=None):
81 """Override default make_ssh_command to use options tuned for Chrome OS.
82
83 Tuning changes:
Chris Sosaf7fcd6e2011-09-27 17:30:47 -070084 - ConnectTimeout=30; maximum of 30 seconds allowed for an SSH connection
85 failure. Consistency with remote_access.sh.
Dale Curtiscb7bfaf2011-06-07 16:21:57 -070086
Dale Curtisaa5eedb2011-08-23 16:18:52 -070087 - ServerAliveInterval=180; which causes SSH to ping connection every
88 180 seconds. In conjunction with ServerAliveCountMax ensures that if the
89 connection dies, Autotest will bail out quickly. Originally tried 60 secs,
90 but saw frequent job ABORTS where the test completed successfully.
Dale Curtiscb7bfaf2011-06-07 16:21:57 -070091
Chris Sosaf7fcd6e2011-09-27 17:30:47 -070092 - ServerAliveCountMax=3; consistency with remote_access.sh.
93
94 - ConnectAttempts=4; reduce flakiness in connection errors; consistency
95 with remote_access.sh.
Dale Curtiscb7bfaf2011-06-07 16:21:57 -070096
97 - UserKnownHostsFile=/dev/null; we don't care about the keys. Host keys
98 change with every new installation, don't waste memory/space saving them.
Chris Sosaf7fcd6e2011-09-27 17:30:47 -070099
100 - SSH protocol forced to 2; needed for ServerAliveInterval.
J. Richard Barnette7214e0b2013-02-06 15:20:49 -0800101
102 @param user User name to use for the ssh connection.
103 @param port Port on the target host to use for ssh connection.
104 @param opts Additional options to the ssh command.
105 @param hosts_file Ignored.
106 @param connect_timeout Ignored.
107 @param alive_interval Ignored.
Dale Curtiscb7bfaf2011-06-07 16:21:57 -0700108 """
109 base_command = ('/usr/bin/ssh -a -x %s -o StrictHostKeyChecking=no'
110 ' -o UserKnownHostsFile=/dev/null -o BatchMode=yes'
Chris Sosaf7fcd6e2011-09-27 17:30:47 -0700111 ' -o ConnectTimeout=30 -o ServerAliveInterval=180'
112 ' -o ServerAliveCountMax=3 -o ConnectionAttempts=4'
113 ' -o Protocol=2 -l %s -p %d')
Dale Curtiscb7bfaf2011-06-07 16:21:57 -0700114 return base_command % (opts, user, port)
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700115
116
J. Richard Barnette7214e0b2013-02-06 15:20:49 -0800117
Aviv Keshet74c89a92013-02-04 15:18:30 -0800118def add_label_detector(label_function_list, label_list=None, label=None):
119 """Decorator used to group functions together into the provided list.
120 @param label_function_list: List of label detecting functions to add
121 decorated function to.
122 @param label_list: List of detectable labels to add detectable labels to.
123 (Default: None)
124 @param label: Label string that is detectable by this detection function
125 (Default: None)
J. Richard Barnette7214e0b2013-02-06 15:20:49 -0800126 """
Simran Basic6f1f7a2012-10-16 10:47:46 -0700127 def add_func(func):
Aviv Keshet74c89a92013-02-04 15:18:30 -0800128 """
129 @param func: The function to be added as a detector.
130 """
131 label_function_list.append(func)
132 if label and label_list is not None:
133 label_list.append(label)
Simran Basic6f1f7a2012-10-16 10:47:46 -0700134 return func
135 return add_func
136
137
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700138class SiteHost(remote.RemoteHost):
139 """Chromium OS specific subclass of Host."""
140
141 _parser = autoserv_parser.autoserv_parser
142
Richard Barnette0c73ffc2012-11-19 15:21:18 -0800143 # Time to wait for new kernel to be marked successful after
144 # auto update.
Chris Masone163cead2012-05-16 11:49:48 -0700145 _KERNEL_UPDATE_TIMEOUT = 120
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700146
Richard Barnette03a0c132012-11-05 12:40:35 -0800147 # Timeout values (in seconds) associated with various Chrome OS
148 # state changes.
J. Richard Barnette134ec2c2012-04-25 12:59:37 -0700149 #
Richard Barnette0c73ffc2012-11-19 15:21:18 -0800150 # In general, a good rule of thumb is that the timeout can be up
151 # to twice the typical measured value on the slowest platform.
152 # The times here have not necessarily been empirically tested to
153 # meet this criterion.
J. Richard Barnetteeb69d722012-06-18 17:29:44 -0700154 #
155 # SLEEP_TIMEOUT: Time to allow for suspend to memory.
Richard Barnette0c73ffc2012-11-19 15:21:18 -0800156 # RESUME_TIMEOUT: Time to allow for resume after suspend, plus
157 # time to restart the netwowrk.
J. Richard Barnetteeb69d722012-06-18 17:29:44 -0700158 # BOOT_TIMEOUT: Time to allow for boot from power off. Among
Richard Barnette0c73ffc2012-11-19 15:21:18 -0800159 # other things, this must account for the 30 second dev-mode
J. Richard Barnetted4649c62013-03-06 17:42:27 -0800160 # screen delay and time to start the network.
J. Richard Barnetteeb69d722012-06-18 17:29:44 -0700161 # USB_BOOT_TIMEOUT: Time to allow for boot from a USB device,
Richard Barnette0c73ffc2012-11-19 15:21:18 -0800162 # including the 30 second dev-mode delay and time to start the
J. Richard Barnetted4649c62013-03-06 17:42:27 -0800163 # network.
Richard Barnette0c73ffc2012-11-19 15:21:18 -0800164 # SHUTDOWN_TIMEOUT: Time to allow for shut down.
J. Richard Barnetteeb69d722012-06-18 17:29:44 -0700165 # REBOOT_TIMEOUT: Combination of shutdown and reboot times.
Richard Barnette03a0c132012-11-05 12:40:35 -0800166 # _INSTALL_TIMEOUT: Time to allow for chromeos-install.
J. Richard Barnetteeb69d722012-06-18 17:29:44 -0700167
168 SLEEP_TIMEOUT = 2
J. Richard Barnetted4649c62013-03-06 17:42:27 -0800169 RESUME_TIMEOUT = 10
J. Richard Barnetteeb69d722012-06-18 17:29:44 -0700170 BOOT_TIMEOUT = 45
171 USB_BOOT_TIMEOUT = 150
172 SHUTDOWN_TIMEOUT = 5
173 REBOOT_TIMEOUT = SHUTDOWN_TIMEOUT + BOOT_TIMEOUT
Richard Barnette03a0c132012-11-05 12:40:35 -0800174 _INSTALL_TIMEOUT = 240
175
Ismail Noorbasha07fdb612013-02-14 14:13:31 -0800176 # _USB_POWER_TIMEOUT: Time to allow for USB to power toggle ON and OFF.
177 # _POWER_CYCLE_TIMEOUT: Time to allow for manual power cycle.
178 _USB_POWER_TIMEOUT = 5
179 _POWER_CYCLE_TIMEOUT = 10
180
Richard Barnette03a0c132012-11-05 12:40:35 -0800181 _DEFAULT_SERVO_URL_FORMAT = ('/static/servo-images/'
182 '%(board)s_test_image.bin')
183
J. Richard Barnettec14897e2013-03-06 15:56:55 -0800184 # TODO(jrbarnette): Servo repair is restricted to specific
185 # boards, because the existing servo client code doesn't account
186 # for board-specific differences in handling for 'cold_reset'.
187 # http://crosbug.com/36973
188 _SERVO_REPAIR_WHITELIST = ('x86-alex', 'lumpy')
Richard Barnette0c73ffc2012-11-19 15:21:18 -0800189
190
Richard Barnette82c35912012-11-20 10:09:10 -0800191 _RPM_RECOVERY_BOARDS = global_config.global_config.get_config_value('CROS',
192 'rpm_recovery_boards', type=str).split(',')
193
194 _MAX_POWER_CYCLE_ATTEMPTS = 6
195 _LAB_MACHINE_FILE = '/mnt/stateful_partition/.labmachine'
196 _RPM_HOSTNAME_REGEX = ('chromeos[0-9]+(-row[0-9]+)?-rack[0-9]+[a-z]*-'
197 'host[0-9]+')
198 _LIGHTSENSOR_FILES = ['in_illuminance0_input',
199 'in_illuminance0_raw',
200 'illuminance0_input']
201 _LIGHTSENSOR_SEARCH_DIR = '/sys/bus/iio/devices'
202 _LABEL_FUNCTIONS = []
Aviv Keshet74c89a92013-02-04 15:18:30 -0800203 _DETECTABLE_LABELS = []
204 label_decorator = functools.partial(add_label_detector, _LABEL_FUNCTIONS,
205 _DETECTABLE_LABELS)
J. Richard Barnette134ec2c2012-04-25 12:59:37 -0700206
J. Richard Barnetteb6de7e32013-02-14 13:28:04 -0800207 # Constants used in ping_wait_up() and ping_wait_down().
208 #
209 # _PING_WAIT_COUNT is the approximate number of polling
210 # cycles to use when waiting for a host state change.
211 #
212 # _PING_STATUS_DOWN and _PING_STATUS_UP are names used
213 # for arguments to the internal _ping_wait_for_status()
214 # method.
215 _PING_WAIT_COUNT = 40
216 _PING_STATUS_DOWN = False
217 _PING_STATUS_UP = True
218
Ismail Noorbasha07fdb612013-02-14 14:13:31 -0800219 # Allowed values for the power_method argument.
220
221 # POWER_CONTROL_RPM: Passed as default arg for power_off/on/cycle() methods.
222 # POWER_CONTROL_SERVO: Used in set_power() and power_cycle() methods.
223 # POWER_CONTROL_MANUAL: Used in set_power() and power_cycle() methods.
224 POWER_CONTROL_RPM = 'RPM'
225 POWER_CONTROL_SERVO = 'servoj10'
226 POWER_CONTROL_MANUAL = 'manual'
227
228 POWER_CONTROL_VALID_ARGS = (POWER_CONTROL_RPM,
229 POWER_CONTROL_SERVO,
230 POWER_CONTROL_MANUAL)
Richard Barnette0c73ffc2012-11-19 15:21:18 -0800231
J. Richard Barnette964fba02012-10-24 17:34:29 -0700232 @staticmethod
J. Richard Barnette7214e0b2013-02-06 15:20:49 -0800233 def get_servo_arguments(args_dict):
234 """Extract servo options from `args_dict` and return the result.
235
236 Take the provided dictionary of argument options and return
237 a subset that represent standard arguments needed to
238 construct a servo object for a host. The intent is to
239 provide standard argument processing from run_remote_tests
240 for tests that require a servo to operate.
241
242 Recommended usage:
243 ~~~~~~~~
244 args_dict = utils.args_to_dict(args)
245 servo_args = hosts.SiteHost.get_servo_arguments(args_dict)
246 host = hosts.create_host(machine, servo_args=servo_args)
247 ~~~~~~~~
248
249 @param args_dict Dictionary from which to extract the servo
250 arguments.
251 """
J. Richard Barnette964fba02012-10-24 17:34:29 -0700252 servo_args = {}
253 for arg in ('servo_host', 'servo_port'):
J. Richard Barnette7214e0b2013-02-06 15:20:49 -0800254 if arg in args_dict:
255 servo_args[arg] = args_dict[arg]
J. Richard Barnette964fba02012-10-24 17:34:29 -0700256 return servo_args
J. Richard Barnette134ec2c2012-04-25 12:59:37 -0700257
J. Richard Barnette964fba02012-10-24 17:34:29 -0700258
259 def _initialize(self, hostname, servo_args=None, *args, **dargs):
J. Richard Barnette67ccb872012-04-19 16:34:56 -0700260 """Initialize superclasses, and |self.servo|.
261
262 For creating the host servo object, there are three
263 possibilities: First, if the host is a lab system known to
264 have a servo board, we connect to that servo unconditionally.
265 Second, if we're called from a control file that requires
J. Richard Barnette55fb8062012-05-23 10:29:31 -0700266 servo features for testing, it will pass settings for
267 `servo_host`, `servo_port`, or both. If neither of these
268 cases apply, `self.servo` will be `None`.
J. Richard Barnette67ccb872012-04-19 16:34:56 -0700269
270 """
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700271 super(SiteHost, self)._initialize(hostname=hostname,
272 *args, **dargs)
J. Richard Barnettef0859852012-08-20 14:55:50 -0700273 # self.env is a dictionary of environment variable settings
274 # to be exported for commands run on the host.
275 # LIBC_FATAL_STDERR_ can be useful for diagnosing certain
276 # errors that might happen.
277 self.env['LIBC_FATAL_STDERR_'] = '1'
J. Richard Barnette1d78b012012-05-15 13:56:30 -0700278 self._xmlrpc_proxy_map = {}
J. Richard Barnettebe5ebcc2013-02-11 16:03:15 -0800279 self.servo = _get_lab_servo(hostname)
J. Richard Barnettead7da482012-10-30 16:46:52 -0700280 if not self.servo and servo_args is not None:
J. Richard Barnette964fba02012-10-24 17:34:29 -0700281 self.servo = servo.Servo(**servo_args)
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700282
283
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500284 def get_repair_image_name(self):
285 """Generate a image_name from variables in the global config.
286
287 @returns a str of $board-version/$BUILD.
288
289 """
290 stable_version = global_config.global_config.get_config_value(
291 'CROS', 'stable_cros_version')
292 build_pattern = global_config.global_config.get_config_value(
293 'CROS', 'stable_build_pattern')
294 board = self._get_board_from_afe()
295 if board is None:
296 raise error.AutoservError('DUT has no board attribute, '
297 'cannot be repaired.')
298 return build_pattern % (board, stable_version)
299
300
301 def clear_cros_version_labels_and_job_repo_url(self):
302 """Clear cros_version labels and host attribute job_repo_url."""
Scott Zawalskieadbf702013-03-14 09:23:06 -0400303 try:
304 host_model = models.Host.objects.get(hostname=self.hostname)
305 except models.Host.DoesNotExist:
306 return
307
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500308 for label in host_model.labels.iterator():
309 if not label.name.startswith(ds_constants.VERSION_PREFIX):
310 continue
Dan Shi0f466e82013-02-22 15:44:58 -0800311
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500312 label.host_set.remove(host_model)
313
314 host_model.set_or_delete_attribute('job_repo_url', None)
315
316
Scott Zawalskieadbf702013-03-14 09:23:06 -0400317 def add_cros_version_labels_and_job_repo_url(self, image_name):
318 """Add cros_version labels and host attribute job_repo_url.
319
320 @param image_name: The name of the image e.g.
321 lumpy-release/R27-3837.0.0
322 """
323 try:
324 host_model = models.Host.objects.get(hostname=self.hostname)
325 except models.Host.DoesNotExist:
326 return
327 cros_label = '%s%s' % (ds_constants.VERSION_PREFIX, image_name)
328 devserver_url = dev_server.ImageServer.resolve(image_name).url()
329 try:
330 label_model = models.Label.objects.get(name=cros_label)
331 except models.Label.DoesNotExist:
332 label_model = models.Label.objects.create(name=cros_label)
333 host_model.labels.add(label_model)
334 repo_url = tools.get_package_url(devserver_url, image_name)
335 host_model.set_or_delete_attribute('job_repo_url', repo_url)
336
337
Dan Shi0f466e82013-02-22 15:44:58 -0800338 def _try_stateful_update(self, update_url, force_update, updater):
339 """Try to use stateful update to initialize DUT.
340
341 When DUT is already running the same version that machine_install
342 tries to install, stateful update is a much faster way to clean up
343 the DUT for testing, compared to a full reimage. It is implemeted
344 by calling autoupdater.run_update, but skipping updating root, as
345 updating the kernel is time consuming and not necessary.
346
347 @param update_url: url of the image.
348 @param force_update: Set to True to update the image even if the DUT
349 is running the same version.
350 @param updater: ChromiumOSUpdater instance used to update the DUT.
351 @returns: True if the DUT was updated with stateful update.
352
353 """
Dan Shi7b7379d2013-03-19 16:26:33 -0700354 # Stateful update is disabled until lsb-release has rc build info.
355 logging.info('Stateful update only is disabled.')
356 return False
Dan Shi0f466e82013-02-22 15:44:58 -0800357 if not updater.check_version():
358 return False
359 if not force_update:
360 logging.info('Canceling stateful update because the new and '
361 'old versions are the same.')
362 return False
363 # Following folders should be rebuilt after stateful update.
364 # A test file is used to confirm each folder gets rebuilt after
365 # the stateful update.
366 folders_to_check = ['/var', '/home', '/mnt/stateful_partition']
367 test_file = '.test_file_to_be_deleted'
368 for folder in folders_to_check:
369 touch_path = os.path.join(folder, test_file)
370 self.run('touch %s' % touch_path)
371
372 if not updater.run_update(force_update=True, update_root=False):
373 return False
374
375 # Reboot to complete stateful update.
376 self.reboot(timeout=60, wait=True)
377 check_file_cmd = 'test -f %s; echo $?'
378 for folder in folders_to_check:
379 test_file_path = os.path.join(folder, test_file)
380 result = self.run(check_file_cmd % test_file_path,
381 ignore_status=True)
382 if result.exit_status == 1:
383 return False
384 return True
385
386
387 def _post_update_processing(self, updater, inactive_kernel=None):
388 """After the DUT is updated, confirm machine_install succeeded.
389
390 @param updater: ChromiumOSUpdater instance used to update the DUT.
391 @param inactive_kernel: kernel state of inactive kernel before reboot.
392
393 """
394
395 # Touch the lab machine file to leave a marker that distinguishes
396 # this image from other test images.
397 self.run('touch %s' % self._LAB_MACHINE_FILE)
398
399 # Kick off the autoreboot script as the _LAB_MACHINE_FILE was
400 # missing on the first boot.
401 self.run('start autoreboot')
402
403 # Following the reboot, verify the correct version.
404 if not updater.check_version():
405 # Print out crossystem to make it easier to debug the rollback.
406 logging.debug('Dumping partition table.')
407 self.host.run('cgpt show $(rootdev -s -d)')
408 logging.debug('Dumping crossystem for firmware debugging.')
409 self.host.run('crossystem --all')
410 logging.error('Expected Chromium OS version: %s. '
411 'Found Chromium OS %s',
412 self.update_version, updater.get_build_id())
413 raise ChromiumOSError('Updater failed on host %s' %
414 self.host.hostname)
415
416 # Figure out newly active kernel.
417 new_active_kernel, _ = updater.get_kernel_state()
418
419 # Ensure that previously inactive kernel is now the active kernel.
420 if inactive_kernel and new_active_kernel != inactive_kernel:
421 raise autoupdater.ChromiumOSError(
422 'Update failed. New kernel partition is not active after'
423 ' boot.')
424
Scott Zawalskieadbf702013-03-14 09:23:06 -0400425 try:
426 host_attributes = site_host_attributes.HostAttributes(self.hostname)
427 except models.Host.DoesNotExist:
428 host_attributes = None
429 if host_attributes and host_attributes.has_chromeos_firmware:
Dan Shi0f466e82013-02-22 15:44:58 -0800430 # Wait until tries == 0 and success, or until timeout.
431 utils.poll_for_condition(
432 lambda: (updater.get_kernel_tries(new_active_kernel) == 0
433 and updater.get_kernel_success(new_active_kernel)),
434 exception=autoupdater.ChromiumOSError(
435 'Update failed. Timed out waiting for system to mark'
436 ' new kernel as successful.'),
437 timeout=self._KERNEL_UPDATE_TIMEOUT, sleep_interval=5)
438
439
Scott Zawalskieadbf702013-03-14 09:23:06 -0400440 def _stage_build_and_return_update_url(self, image_name):
441 """Stage a build on a devserver and return the update_url.
442
443 @param image_name: a name like lumpy-release/R27-3837.0.0
444 @returns an update URL like:
445 http://172.22.50.205:8082/update/lumpy-release/R27-3837.0.0
446 """
447 logging.info('Staging requested build: %s', image_name)
448 devserver = dev_server.ImageServer.resolve(image_name)
449 devserver.trigger_download(image_name, synchronous=False)
450 return tools.image_url_pattern() % (devserver.url(), image_name)
451
452
Chris Sosaa3ac2152012-05-23 22:23:13 -0700453 def machine_install(self, update_url=None, force_update=False,
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500454 local_devserver=False, repair=False):
455 """Install the DUT.
456
Dan Shi0f466e82013-02-22 15:44:58 -0800457 Use stateful update if the DUT is already running the same build.
458 Stateful update does not update kernel and tends to run much faster
459 than a full reimage. If the DUT is running a different build, or it
460 failed to do a stateful update, full update, including kernel update,
461 will be applied to the DUT.
462
Scott Zawalskieadbf702013-03-14 09:23:06 -0400463 Once a host enters machine_install its cros_version label will be
464 removed as well as its host attribute job_repo_url (used for
465 package install).
466
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500467 @param update_url: The url to use for the update
468 pattern: http://$devserver:###/update/$build
469 If update_url is None and repair is True we will install the
470 stable image listed in global_config under
471 CROS.stable_cros_version.
472 @param force_update: Force an update even if the version installed
473 is the same. Default:False
474 @param local_devserver: Used by run_remote_test to allow people to
475 use their local devserver. Default: False
476 @param repair: Whether or not we are in repair mode. This adds special
477 cases for repairing a machine like starting update_engine.
478 Setting repair to True sets force_update to True as well.
479 default: False
480 @raises autoupdater.ChromiumOSError
481
482 """
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700483 if not update_url and self._parser.options.image:
Scott Zawalskieadbf702013-03-14 09:23:06 -0400484 requested_build = self._parser.options.image
485 if requested_build.startswith('http://'):
486 update_url = requested_build
487 else:
488 # Try to stage any build that does not start with http:// on
489 # the devservers defined in global_config.ini.
490 update_url = self._stage_build_and_return_update_url(
491 requested_build)
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500492 elif not update_url and not repair:
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700493 raise autoupdater.ChromiumOSError(
494 'Update failed. No update URL provided.')
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500495 elif not update_url and repair:
Scott Zawalskieadbf702013-03-14 09:23:06 -0400496 update_url = self._stage_build_and_return_update_url(
497 self.get_repair_image_name())
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500498
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500499 if repair:
Dan Shi0f466e82013-02-22 15:44:58 -0800500 # In case the system is in a bad state, we always reboot the machine
501 # before machine_install.
502 self.reboot(timeout=60, wait=True)
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500503 self.run('stop update-engine; start update-engine')
504 force_update = True
Dan Shi0f466e82013-02-22 15:44:58 -0800505
Chris Sosaa3ac2152012-05-23 22:23:13 -0700506 updater = autoupdater.ChromiumOSUpdater(update_url, host=self,
Dan Shi0f466e82013-02-22 15:44:58 -0800507 local_devserver=local_devserver)
508 updated = False
Scott Zawalskieadbf702013-03-14 09:23:06 -0400509 # Remove cros-version and job_repo_url host attribute from host.
510 self.clear_cros_version_labels_and_job_repo_url()
Dan Shi0f466e82013-02-22 15:44:58 -0800511 # If the DUT is already running the same build, try stateful update
512 # first. Stateful update does not update kernel and tends to run much
513 # faster than a full reimage.
514 try:
515 updated = self._try_stateful_update(update_url, force_update,
516 updater)
517 if updated:
518 logging.info('DUT is updated with stateful update.')
519 except Exception as e:
520 logging.exception(e)
521 logging.warn('Failed to stateful update DUT, force to update.')
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700522
Dan Shi0f466e82013-02-22 15:44:58 -0800523 inactive_kernel = None
524 # Do a full update if stateful update is not applicable or failed.
525 if not updated:
526 # In case the system is in a bad state, we always reboot the
527 # machine before machine_install.
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700528 self.reboot(timeout=60, wait=True)
Dan Shi0f466e82013-02-22 15:44:58 -0800529 if updater.run_update(force_update):
530 updated = True
531 # Figure out active and inactive kernel.
532 active_kernel, inactive_kernel = updater.get_kernel_state()
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700533
Dan Shi0f466e82013-02-22 15:44:58 -0800534 # Ensure inactive kernel has higher priority than active.
535 if (updater.get_kernel_priority(inactive_kernel)
536 < updater.get_kernel_priority(active_kernel)):
537 raise autoupdater.ChromiumOSError(
538 'Update failed. The priority of the inactive kernel'
539 ' partition is less than that of the active kernel'
540 ' partition.')
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700541
Dan Shi0f466e82013-02-22 15:44:58 -0800542 update_engine_log = '/var/log/update_engine.log'
543 logging.info('Dumping %s', update_engine_log)
544 self.run('cat %s' % update_engine_log)
545 # Updater has returned successfully; reboot the host.
546 self.reboot(timeout=60, wait=True)
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700547
Dan Shi0f466e82013-02-22 15:44:58 -0800548 if updated:
549 self._post_update_processing(updater, inactive_kernel)
Scott Zawalskieadbf702013-03-14 09:23:06 -0400550 image_name = autoupdater.url_to_image_name(update_url)
551 self.add_cros_version_labels_and_job_repo_url(image_name)
Simran Basi13fa1ba2013-03-04 10:56:47 -0800552
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700553 # Clean up any old autotest directories which may be lying around.
554 for path in global_config.global_config.get_config_value(
555 'AUTOSERV', 'client_autodir_paths', type=list):
556 self.run('rm -rf ' + path)
557
558
Simran Basi833814b2013-01-29 13:13:43 -0800559 def _get_label_from_afe(self, label_prefix):
560 """Retrieve a host's specific label from the AFE.
561
562 Looks for a host label that has the form <label_prefix>:<value>
563 and returns the "<value>" part of the label. None is returned
564 if there is not a label matching the pattern
565
566 @returns the label that matches the prefix or 'None'
567 """
568 host_model = models.Host.objects.get(hostname=self.hostname)
569 host_label = host_model.labels.get(name__startswith=label_prefix)
570 if not host_label:
571 return None
572 return host_label.name.split(label_prefix, 1)[1]
573
574
Richard Barnette82c35912012-11-20 10:09:10 -0800575 def _get_board_from_afe(self):
576 """Retrieve this host's board from its labels in the AFE.
577
578 Looks for a host label of the form "board:<board>", and
579 returns the "<board>" part of the label. `None` is returned
580 if there is not a single, unique label matching the pattern.
581
582 @returns board from label, or `None`.
583 """
Simran Basi833814b2013-01-29 13:13:43 -0800584 return self._get_label_from_afe(ds_constants.BOARD_PREFIX)
585
586
587 def get_build(self):
588 """Retrieve the current build for this Host from the AFE.
589
590 Looks through this host's labels in the AFE to determine its build.
591
592 @returns The current build or None if it could not find it or if there
593 were multiple build labels assigned to this host.
594 """
595 return self._get_label_from_afe(ds_constants.VERSION_PREFIX)
Richard Barnette82c35912012-11-20 10:09:10 -0800596
597
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500598 def _install_repair(self):
599 """Attempt to repair this host using upate-engine.
600
601 If the host is up, try installing the DUT with a stable
602 "repair" version of Chrome OS as defined in the global_config
603 under CROS.stable_cros_version.
604
605 @returns True if successful, False if update_engine failed.
606
607 """
608 if not self.is_up():
609 return False
610
611 logging.info('Attempting to reimage machine to repair image.')
612 try:
613 self.machine_install(repair=True)
Fang Dengd0672f32013-03-18 17:18:09 -0700614 except autoupdater.ChromiumOSError as e:
615 logging.exception(e)
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500616 logging.info('Repair via install failed.')
617 return False
618
619 return True
620
621
Richard Barnette03a0c132012-11-05 12:40:35 -0800622 def _servo_repair(self, board):
623 """Attempt to repair this host using an attached Servo.
624
625 Re-install the OS on the DUT by 1) installing a test image
626 on a USB storage device attached to the Servo board,
627 2) booting that image in recovery mode, and then
628 3) installing the image.
629
630 """
631 server = dev_server.ImageServer.devserver_url_for_servo(board)
632 image = server + (self._DEFAULT_SERVO_URL_FORMAT %
633 { 'board': board })
634 self.servo.install_recovery_image(image)
635 if not self.wait_up(timeout=self.USB_BOOT_TIMEOUT):
636 raise error.AutoservError('DUT failed to boot from USB'
637 ' after %d seconds' %
638 self.USB_BOOT_TIMEOUT)
639 self.run('chromeos-install --yes',
640 timeout=self._INSTALL_TIMEOUT)
641 self.servo.power_long_press()
642 self.servo.set('usb_mux_sel1', 'servo_sees_usbkey')
643 self.servo.power_short_press()
644 if not self.wait_up(timeout=self.BOOT_TIMEOUT):
645 raise error.AutoservError('DUT failed to reboot installed '
646 'test image after %d seconds' %
647 self.BOOT_TIMEOUT)
648
649
Richard Barnette82c35912012-11-20 10:09:10 -0800650 def _powercycle_to_repair(self):
651 """Utilize the RPM Infrastructure to bring the host back up.
652
653 If the host is not up/repaired after the first powercycle we utilize
654 auto fallback to the last good install by powercycling and rebooting the
655 host 6 times.
656 """
657 logging.info('Attempting repair via RPM powercycle.')
658 failed_cycles = 0
659 self.power_cycle()
660 while not self.wait_up(timeout=self.BOOT_TIMEOUT):
661 failed_cycles += 1
662 if failed_cycles >= self._MAX_POWER_CYCLE_ATTEMPTS:
663 raise error.AutoservError('Powercycled host %s %d times; '
664 'device did not come back online.' %
665 (self.hostname, failed_cycles))
666 self.power_cycle()
667 if failed_cycles == 0:
668 logging.info('Powercycling was successful first time.')
669 else:
670 logging.info('Powercycling was successful after %d failures.',
671 failed_cycles)
672
673
674 def repair_full(self):
675 """Repair a host for repair level NO_PROTECTION.
676
677 This overrides the base class function for repair; it does
678 not call back to the parent class, but instead offers a
679 simplified implementation based on the capabilities in the
680 Chrome OS test lab.
681
J. Richard Barnettefde55fc2013-03-15 17:47:01 -0700682 If `self.verify()` fails, the following procedures are
683 attempted:
684 1. Try to re-install to a known stable image using
685 auto-update.
686 2. If there's a servo for the DUT, try to re-install via
687 the servo.
688 3. If the DUT can be power-cycled via RPM, try to repair
Richard Barnette82c35912012-11-20 10:09:10 -0800689 by power-cycling.
690
691 As with the parent method, the last operation performed on
692 the DUT must be to call `self.verify()`; if that call fails,
693 the exception it raises is passed back to the caller.
J. Richard Barnettefde55fc2013-03-15 17:47:01 -0700694
Richard Barnette82c35912012-11-20 10:09:10 -0800695 """
696 try:
697 self.verify()
698 except:
699 host_board = self._get_board_from_afe()
Richard Barnette03a0c132012-11-05 12:40:35 -0800700 if host_board is None:
701 logging.error('host %s has no board; failing repair',
702 self.hostname)
Richard Barnette82c35912012-11-20 10:09:10 -0800703 raise
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500704
J. Richard Barnettefde55fc2013-03-15 17:47:01 -0700705 if not self._install_repair():
706 # TODO(scottz): All repair pathways should be
707 # executed until we've exhausted all options. Below
708 # we favor servo over powercycle when we really
709 # should be falling back to power if servo fails.
710 if (self.servo and
711 host_board in self._SERVO_REPAIR_WHITELIST):
712 self._servo_repair(host_board)
713 elif (self.has_power() and
714 host_board in self._RPM_RECOVERY_BOARDS):
715 self._powercycle_to_repair()
716 else:
717 logging.error('host %s has no servo and no RPM control; '
718 'failing repair', self.hostname)
719 raise
Richard Barnette82c35912012-11-20 10:09:10 -0800720 self.verify()
721
722
J. Richard Barnette1d78b012012-05-15 13:56:30 -0700723 def close(self):
724 super(SiteHost, self).close()
725 self.xmlrpc_disconnect_all()
726
727
Chris Sosaf4d43ff2012-10-30 11:21:05 -0700728 def cleanup(self):
Chris Sosaf4d43ff2012-10-30 11:21:05 -0700729 client_at = autotest.Autotest(self)
Richard Barnette82c35912012-11-20 10:09:10 -0800730 self.run('rm -f %s' % constants.CLEANUP_LOGS_PAUSED_FILE)
Scott Zawalskiddbc31e2012-11-15 11:29:01 -0500731 try:
732 client_at.run_static_method('autotest_lib.client.cros.cros_ui',
733 '_clear_login_prompt_state')
734 self.run('restart ui')
735 client_at.run_static_method('autotest_lib.client.cros.cros_ui',
736 '_wait_for_login_prompt')
Alex Millerf4517962013-02-25 15:03:02 -0800737 except (error.AutotestRunError, error.AutoservRunError):
Scott Zawalskiddbc31e2012-11-15 11:29:01 -0500738 logging.warn('Unable to restart ui, rebooting device.')
739 # Since restarting the UI fails fall back to normal Autotest
740 # cleanup routines, i.e. reboot the machine.
741 super(SiteHost, self).cleanup()
Chris Sosaf4d43ff2012-10-30 11:21:05 -0700742
743
Simran Basi154f5582012-10-23 16:27:11 -0700744 # TODO (sbasi) crosbug.com/35656
745 # Renamed the sitehost cleanup method so we don't go down this pathway.
746 # def cleanup(self):
747 def cleanup_poweron(self):
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700748 """Special cleanup method to make sure hosts always get power back."""
Chris Sosa9479fcd2012-10-09 13:44:22 -0700749 super(SiteHost, self).cleanup()
Simran Basid5e5e272012-09-24 15:23:59 -0700750 if self.has_power():
Simran Basifd23fb22012-10-22 17:56:22 -0700751 try:
752 self.power_on()
Chris Sosafab08082013-01-04 15:21:20 -0800753 except rpm_client.RemotePowerException:
Simran Basifd23fb22012-10-22 17:56:22 -0700754 # If cleanup has completed but there was an issue with the RPM
755 # Infrastructure, log an error message rather than fail cleanup
756 logging.error('Failed to turn Power On for this host after '
757 'cleanup through the RPM Infrastructure.')
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700758
759
Yu-Ju Honga2be94a2012-07-31 09:48:52 -0700760 def reboot(self, **dargs):
761 """
762 This function reboots the site host. The more generic
763 RemoteHost.reboot() performs sync and sleeps for 5
764 seconds. This is not necessary for Chrome OS devices as the
765 sync should be finished in a short time during the reboot
766 command.
767 """
Tom Wai-Hong Tamf5cd1d42012-08-13 12:04:08 +0800768 if 'reboot_cmd' not in dargs:
769 dargs['reboot_cmd'] = ('((reboot & sleep 10; reboot -f &)'
770 ' </dev/null >/dev/null 2>&1 &)')
Yu-Ju Honga2be94a2012-07-31 09:48:52 -0700771 # Enable fastsync to avoid running extra sync commands before reboot.
Tom Wai-Hong Tamf5cd1d42012-08-13 12:04:08 +0800772 if 'fastsync' not in dargs:
773 dargs['fastsync'] = True
Yu-Ju Honga2be94a2012-07-31 09:48:52 -0700774 super(SiteHost, self).reboot(**dargs)
775
776
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700777 def verify_software(self):
Richard Barnetteb2bc13c2013-01-08 17:32:51 -0800778 """Verify working software on a Chrome OS system.
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700779
Richard Barnetteb2bc13c2013-01-08 17:32:51 -0800780 Tests for the following conditions:
781 1. All conditions tested by the parent version of this
782 function.
783 2. Sufficient space in /mnt/stateful_partition.
784 3. update_engine answers a simple status request over DBus.
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700785
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700786 """
787 super(SiteHost, self).verify_software()
788 self.check_diskspace(
789 '/mnt/stateful_partition',
790 global_config.global_config.get_config_value(
791 'SERVER', 'gb_diskspace_required', type=int,
792 default=20))
Richard Barnetteb2bc13c2013-01-08 17:32:51 -0800793 self.run('update_engine_client --status')
Scott Zawalskifbca4a92013-03-04 15:56:42 -0500794 # Makes sure python is present, loads and can use built in functions.
795 # We have seen cases where importing cPickle fails with undefined
796 # symbols in cPickle.so.
797 self.run('python -c "import cPickle"')
J. Richard Barnette134ec2c2012-04-25 12:59:37 -0700798
799
Christopher Wileyd78249a2013-03-01 13:05:31 -0800800 def xmlrpc_connect(self, command, port, command_name=None,
801 ready_test_name=None, timeout_seconds=10):
J. Richard Barnette1d78b012012-05-15 13:56:30 -0700802 """Connect to an XMLRPC server on the host.
803
804 The `command` argument should be a simple shell command that
805 starts an XMLRPC server on the given `port`. The command
806 must not daemonize, and must terminate cleanly on SIGTERM.
807 The command is started in the background on the host, and a
808 local XMLRPC client for the server is created and returned
809 to the caller.
810
811 Note that the process of creating an XMLRPC client makes no
812 attempt to connect to the remote server; the caller is
813 responsible for determining whether the server is running
814 correctly, and is ready to serve requests.
815
Christopher Wileyd78249a2013-03-01 13:05:31 -0800816 Optionally, the caller can pass ready_test_name, a string
817 containing the name of a method to call on the proxy. This
818 method should take no parameters and return successfully only
819 when the server is ready to process client requests. When
820 ready_test_name is set, xmlrpc_connect will block until the
821 proxy is ready, and throw a TestError if the server isn't
822 ready by timeout_seconds.
823
J. Richard Barnette1d78b012012-05-15 13:56:30 -0700824 @param command Shell command to start the server.
825 @param port Port number on which the server is expected to
826 be serving.
J. Richard Barnette7214e0b2013-02-06 15:20:49 -0800827 @param command_name String to use as input to `pkill` to
828 terminate the XMLRPC server on the host.
Christopher Wileyd78249a2013-03-01 13:05:31 -0800829 @param ready_test_name String containing the name of a
830 method defined on the XMLRPC server.
831 @param timeout_seconds Number of seconds to wait
832 for the server to become 'ready.' Will throw a
833 TestFail error if server is not ready in time.
834
J. Richard Barnette1d78b012012-05-15 13:56:30 -0700835 """
836 self.xmlrpc_disconnect(port)
837
838 # Chrome OS on the target closes down most external ports
839 # for security. We could open the port, but doing that
840 # would conflict with security tests that check that only
841 # expected ports are open. So, to get to the port on the
842 # target we use an ssh tunnel.
843 local_port = utils.get_unused_port()
844 tunnel_options = '-n -N -q -L %d:localhost:%d' % (local_port, port)
845 ssh_cmd = make_ssh_command(opts=tunnel_options)
846 tunnel_cmd = '%s %s' % (ssh_cmd, self.hostname)
847 logging.debug('Full tunnel command: %s', tunnel_cmd)
848 tunnel_proc = subprocess.Popen(tunnel_cmd, shell=True, close_fds=True)
849 logging.debug('Started XMLRPC tunnel, local = %d'
850 ' remote = %d, pid = %d',
851 local_port, port, tunnel_proc.pid)
852
853 # Start the server on the host. Redirection in the command
854 # below is necessary, because 'ssh' won't terminate until
855 # background child processes close stdin, stdout, and
856 # stderr.
857 remote_cmd = '( %s ) </dev/null >/dev/null 2>&1 & echo $!' % command
858 remote_pid = self.run(remote_cmd).stdout.rstrip('\n')
859 logging.debug('Started XMLRPC server on host %s, pid = %s',
860 self.hostname, remote_pid)
861
J. Richard Barnette7214e0b2013-02-06 15:20:49 -0800862 self._xmlrpc_proxy_map[port] = (command_name, tunnel_proc)
J. Richard Barnette1d78b012012-05-15 13:56:30 -0700863 rpc_url = 'http://localhost:%d' % local_port
Christopher Wileyd78249a2013-03-01 13:05:31 -0800864 proxy = xmlrpclib.ServerProxy(rpc_url, allow_none=True)
865 if ready_test_name is not None:
J. Richard Barnette13eb7c02013-03-07 12:06:29 -0800866 # retry.retry logs each attempt; calculate delay_sec to
867 # keep log spam to a dull roar.
Christopher Wileyd78249a2013-03-01 13:05:31 -0800868 @retry.retry((socket.error, xmlrpclib.ProtocolError),
869 timeout_min=timeout_seconds/60.0,
J. Richard Barnette13eb7c02013-03-07 12:06:29 -0800870 delay_sec=min(max(timeout_seconds/20.0, 0.1), 1))
Christopher Wileyd78249a2013-03-01 13:05:31 -0800871 def ready_test():
872 """ Call proxy.ready_test_name(). """
873 getattr(proxy, ready_test_name)()
874 successful = False
875 try:
876 logging.info('Waiting %d seconds for XMLRPC server '
877 'to start.', timeout_seconds)
878 ready_test()
879 successful = True
880 except retry.TimeoutException:
881 raise error.TestError('Unable to start XMLRPC server after '
882 '%d seconds.' % timeout_seconds)
883 finally:
884 if not successful:
885 logging.error('Failed to start XMLRPC server.')
886 self.xmlrpc_disconnect(port)
887 logging.info('XMLRPC server started successfully.')
888 return proxy
J. Richard Barnette1d78b012012-05-15 13:56:30 -0700889
890 def xmlrpc_disconnect(self, port):
891 """Disconnect from an XMLRPC server on the host.
892
893 Terminates the remote XMLRPC server previously started for
894 the given `port`. Also closes the local ssh tunnel created
895 for the connection to the host. This function does not
896 directly alter the state of a previously returned XMLRPC
897 client object; however disconnection will cause all
898 subsequent calls to methods on the object to fail.
899
900 This function does nothing if requested to disconnect a port
901 that was not previously connected via `self.xmlrpc_connect()`
902
903 @param port Port number passed to a previous call to
904 `xmlrpc_connect()`
905 """
906 if port not in self._xmlrpc_proxy_map:
907 return
908 entry = self._xmlrpc_proxy_map[port]
909 remote_name = entry[0]
910 tunnel_proc = entry[1]
911 if remote_name:
912 # We use 'pkill' to find our target process rather than
913 # a PID, because the host may have rebooted since
914 # connecting, and we don't want to kill an innocent
915 # process with the same PID.
916 #
917 # 'pkill' helpfully exits with status 1 if no target
918 # process is found, for which run() will throw an
Simran Basid5e5e272012-09-24 15:23:59 -0700919 # exception. We don't want that, so we the ignore
J. Richard Barnette1d78b012012-05-15 13:56:30 -0700920 # status.
921 self.run("pkill -f '%s'" % remote_name, ignore_status=True)
922
923 if tunnel_proc.poll() is None:
924 tunnel_proc.terminate()
925 logging.debug('Terminated tunnel, pid %d', tunnel_proc.pid)
926 else:
927 logging.debug('Tunnel pid %d terminated early, status %d',
928 tunnel_proc.pid, tunnel_proc.returncode)
929 del self._xmlrpc_proxy_map[port]
930
931
932 def xmlrpc_disconnect_all(self):
933 """Disconnect all known XMLRPC proxy ports."""
934 for port in self._xmlrpc_proxy_map.keys():
935 self.xmlrpc_disconnect(port)
936
937
J. Richard Barnetteb6de7e32013-02-14 13:28:04 -0800938 def _ping_check_status(self, status):
939 """Ping the host once, and return whether it has a given status.
J. Richard Barnette134ec2c2012-04-25 12:59:37 -0700940
J. Richard Barnetteb6de7e32013-02-14 13:28:04 -0800941 @param status Check the ping status against this value.
942 @return True iff `status` and the result of ping are the same
943 (i.e. both True or both False).
944
945 """
946 ping_val = utils.ping(self.hostname, tries=1, deadline=1)
947 return not (status ^ (ping_val == 0))
948
949 def _ping_wait_for_status(self, status, timeout):
950 """Wait for the host to have a given status (UP or DOWN).
951
952 Status is checked by polling. Polling will not last longer
953 than the number of seconds in `timeout`. The polling
954 interval will be long enough that only approximately
955 _PING_WAIT_COUNT polling cycles will be executed, subject
956 to a maximum interval of about one minute.
957
958 @param status Waiting will stop immediately if `ping` of the
959 host returns this status.
960 @param timeout Poll for at most this many seconds.
961 @return True iff the host status from `ping` matched the
962 requested status at the time of return.
963
964 """
965 # _ping_check_status() takes about 1 second, hence the
966 # "- 1" in the formula below.
967 poll_interval = min(int(timeout / self._PING_WAIT_COUNT), 60) - 1
968 end_time = time.time() + timeout
969 while time.time() <= end_time:
970 if self._ping_check_status(status):
971 return True
972 if poll_interval > 0:
973 time.sleep(poll_interval)
974
975 # The last thing we did was sleep(poll_interval), so it may
976 # have been too long since the last `ping`. Check one more
977 # time, just to be sure.
978 return self._ping_check_status(status)
979
980 def ping_wait_up(self, timeout):
981 """Wait for the host to respond to `ping`.
982
983 N.B. This method is not a reliable substitute for
984 `wait_up()`, because a host that responds to ping will not
985 necessarily respond to ssh. This method should only be used
986 if the target DUT can be considered functional even if it
987 can't be reached via ssh.
988
989 @param timeout Minimum time to allow before declaring the
990 host to be non-responsive.
991 @return True iff the host answered to ping before the timeout.
992
993 """
994 return self._ping_wait_for_status(self._PING_STATUS_UP, timeout)
J. Richard Barnette134ec2c2012-04-25 12:59:37 -0700995
Andrew Bresticker678c0c72013-01-22 10:44:09 -0800996 def ping_wait_down(self, timeout):
J. Richard Barnette134ec2c2012-04-25 12:59:37 -0700997 """Wait until the host no longer responds to `ping`.
998
J. Richard Barnetteb6de7e32013-02-14 13:28:04 -0800999 This function can be used as a slightly faster version of
1000 `wait_down()`, by avoiding potentially long ssh timeouts.
1001
1002 @param timeout Minimum time to allow for the host to become
1003 non-responsive.
1004 @return True iff the host quit answering ping before the
1005 timeout.
1006
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001007 """
J. Richard Barnetteb6de7e32013-02-14 13:28:04 -08001008 return self._ping_wait_for_status(self._PING_STATUS_DOWN, timeout)
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001009
1010 def test_wait_for_sleep(self):
1011 """Wait for the client to enter low-power sleep mode.
1012
1013 The test for "is asleep" can't distinguish a system that is
1014 powered off; to confirm that the unit was asleep, it is
1015 necessary to force resume, and then call
1016 `test_wait_for_resume()`.
1017
1018 This function is expected to be called from a test as part
1019 of a sequence like the following:
1020
1021 ~~~~~~~~
1022 boot_id = host.get_boot_id()
1023 # trigger sleep on the host
1024 host.test_wait_for_sleep()
1025 # trigger resume on the host
1026 host.test_wait_for_resume(boot_id)
1027 ~~~~~~~~
1028
1029 @exception TestFail The host did not go to sleep within
1030 the allowed time.
1031 """
Andrew Bresticker678c0c72013-01-22 10:44:09 -08001032 if not self.ping_wait_down(timeout=self.SLEEP_TIMEOUT):
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001033 raise error.TestFail(
1034 'client failed to sleep after %d seconds' %
J. Richard Barnetteeb69d722012-06-18 17:29:44 -07001035 self.SLEEP_TIMEOUT)
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001036
1037
1038 def test_wait_for_resume(self, old_boot_id):
1039 """Wait for the client to resume from low-power sleep mode.
1040
1041 The `old_boot_id` parameter should be the value from
1042 `get_boot_id()` obtained prior to entering sleep mode. A
1043 `TestFail` exception is raised if the boot id changes.
1044
1045 See @ref test_wait_for_sleep for more on this function's
1046 usage.
1047
J. Richard Barnette7214e0b2013-02-06 15:20:49 -08001048 @param old_boot_id A boot id value obtained before the
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001049 target host went to sleep.
1050
1051 @exception TestFail The host did not respond within the
1052 allowed time.
1053 @exception TestFail The host responded, but the boot id test
1054 indicated a reboot rather than a sleep
1055 cycle.
1056 """
J. Richard Barnetteeb69d722012-06-18 17:29:44 -07001057 if not self.wait_up(timeout=self.RESUME_TIMEOUT):
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001058 raise error.TestFail(
1059 'client failed to resume from sleep after %d seconds' %
J. Richard Barnetteeb69d722012-06-18 17:29:44 -07001060 self.RESUME_TIMEOUT)
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001061 else:
1062 new_boot_id = self.get_boot_id()
1063 if new_boot_id != old_boot_id:
1064 raise error.TestFail(
1065 'client rebooted, but sleep was expected'
1066 ' (old boot %s, new boot %s)'
1067 % (old_boot_id, new_boot_id))
1068
1069
1070 def test_wait_for_shutdown(self):
1071 """Wait for the client to shut down.
1072
1073 The test for "has shut down" can't distinguish a system that
1074 is merely asleep; to confirm that the unit was down, it is
1075 necessary to force boot, and then call test_wait_for_boot().
1076
1077 This function is expected to be called from a test as part
1078 of a sequence like the following:
1079
1080 ~~~~~~~~
1081 boot_id = host.get_boot_id()
1082 # trigger shutdown on the host
1083 host.test_wait_for_shutdown()
1084 # trigger boot on the host
1085 host.test_wait_for_boot(boot_id)
1086 ~~~~~~~~
1087
1088 @exception TestFail The host did not shut down within the
1089 allowed time.
1090 """
Andrew Bresticker678c0c72013-01-22 10:44:09 -08001091 if not self.ping_wait_down(timeout=self.SHUTDOWN_TIMEOUT):
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001092 raise error.TestFail(
1093 'client failed to shut down after %d seconds' %
J. Richard Barnetteeb69d722012-06-18 17:29:44 -07001094 self.SHUTDOWN_TIMEOUT)
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001095
1096
1097 def test_wait_for_boot(self, old_boot_id=None):
1098 """Wait for the client to boot from cold power.
1099
1100 The `old_boot_id` parameter should be the value from
1101 `get_boot_id()` obtained prior to shutting down. A
1102 `TestFail` exception is raised if the boot id does not
1103 change. The boot id test is omitted if `old_boot_id` is not
1104 specified.
1105
1106 See @ref test_wait_for_shutdown for more on this function's
1107 usage.
1108
J. Richard Barnette7214e0b2013-02-06 15:20:49 -08001109 @param old_boot_id A boot id value obtained before the
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001110 shut down.
1111
1112 @exception TestFail The host did not respond within the
1113 allowed time.
1114 @exception TestFail The host responded, but the boot id test
1115 indicated that there was no reboot.
1116 """
J. Richard Barnetteeb69d722012-06-18 17:29:44 -07001117 if not self.wait_up(timeout=self.REBOOT_TIMEOUT):
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001118 raise error.TestFail(
1119 'client failed to reboot after %d seconds' %
J. Richard Barnetteeb69d722012-06-18 17:29:44 -07001120 self.REBOOT_TIMEOUT)
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001121 elif old_boot_id:
1122 if self.get_boot_id() == old_boot_id:
1123 raise error.TestFail(
1124 'client is back up, but did not reboot'
1125 ' (boot %s)' % old_boot_id)
Simran Basid5e5e272012-09-24 15:23:59 -07001126
1127
1128 @staticmethod
1129 def check_for_rpm_support(hostname):
1130 """For a given hostname, return whether or not it is powered by an RPM.
1131
1132 @return None if this host does not follows the defined naming format
1133 for RPM powered DUT's in the lab. If it does follow the format,
1134 it returns a regular expression MatchObject instead.
1135 """
Richard Barnette82c35912012-11-20 10:09:10 -08001136 return re.match(SiteHost._RPM_HOSTNAME_REGEX, hostname)
Simran Basid5e5e272012-09-24 15:23:59 -07001137
1138
1139 def has_power(self):
1140 """For this host, return whether or not it is powered by an RPM.
1141
1142 @return True if this host is in the CROS lab and follows the defined
1143 naming format.
1144 """
1145 return SiteHost.check_for_rpm_support(self.hostname)
1146
1147
Ismail Noorbasha07fdb612013-02-14 14:13:31 -08001148 def _set_power(self, state, power_method):
1149 """Sets the power to the host via RPM, Servo or manual.
1150
1151 @param state Specifies which power state to set to DUT
1152 @param power_method Specifies which method of power control to
1153 use. By default "RPM" will be used. Valid values
1154 are the strings "RPM", "manual", "servoj10".
1155
1156 """
1157 ACCEPTABLE_STATES = ['ON', 'OFF']
1158
1159 if state.upper() not in ACCEPTABLE_STATES:
1160 raise error.TestError('State must be one of: %s.'
1161 % (ACCEPTABLE_STATES,))
1162
1163 if power_method == self.POWER_CONTROL_SERVO:
1164 logging.info('Setting servo port J10 to %s', state)
1165 self.servo.set('prtctl3_pwren', state.lower())
1166 time.sleep(self._USB_POWER_TIMEOUT)
1167 elif power_method == self.POWER_CONTROL_MANUAL:
1168 logging.info('You have %d seconds to set the AC power to %s.',
1169 self._POWER_CYCLE_TIMEOUT, state)
1170 time.sleep(self._POWER_CYCLE_TIMEOUT)
1171 else:
1172 if not self.has_power():
1173 raise error.TestFail('DUT does not have RPM connected.')
1174 rpm_client.set_power(self.hostname, state.upper())
Simran Basid5e5e272012-09-24 15:23:59 -07001175
1176
Ismail Noorbasha07fdb612013-02-14 14:13:31 -08001177 def power_off(self, power_method=POWER_CONTROL_RPM):
1178 """Turn off power to this host via RPM, Servo or manual.
1179
1180 @param power_method Specifies which method of power control to
1181 use. By default "RPM" will be used. Valid values
1182 are the strings "RPM", "manual", "servoj10".
1183
1184 """
1185 self._set_power('OFF', power_method)
Simran Basid5e5e272012-09-24 15:23:59 -07001186
1187
Ismail Noorbasha07fdb612013-02-14 14:13:31 -08001188 def power_on(self, power_method=POWER_CONTROL_RPM):
1189 """Turn on power to this host via RPM, Servo or manual.
1190
1191 @param power_method Specifies which method of power control to
1192 use. By default "RPM" will be used. Valid values
1193 are the strings "RPM", "manual", "servoj10".
1194
1195 """
1196 self._set_power('ON', power_method)
1197
1198
1199 def power_cycle(self, power_method=POWER_CONTROL_RPM):
1200 """Cycle power to this host by turning it OFF, then ON.
1201
1202 @param power_method Specifies which method of power control to
1203 use. By default "RPM" will be used. Valid values
1204 are the strings "RPM", "manual", "servoj10".
1205
1206 """
1207 if power_method in (self.POWER_CONTROL_SERVO,
1208 self.POWER_CONTROL_MANUAL):
1209 self.power_off(power_method=power_method)
1210 time.sleep(self._POWER_CYCLE_TIMEOUT)
1211 self.power_on(power_method=power_method)
1212 else:
1213 rpm_client.set_power(self.hostname, 'CYCLE')
Simran Basic6f1f7a2012-10-16 10:47:46 -07001214
1215
1216 def get_platform(self):
1217 """Determine the correct platform label for this host.
1218
1219 @returns a string representing this host's platform.
1220 """
1221 crossystem = utils.Crossystem(self)
1222 crossystem.init()
1223 # Extract fwid value and use the leading part as the platform id.
1224 # fwid generally follow the format of {platform}.{firmware version}
1225 # Example: Alex.X.YYY.Z or Google_Alex.X.YYY.Z
1226 platform = crossystem.fwid().split('.')[0].lower()
1227 # Newer platforms start with 'Google_' while the older ones do not.
1228 return platform.replace('google_', '')
1229
1230
Aviv Keshet74c89a92013-02-04 15:18:30 -08001231 @label_decorator()
Simran Basic6f1f7a2012-10-16 10:47:46 -07001232 def get_board(self):
1233 """Determine the correct board label for this host.
1234
1235 @returns a string representing this host's board.
1236 """
1237 release_info = utils.parse_cmd_output('cat /etc/lsb-release',
1238 run_method=self.run)
1239 board = release_info['CHROMEOS_RELEASE_BOARD']
1240 # Devices in the lab generally have the correct board name but our own
1241 # development devices have {board_name}-signed-{key_type}. The board
1242 # name may also begin with 'x86-' which we need to keep.
Simran Basi833814b2013-01-29 13:13:43 -08001243 board_format_string = ds_constants.BOARD_PREFIX + '%s'
Simran Basic6f1f7a2012-10-16 10:47:46 -07001244 if 'x86' not in board:
Simran Basi833814b2013-01-29 13:13:43 -08001245 return board_format_string % board.split('-')[0]
1246 return board_format_string % '-'.join(board.split('-')[0:2])
Simran Basic6f1f7a2012-10-16 10:47:46 -07001247
1248
Aviv Keshet74c89a92013-02-04 15:18:30 -08001249 @label_decorator('lightsensor')
Simran Basic6f1f7a2012-10-16 10:47:46 -07001250 def has_lightsensor(self):
1251 """Determine the correct board label for this host.
1252
1253 @returns the string 'lightsensor' if this host has a lightsensor or
1254 None if it does not.
1255 """
1256 search_cmd = "find -L %s -maxdepth 4 | egrep '%s'" % (
Richard Barnette82c35912012-11-20 10:09:10 -08001257 self._LIGHTSENSOR_SEARCH_DIR, '|'.join(self._LIGHTSENSOR_FILES))
Simran Basic6f1f7a2012-10-16 10:47:46 -07001258 try:
1259 # Run the search cmd following the symlinks. Stderr_tee is set to
1260 # None as there can be a symlink loop, but the command will still
1261 # execute correctly with a few messages printed to stderr.
1262 self.run(search_cmd, stdout_tee=None, stderr_tee=None)
1263 return 'lightsensor'
1264 except error.AutoservRunError:
1265 # egrep exited with a return code of 1 meaning none of the possible
1266 # lightsensor files existed.
1267 return None
1268
1269
Aviv Keshet74c89a92013-02-04 15:18:30 -08001270 @label_decorator('bluetooth')
Simran Basic6f1f7a2012-10-16 10:47:46 -07001271 def has_bluetooth(self):
1272 """Determine the correct board label for this host.
1273
1274 @returns the string 'bluetooth' if this host has bluetooth or
1275 None if it does not.
1276 """
1277 try:
1278 self.run('test -d /sys/class/bluetooth/hci0')
1279 # test exited with a return code of 0.
1280 return 'bluetooth'
1281 except error.AutoservRunError:
1282 # test exited with a return code 1 meaning the directory did not
1283 # exist.
1284 return None
1285
1286
1287 def get_labels(self):
1288 """Return a list of labels for this given host.
1289
1290 This is the main way to retrieve all the automatic labels for a host
1291 as it will run through all the currently implemented label functions.
1292 """
1293 labels = []
Richard Barnette82c35912012-11-20 10:09:10 -08001294 for label_function in self._LABEL_FUNCTIONS:
Simran Basic6f1f7a2012-10-16 10:47:46 -07001295 label = label_function(self)
1296 if label:
1297 labels.append(label)
1298 return labels