blob: cb4a74cd4a3711fe54a7861a4f88c435a8e01f9f [file] [log] [blame]
J. Richard Barnette24adbf42012-04-11 15:04:53 -07001# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
Dale Curtisaa5eedb2011-08-23 16:18:52 -07002# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
Aviv Keshet74c89a92013-02-04 15:18:30 -08005import functools
Christopher Wiley0ed712b2013-04-09 15:25:12 -07006import httplib
J. Richard Barnette1d78b012012-05-15 13:56:30 -07007import logging
Dan Shi0f466e82013-02-22 15:44:58 -08008import os
Simran Basid5e5e272012-09-24 15:23:59 -07009import re
Christopher Wileyd78249a2013-03-01 13:05:31 -080010import socket
J. Richard Barnette1d78b012012-05-15 13:56:30 -070011import subprocess
J. Richard Barnette134ec2c2012-04-25 12:59:37 -070012import time
J. Richard Barnette1d78b012012-05-15 13:56:30 -070013import xmlrpclib
J. Richard Barnette134ec2c2012-04-25 12:59:37 -070014
J. Richard Barnette45e93de2012-04-11 17:24:15 -070015from autotest_lib.client.bin import utils
Richard Barnette0c73ffc2012-11-19 15:21:18 -080016from autotest_lib.client.common_lib import error
17from autotest_lib.client.common_lib import global_config
J. Richard Barnette45e93de2012-04-11 17:24:15 -070018from autotest_lib.client.common_lib.cros import autoupdater
Richard Barnette03a0c132012-11-05 12:40:35 -080019from autotest_lib.client.common_lib.cros import dev_server
Christopher Wileyd78249a2013-03-01 13:05:31 -080020from autotest_lib.client.common_lib.cros import retry
Richard Barnette82c35912012-11-20 10:09:10 -080021from autotest_lib.client.cros import constants
J. Richard Barnette84890bd2014-02-21 11:05:47 -080022from autotest_lib.client.cros import cros_ui
J. Richard Barnette45e93de2012-04-11 17:24:15 -070023from autotest_lib.server import autoserv_parser
Dan Shia1ecd5c2013-06-06 11:21:31 -070024from autotest_lib.server import utils as server_utils
Scott Zawalski89c44dd2013-02-26 09:28:02 -050025from autotest_lib.server.cros.dynamic_suite import constants as ds_constants
Simran Basi5e6339a2013-03-21 11:34:32 -070026from autotest_lib.server.cros.dynamic_suite import tools, frontend_wrappers
Fang Deng96667ca2013-08-01 17:46:18 -070027from autotest_lib.server.hosts import abstract_ssh
Tom Wai-Hong Tamefe1c7f2014-01-02 14:00:11 +080028from autotest_lib.server.hosts import chameleon_host
Fang Deng5d518f42013-08-02 14:04:32 -070029from autotest_lib.server.hosts import servo_host
beeps687243d2013-07-18 15:29:27 -070030from autotest_lib.site_utils.graphite import stats
Simran Basidcff4252012-11-20 16:13:20 -080031from autotest_lib.site_utils.rpm_control_system import rpm_client
Simran Basid5e5e272012-09-24 15:23:59 -070032
33
beeps32a63082013-08-22 14:02:29 -070034try:
35 import jsonrpclib
36except ImportError:
37 jsonrpclib = None
Fang Deng96667ca2013-08-01 17:46:18 -070038
Fang Dengd1c2b732013-08-20 12:59:46 -070039
beepsc87ff602013-07-31 21:53:00 -070040class FactoryImageCheckerException(error.AutoservError):
41 """Exception raised when an image is a factory image."""
42 pass
43
44
Aviv Keshet74c89a92013-02-04 15:18:30 -080045def add_label_detector(label_function_list, label_list=None, label=None):
46 """Decorator used to group functions together into the provided list.
47 @param label_function_list: List of label detecting functions to add
48 decorated function to.
49 @param label_list: List of detectable labels to add detectable labels to.
50 (Default: None)
51 @param label: Label string that is detectable by this detection function
52 (Default: None)
J. Richard Barnette7214e0b2013-02-06 15:20:49 -080053 """
Simran Basic6f1f7a2012-10-16 10:47:46 -070054 def add_func(func):
Aviv Keshet74c89a92013-02-04 15:18:30 -080055 """
56 @param func: The function to be added as a detector.
57 """
58 label_function_list.append(func)
59 if label and label_list is not None:
60 label_list.append(label)
Simran Basic6f1f7a2012-10-16 10:47:46 -070061 return func
62 return add_func
63
64
Fang Deng0ca40e22013-08-27 17:47:44 -070065class CrosHost(abstract_ssh.AbstractSSHHost):
J. Richard Barnette45e93de2012-04-11 17:24:15 -070066 """Chromium OS specific subclass of Host."""
67
68 _parser = autoserv_parser.autoserv_parser
Scott Zawalski62bacae2013-03-05 10:40:32 -050069 _AFE = frontend_wrappers.RetryingAFE(timeout_min=5, delay_sec=10)
J. Richard Barnette45e93de2012-04-11 17:24:15 -070070
Richard Barnette03a0c132012-11-05 12:40:35 -080071 # Timeout values (in seconds) associated with various Chrome OS
72 # state changes.
J. Richard Barnette134ec2c2012-04-25 12:59:37 -070073 #
Richard Barnette0c73ffc2012-11-19 15:21:18 -080074 # In general, a good rule of thumb is that the timeout can be up
75 # to twice the typical measured value on the slowest platform.
76 # The times here have not necessarily been empirically tested to
77 # meet this criterion.
J. Richard Barnetteeb69d722012-06-18 17:29:44 -070078 #
79 # SLEEP_TIMEOUT: Time to allow for suspend to memory.
Richard Barnette0c73ffc2012-11-19 15:21:18 -080080 # RESUME_TIMEOUT: Time to allow for resume after suspend, plus
81 # time to restart the netwowrk.
J. Richard Barnette84890bd2014-02-21 11:05:47 -080082 # SHUTDOWN_TIMEOUT: Time to allow for shut down.
J. Richard Barnetteeb69d722012-06-18 17:29:44 -070083 # BOOT_TIMEOUT: Time to allow for boot from power off. Among
Richard Barnette0c73ffc2012-11-19 15:21:18 -080084 # other things, this must account for the 30 second dev-mode
J. Richard Barnetted4649c62013-03-06 17:42:27 -080085 # screen delay and time to start the network.
J. Richard Barnetteeb69d722012-06-18 17:29:44 -070086 # USB_BOOT_TIMEOUT: Time to allow for boot from a USB device,
Richard Barnette0c73ffc2012-11-19 15:21:18 -080087 # including the 30 second dev-mode delay and time to start the
J. Richard Barnetted4649c62013-03-06 17:42:27 -080088 # network.
beepsf079cfb2013-09-18 17:49:51 -070089 # INSTALL_TIMEOUT: Time to allow for chromeos-install.
J. Richard Barnette84890bd2014-02-21 11:05:47 -080090 # POWERWASH_BOOT_TIMEOUT: Time to allow for a reboot that
91 # includes powerwash.
J. Richard Barnetteeb69d722012-06-18 17:29:44 -070092
93 SLEEP_TIMEOUT = 2
J. Richard Barnetted4649c62013-03-06 17:42:27 -080094 RESUME_TIMEOUT = 10
Tom Wai-Hong Tam4d169ed2014-02-14 11:05:40 +080095 SHUTDOWN_TIMEOUT = 5
J. Richard Barnettefbcc7122013-07-24 18:24:59 -070096 BOOT_TIMEOUT = 60
J. Richard Barnetteeb69d722012-06-18 17:29:44 -070097 USB_BOOT_TIMEOUT = 150
J. Richard Barnette84890bd2014-02-21 11:05:47 -080098 INSTALL_TIMEOUT = 240
Dan Shi2c88eed2013-11-12 10:18:38 -080099 POWERWASH_BOOT_TIMEOUT = 60
Chris Sosab76e0ee2013-05-22 16:55:41 -0700100
J. Richard Barnette84890bd2014-02-21 11:05:47 -0800101 # REBOOT_TIMEOUT: How long to wait for a reboot.
102 #
Chris Sosab76e0ee2013-05-22 16:55:41 -0700103 # We have a long timeout to ensure we don't flakily fail due to other
104 # issues. Shorter timeouts are vetted in platform_RebootAfterUpdate.
Simran Basi1160e2c2013-10-04 16:00:24 -0700105 # TODO(sbasi - crbug.com/276094) Restore to 5 mins once the 'host did not
106 # return from reboot' bug is solved.
107 REBOOT_TIMEOUT = 480
Chris Sosab76e0ee2013-05-22 16:55:41 -0700108
Ismail Noorbasha07fdb612013-02-14 14:13:31 -0800109 # _USB_POWER_TIMEOUT: Time to allow for USB to power toggle ON and OFF.
110 # _POWER_CYCLE_TIMEOUT: Time to allow for manual power cycle.
111 _USB_POWER_TIMEOUT = 5
112 _POWER_CYCLE_TIMEOUT = 10
113
beeps32a63082013-08-22 14:02:29 -0700114 _RPC_PROXY_URL = 'http://localhost:%d'
Christopher Wileydd181852013-10-10 19:56:58 -0700115 _RPC_SHUTDOWN_POLLING_PERIOD_SECONDS = 2
116 _RPC_SHUTDOWN_TIMEOUT_SECONDS = 20
Richard Barnette0c73ffc2012-11-19 15:21:18 -0800117
Richard Barnette82c35912012-11-20 10:09:10 -0800118 _RPM_RECOVERY_BOARDS = global_config.global_config.get_config_value('CROS',
119 'rpm_recovery_boards', type=str).split(',')
120
121 _MAX_POWER_CYCLE_ATTEMPTS = 6
122 _LAB_MACHINE_FILE = '/mnt/stateful_partition/.labmachine'
123 _RPM_HOSTNAME_REGEX = ('chromeos[0-9]+(-row[0-9]+)?-rack[0-9]+[a-z]*-'
124 'host[0-9]+')
125 _LIGHTSENSOR_FILES = ['in_illuminance0_input',
126 'in_illuminance0_raw',
127 'illuminance0_input']
128 _LIGHTSENSOR_SEARCH_DIR = '/sys/bus/iio/devices'
129 _LABEL_FUNCTIONS = []
Aviv Keshet74c89a92013-02-04 15:18:30 -0800130 _DETECTABLE_LABELS = []
131 label_decorator = functools.partial(add_label_detector, _LABEL_FUNCTIONS,
132 _DETECTABLE_LABELS)
J. Richard Barnette134ec2c2012-04-25 12:59:37 -0700133
J. Richard Barnetteb6de7e32013-02-14 13:28:04 -0800134 # Constants used in ping_wait_up() and ping_wait_down().
135 #
136 # _PING_WAIT_COUNT is the approximate number of polling
137 # cycles to use when waiting for a host state change.
138 #
139 # _PING_STATUS_DOWN and _PING_STATUS_UP are names used
140 # for arguments to the internal _ping_wait_for_status()
141 # method.
142 _PING_WAIT_COUNT = 40
143 _PING_STATUS_DOWN = False
144 _PING_STATUS_UP = True
145
Ismail Noorbasha07fdb612013-02-14 14:13:31 -0800146 # Allowed values for the power_method argument.
147
148 # POWER_CONTROL_RPM: Passed as default arg for power_off/on/cycle() methods.
149 # POWER_CONTROL_SERVO: Used in set_power() and power_cycle() methods.
150 # POWER_CONTROL_MANUAL: Used in set_power() and power_cycle() methods.
151 POWER_CONTROL_RPM = 'RPM'
152 POWER_CONTROL_SERVO = 'servoj10'
153 POWER_CONTROL_MANUAL = 'manual'
154
155 POWER_CONTROL_VALID_ARGS = (POWER_CONTROL_RPM,
156 POWER_CONTROL_SERVO,
157 POWER_CONTROL_MANUAL)
Richard Barnette0c73ffc2012-11-19 15:21:18 -0800158
Simran Basi5e6339a2013-03-21 11:34:32 -0700159 _RPM_OUTLET_CHANGED = 'outlet_changed'
160
beeps687243d2013-07-18 15:29:27 -0700161
J. Richard Barnette964fba02012-10-24 17:34:29 -0700162 @staticmethod
beeps46dadc92013-11-07 14:07:10 -0800163 def check_host(host, timeout=10):
164 """
165 Check if the given host is a chrome-os host.
166
167 @param host: An ssh host representing a device.
168 @param timeout: The timeout for the run command.
169
170 @return: True if the host device is chromeos.
171
beeps46dadc92013-11-07 14:07:10 -0800172 """
173 try:
Christopher Wiley1ea80942014-02-26 16:45:08 -0800174 result = host.run('grep -q CHROMEOS /etc/lsb-release && '
175 '! which adb >/dev/null 2>&1',
Christopher Wileyfc3eac02013-11-21 16:24:57 -0800176 ignore_status=True, timeout=timeout)
beeps46dadc92013-11-07 14:07:10 -0800177 except (error.AutoservRunError, error.AutoservSSHTimeout):
178 return False
179 return result.exit_status == 0
180
181
182 @staticmethod
Tom Wai-Hong Tamefe1c7f2014-01-02 14:00:11 +0800183 def _extract_arguments(args_dict, key_subset):
184 """Extract options from `args_dict` and return a subset result.
J. Richard Barnette7214e0b2013-02-06 15:20:49 -0800185
186 Take the provided dictionary of argument options and return
Tom Wai-Hong Tamefe1c7f2014-01-02 14:00:11 +0800187 a subset that represent standard arguments needed to construct
188 a test-assistant object (chameleon or servo) for a host. The
189 intent is to provide standard argument processing from
190 run_remote_tests for tests that require a test-assistant board
191 to operate.
192
193 @param args_dict Dictionary from which to extract the arguments.
194 @param key_subset Tuple of keys to extract from the args_dict, e.g.
195 ('servo_host', 'servo_port').
196 """
197 result = {}
198 for arg in key_subset:
199 if arg in args_dict:
200 result[arg] = args_dict[arg]
201 return result
202
203
204 @staticmethod
205 def get_chameleon_arguments(args_dict):
206 """Extract chameleon options from `args_dict` and return the result.
207
208 Recommended usage:
209 ~~~~~~~~
210 args_dict = utils.args_to_dict(args)
211 chameleon_args = hosts.CrosHost.get_chameleon_arguments(args_dict)
212 host = hosts.create_host(machine, chameleon_args=chameleon_args)
213 ~~~~~~~~
214
215 @param args_dict Dictionary from which to extract the chameleon
216 arguments.
217 """
218 return CrosHost._extract_arguments(
219 args_dict, ('chameleon_host', 'chameleon_port'))
220
221
222 @staticmethod
223 def get_servo_arguments(args_dict):
224 """Extract servo options from `args_dict` and return the result.
J. Richard Barnette7214e0b2013-02-06 15:20:49 -0800225
226 Recommended usage:
227 ~~~~~~~~
228 args_dict = utils.args_to_dict(args)
Fang Deng0ca40e22013-08-27 17:47:44 -0700229 servo_args = hosts.CrosHost.get_servo_arguments(args_dict)
J. Richard Barnette7214e0b2013-02-06 15:20:49 -0800230 host = hosts.create_host(machine, servo_args=servo_args)
231 ~~~~~~~~
232
233 @param args_dict Dictionary from which to extract the servo
234 arguments.
235 """
Tom Wai-Hong Tamefe1c7f2014-01-02 14:00:11 +0800236 return CrosHost._extract_arguments(
237 args_dict, ('servo_host', 'servo_port'))
J. Richard Barnette134ec2c2012-04-25 12:59:37 -0700238
J. Richard Barnette964fba02012-10-24 17:34:29 -0700239
Tom Wai-Hong Tamefe1c7f2014-01-02 14:00:11 +0800240 def _initialize(self, hostname, chameleon_args=None, servo_args=None,
241 ssh_verbosity_flag='', ssh_options='',
Fang Dengd1c2b732013-08-20 12:59:46 -0700242 *args, **dargs):
Tom Wai-Hong Tamefe1c7f2014-01-02 14:00:11 +0800243 """Initialize superclasses, |self.chameleon|, and |self.servo|.
J. Richard Barnette67ccb872012-04-19 16:34:56 -0700244
Tom Wai-Hong Tamefe1c7f2014-01-02 14:00:11 +0800245 This method checks whether a chameleon/servo (aka
246 test-assistant objects) is required by checking whether
247 chameleon_args/servo_args is None. This method will only
248 attempt to create the test-assistant object when it is
249 required by the test.
Fang Deng5d518f42013-08-02 14:04:32 -0700250
Tom Wai-Hong Tamefe1c7f2014-01-02 14:00:11 +0800251 For creating the test-assistant object, there are three
252 possibilities: First, if the host is a lab system known to have
253 a test-assistant board, we connect to that board unconditionally.
J. Richard Barnette67ccb872012-04-19 16:34:56 -0700254 Second, if we're called from a control file that requires
Tom Wai-Hong Tamefe1c7f2014-01-02 14:00:11 +0800255 test-assistant features for testing, it will pass settings from
256 the arguments, like `servo_host`, `servo_port`. If neither of
257 these cases apply, the test-assistant object will be `None`.
J. Richard Barnette67ccb872012-04-19 16:34:56 -0700258
259 """
Fang Deng0ca40e22013-08-27 17:47:44 -0700260 super(CrosHost, self)._initialize(hostname=hostname,
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700261 *args, **dargs)
J. Richard Barnettef0859852012-08-20 14:55:50 -0700262 # self.env is a dictionary of environment variable settings
263 # to be exported for commands run on the host.
264 # LIBC_FATAL_STDERR_ can be useful for diagnosing certain
265 # errors that might happen.
266 self.env['LIBC_FATAL_STDERR_'] = '1'
beeps32a63082013-08-22 14:02:29 -0700267 self._rpc_proxy_map = {}
Fang Dengd1c2b732013-08-20 12:59:46 -0700268 self._ssh_verbosity_flag = ssh_verbosity_flag
Aviv Keshetc5947fa2013-09-04 14:06:29 -0700269 self._ssh_options = ssh_options
Fang Deng5d518f42013-08-02 14:04:32 -0700270 # TODO(fdeng): We need to simplify the
271 # process of servo and servo_host initialization.
272 # crbug.com/298432
Dan Shi4d478522014-02-14 13:46:32 -0800273 self._servo_host = servo_host.create_servo_host(dut=self.hostname,
274 servo_args=servo_args)
Tom Wai-Hong Tamefe1c7f2014-01-02 14:00:11 +0800275 # TODO(waihong): Do the simplication on Chameleon too.
Tom Wai-Hong Tam3d6790d2014-04-14 16:15:47 +0800276 self._chameleon_host = chameleon_host.create_chameleon_host(
277 dut=self.hostname, chameleon_args=chameleon_args)
278
Dan Shi4d478522014-02-14 13:46:32 -0800279 if self._servo_host is not None:
280 self.servo = self._servo_host.get_servo()
281 else:
282 self.servo = None
283
Tom Wai-Hong Tam3d6790d2014-04-14 16:15:47 +0800284 if self._chameleon_host:
Tom Wai-Hong Tameaee3402014-01-22 08:52:10 +0800285 self.chameleon = self._chameleon_host.create_chameleon_board()
Tom Wai-Hong Tamefe1c7f2014-01-02 14:00:11 +0800286 else:
Tom Wai-Hong Tam3d6790d2014-04-14 16:15:47 +0800287 self.chameleon = None
Fang Deng5d518f42013-08-02 14:04:32 -0700288
289
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500290 def get_repair_image_name(self):
291 """Generate a image_name from variables in the global config.
292
293 @returns a str of $board-version/$BUILD.
294
295 """
296 stable_version = global_config.global_config.get_config_value(
297 'CROS', 'stable_cros_version')
298 build_pattern = global_config.global_config.get_config_value(
299 'CROS', 'stable_build_pattern')
300 board = self._get_board_from_afe()
301 if board is None:
302 raise error.AutoservError('DUT has no board attribute, '
303 'cannot be repaired.')
304 return build_pattern % (board, stable_version)
305
306
Scott Zawalski62bacae2013-03-05 10:40:32 -0500307 def _host_in_AFE(self):
308 """Check if the host is an object the AFE knows.
309
310 @returns the host object.
311 """
312 return self._AFE.get_hosts(hostname=self.hostname)
313
314
Chris Sosab76e0ee2013-05-22 16:55:41 -0700315 def lookup_job_repo_url(self):
316 """Looks up the job_repo_url for the host.
317
318 @returns job_repo_url from AFE or None if not found.
319
320 @raises KeyError if the host does not have a job_repo_url
321 """
322 if not self._host_in_AFE():
323 return None
324
325 hosts = self._AFE.get_hosts(hostname=self.hostname)
beepsb5efc532013-06-04 11:29:34 -0700326 if hosts and ds_constants.JOB_REPO_URL in hosts[0].attributes:
327 return hosts[0].attributes[ds_constants.JOB_REPO_URL]
Chris Sosab76e0ee2013-05-22 16:55:41 -0700328
329
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500330 def clear_cros_version_labels_and_job_repo_url(self):
331 """Clear cros_version labels and host attribute job_repo_url."""
Scott Zawalski62bacae2013-03-05 10:40:32 -0500332 if not self._host_in_AFE():
Scott Zawalskieadbf702013-03-14 09:23:06 -0400333 return
334
Scott Zawalski62bacae2013-03-05 10:40:32 -0500335 host_list = [self.hostname]
336 labels = self._AFE.get_labels(
337 name__startswith=ds_constants.VERSION_PREFIX,
338 host__hostname=self.hostname)
Dan Shi0f466e82013-02-22 15:44:58 -0800339
Scott Zawalski62bacae2013-03-05 10:40:32 -0500340 for label in labels:
341 label.remove_hosts(hosts=host_list)
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500342
beepscb6f1e22013-06-28 19:14:10 -0700343 self.update_job_repo_url(None, None)
344
345
346 def update_job_repo_url(self, devserver_url, image_name):
347 """
348 Updates the job_repo_url host attribute and asserts it's value.
349
350 @param devserver_url: The devserver to use in the job_repo_url.
351 @param image_name: The name of the image to use in the job_repo_url.
352
353 @raises AutoservError: If we failed to update the job_repo_url.
354 """
355 repo_url = None
356 if devserver_url and image_name:
357 repo_url = tools.get_package_url(devserver_url, image_name)
358 self._AFE.set_host_attribute(ds_constants.JOB_REPO_URL, repo_url,
Scott Zawalski62bacae2013-03-05 10:40:32 -0500359 hostname=self.hostname)
beepscb6f1e22013-06-28 19:14:10 -0700360 if self.lookup_job_repo_url() != repo_url:
361 raise error.AutoservError('Failed to update job_repo_url with %s, '
362 'host %s' % (repo_url, self.hostname))
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500363
364
Dan Shie9309262013-06-19 22:50:21 -0700365 def add_cros_version_labels_and_job_repo_url(self, image_name):
Scott Zawalskieadbf702013-03-14 09:23:06 -0400366 """Add cros_version labels and host attribute job_repo_url.
367
368 @param image_name: The name of the image e.g.
369 lumpy-release/R27-3837.0.0
Dan Shi7458bf62013-06-10 12:50:16 -0700370
Scott Zawalskieadbf702013-03-14 09:23:06 -0400371 """
Scott Zawalski62bacae2013-03-05 10:40:32 -0500372 if not self._host_in_AFE():
Scott Zawalskieadbf702013-03-14 09:23:06 -0400373 return
Scott Zawalski62bacae2013-03-05 10:40:32 -0500374
Scott Zawalskieadbf702013-03-14 09:23:06 -0400375 cros_label = '%s%s' % (ds_constants.VERSION_PREFIX, image_name)
Dan Shie9309262013-06-19 22:50:21 -0700376 devserver_url = dev_server.ImageServer.resolve(image_name).url()
Scott Zawalski62bacae2013-03-05 10:40:32 -0500377
378 labels = self._AFE.get_labels(name=cros_label)
379 if labels:
380 label = labels[0]
381 else:
382 label = self._AFE.create_label(name=cros_label)
383
384 label.add_hosts([self.hostname])
beepscb6f1e22013-06-28 19:14:10 -0700385 self.update_job_repo_url(devserver_url, image_name)
386
387
beepsdae65fd2013-07-26 16:24:41 -0700388 def verify_job_repo_url(self, tag=''):
beepscb6f1e22013-06-28 19:14:10 -0700389 """
390 Make sure job_repo_url of this host is valid.
391
joychen03eaad92013-06-26 09:55:21 -0700392 Eg: The job_repo_url "http://lmn.cd.ab.xyx:8080/static/\
beepscb6f1e22013-06-28 19:14:10 -0700393 lumpy-release/R29-4279.0.0/autotest/packages" claims to have the
394 autotest package for lumpy-release/R29-4279.0.0. If this isn't the case,
395 download and extract it. If the devserver embedded in the url is
396 unresponsive, update the job_repo_url of the host after staging it on
397 another devserver.
398
399 @param job_repo_url: A url pointing to the devserver where the autotest
400 package for this build should be staged.
beepsdae65fd2013-07-26 16:24:41 -0700401 @param tag: The tag from the server job, in the format
402 <job_id>-<user>/<hostname>, or <hostless> for a server job.
beepscb6f1e22013-06-28 19:14:10 -0700403
404 @raises DevServerException: If we could not resolve a devserver.
405 @raises AutoservError: If we're unable to save the new job_repo_url as
406 a result of choosing a new devserver because the old one failed to
407 respond to a health check.
beeps0c865032013-07-30 11:37:06 -0700408 @raises urllib2.URLError: If the devserver embedded in job_repo_url
409 doesn't respond within the timeout.
beepscb6f1e22013-06-28 19:14:10 -0700410 """
411 job_repo_url = self.lookup_job_repo_url()
412 if not job_repo_url:
413 logging.warning('No job repo url set on host %s', self.hostname)
414 return
415
416 logging.info('Verifying job repo url %s', job_repo_url)
417 devserver_url, image_name = tools.get_devserver_build_from_package_url(
418 job_repo_url)
419
beeps0c865032013-07-30 11:37:06 -0700420 ds = dev_server.ImageServer(devserver_url)
beepscb6f1e22013-06-28 19:14:10 -0700421
422 logging.info('Staging autotest artifacts for %s on devserver %s',
423 image_name, ds.url())
beeps687243d2013-07-18 15:29:27 -0700424
425 start_time = time.time()
beepscb6f1e22013-06-28 19:14:10 -0700426 ds.stage_artifacts(image_name, ['autotest'])
beeps687243d2013-07-18 15:29:27 -0700427 stage_time = time.time() - start_time
428
429 # Record how much of the verification time comes from a devserver
430 # restage. If we're doing things right we should not see multiple
431 # devservers for a given board/build/branch path.
432 try:
J. Richard Barnette3cbd76b2013-11-27 12:11:25 -0800433 board, build_type, branch = server_utils.ParseBuildName(
beeps687243d2013-07-18 15:29:27 -0700434 image_name)[:3]
J. Richard Barnette3cbd76b2013-11-27 12:11:25 -0800435 except server_utils.ParseBuildNameException:
beeps687243d2013-07-18 15:29:27 -0700436 pass
437 else:
beeps0c865032013-07-30 11:37:06 -0700438 devserver = devserver_url[
Chris Sosa65425082013-10-16 13:26:22 -0700439 devserver_url.find('/') + 2:devserver_url.rfind(':')]
beeps687243d2013-07-18 15:29:27 -0700440 stats_key = {
441 'board': board,
442 'build_type': build_type,
443 'branch': branch,
beeps0c865032013-07-30 11:37:06 -0700444 'devserver': devserver.replace('.', '_'),
beeps687243d2013-07-18 15:29:27 -0700445 }
446 stats.Gauge('verify_job_repo_url').send(
447 '%(board)s.%(build_type)s.%(branch)s.%(devserver)s' % stats_key,
448 stage_time)
beepscb6f1e22013-06-28 19:14:10 -0700449
Scott Zawalskieadbf702013-03-14 09:23:06 -0400450
Dan Shi0f466e82013-02-22 15:44:58 -0800451 def _try_stateful_update(self, update_url, force_update, updater):
452 """Try to use stateful update to initialize DUT.
453
454 When DUT is already running the same version that machine_install
455 tries to install, stateful update is a much faster way to clean up
456 the DUT for testing, compared to a full reimage. It is implemeted
457 by calling autoupdater.run_update, but skipping updating root, as
458 updating the kernel is time consuming and not necessary.
459
460 @param update_url: url of the image.
461 @param force_update: Set to True to update the image even if the DUT
462 is running the same version.
463 @param updater: ChromiumOSUpdater instance used to update the DUT.
464 @returns: True if the DUT was updated with stateful update.
465
466 """
J. Richard Barnette3f731032014-04-07 17:42:59 -0700467 # TODO(jrbarnette): Yes, I hate this re.match() test case.
468 # It's better than the alternative: see crbug.com/360944.
469 image_name = autoupdater.url_to_image_name(update_url)
470 release_pattern = r'^.*-release/R[0-9]+-[0-9]+\.[0-9]+\.0$'
471 if not re.match(release_pattern, image_name):
472 return False
Dan Shi0f466e82013-02-22 15:44:58 -0800473 if not updater.check_version():
474 return False
475 if not force_update:
476 logging.info('Canceling stateful update because the new and '
477 'old versions are the same.')
478 return False
479 # Following folders should be rebuilt after stateful update.
480 # A test file is used to confirm each folder gets rebuilt after
481 # the stateful update.
482 folders_to_check = ['/var', '/home', '/mnt/stateful_partition']
483 test_file = '.test_file_to_be_deleted'
484 for folder in folders_to_check:
485 touch_path = os.path.join(folder, test_file)
486 self.run('touch %s' % touch_path)
487
488 if not updater.run_update(force_update=True, update_root=False):
489 return False
490
491 # Reboot to complete stateful update.
Chris Sosab76e0ee2013-05-22 16:55:41 -0700492 self.reboot(timeout=self.REBOOT_TIMEOUT, wait=True)
Dan Shi0f466e82013-02-22 15:44:58 -0800493 check_file_cmd = 'test -f %s; echo $?'
494 for folder in folders_to_check:
495 test_file_path = os.path.join(folder, test_file)
496 result = self.run(check_file_cmd % test_file_path,
497 ignore_status=True)
498 if result.exit_status == 1:
499 return False
500 return True
501
502
J. Richard Barnette7275b612013-06-04 18:13:11 -0700503 def _post_update_processing(self, updater, expected_kernel=None):
Dan Shi0f466e82013-02-22 15:44:58 -0800504 """After the DUT is updated, confirm machine_install succeeded.
505
506 @param updater: ChromiumOSUpdater instance used to update the DUT.
J. Richard Barnette7275b612013-06-04 18:13:11 -0700507 @param expected_kernel: kernel expected to be active after reboot,
508 or `None` to skip rollback checking.
Dan Shi0f466e82013-02-22 15:44:58 -0800509
510 """
J. Richard Barnette7275b612013-06-04 18:13:11 -0700511 # Touch the lab machine file to leave a marker that
512 # distinguishes this image from other test images.
513 # Afterwards, we must re-run the autoreboot script because
514 # it depends on the _LAB_MACHINE_FILE.
Dan Shi0f466e82013-02-22 15:44:58 -0800515 self.run('touch %s' % self._LAB_MACHINE_FILE)
Dan Shi0f466e82013-02-22 15:44:58 -0800516 self.run('start autoreboot')
Chris Sosa65425082013-10-16 13:26:22 -0700517 updater.verify_boot_expectations(
518 expected_kernel, rollback_message=
519 'Build %s failed to boot on %s; system rolled back to previous'
520 'build' % (updater.update_version, self.hostname))
J. Richard Barnette7275b612013-06-04 18:13:11 -0700521 # Check that we've got the build we meant to install.
522 if not updater.check_version_to_confirm_install():
523 raise autoupdater.ChromiumOSError(
524 'Failed to update %s to build %s; found build '
525 '%s instead' % (self.hostname,
Chris Sosa65425082013-10-16 13:26:22 -0700526 updater.update_version,
527 updater.get_build_id()))
Dan Shi0f466e82013-02-22 15:44:58 -0800528
529
J. Richard Barnettee4af8b92013-05-01 13:16:12 -0700530 def _stage_image_for_update(self, image_name=None):
Scott Zawalskieadbf702013-03-14 09:23:06 -0400531 """Stage a build on a devserver and return the update_url.
532
533 @param image_name: a name like lumpy-release/R27-3837.0.0
534 @returns an update URL like:
535 http://172.22.50.205:8082/update/lumpy-release/R27-3837.0.0
536 """
J. Richard Barnettee4af8b92013-05-01 13:16:12 -0700537 if not image_name:
538 image_name = self.get_repair_image_name()
539 logging.info('Staging build for AU: %s', image_name)
Scott Zawalskieadbf702013-03-14 09:23:06 -0400540 devserver = dev_server.ImageServer.resolve(image_name)
541 devserver.trigger_download(image_name, synchronous=False)
542 return tools.image_url_pattern() % (devserver.url(), image_name)
543
544
J. Richard Barnettee4af8b92013-05-01 13:16:12 -0700545 def stage_image_for_servo(self, image_name=None):
546 """Stage a build on a devserver and return the update_url.
547
548 @param image_name: a name like lumpy-release/R27-3837.0.0
549 @returns an update URL like:
550 http://172.22.50.205:8082/update/lumpy-release/R27-3837.0.0
551 """
552 if not image_name:
553 image_name = self.get_repair_image_name()
554 logging.info('Staging build for servo install: %s', image_name)
555 devserver = dev_server.ImageServer.resolve(image_name)
556 devserver.stage_artifacts(image_name, ['test_image'])
557 return devserver.get_test_image_url(image_name)
558
559
beepse539be02013-07-31 21:57:39 -0700560 def stage_factory_image_for_servo(self, image_name):
561 """Stage a build on a devserver and return the update_url.
562
563 @param image_name: a name like <baord>/4262.204.0
beeps12c0a3c2013-09-03 11:58:27 -0700564
beepse539be02013-07-31 21:57:39 -0700565 @return: An update URL, eg:
566 http://<devserver>/static/canary-channel/\
567 <board>/4262.204.0/factory_test/chromiumos_factory_image.bin
beeps12c0a3c2013-09-03 11:58:27 -0700568
569 @raises: ValueError if the factory artifact name is missing from
570 the config.
571
beepse539be02013-07-31 21:57:39 -0700572 """
573 if not image_name:
574 logging.error('Need an image_name to stage a factory image.')
575 return
576
beeps12c0a3c2013-09-03 11:58:27 -0700577 factory_artifact = global_config.global_config.get_config_value(
578 'CROS', 'factory_artifact', type=str, default='')
579 if not factory_artifact:
580 raise ValueError('Cannot retrieve the factory artifact name from '
581 'autotest config, and hence cannot stage factory '
582 'artifacts.')
583
beepse539be02013-07-31 21:57:39 -0700584 logging.info('Staging build for servo install: %s', image_name)
585 devserver = dev_server.ImageServer.resolve(image_name)
586 devserver.stage_artifacts(
587 image_name,
beeps12c0a3c2013-09-03 11:58:27 -0700588 [factory_artifact],
589 archive_url=None)
beepse539be02013-07-31 21:57:39 -0700590
591 return tools.factory_image_url_pattern() % (devserver.url(), image_name)
592
593
Chris Sosaa3ac2152012-05-23 22:23:13 -0700594 def machine_install(self, update_url=None, force_update=False,
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500595 local_devserver=False, repair=False):
596 """Install the DUT.
597
Dan Shi0f466e82013-02-22 15:44:58 -0800598 Use stateful update if the DUT is already running the same build.
599 Stateful update does not update kernel and tends to run much faster
600 than a full reimage. If the DUT is running a different build, or it
601 failed to do a stateful update, full update, including kernel update,
602 will be applied to the DUT.
603
Scott Zawalskieadbf702013-03-14 09:23:06 -0400604 Once a host enters machine_install its cros_version label will be
605 removed as well as its host attribute job_repo_url (used for
606 package install).
607
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500608 @param update_url: The url to use for the update
609 pattern: http://$devserver:###/update/$build
610 If update_url is None and repair is True we will install the
611 stable image listed in global_config under
612 CROS.stable_cros_version.
613 @param force_update: Force an update even if the version installed
614 is the same. Default:False
615 @param local_devserver: Used by run_remote_test to allow people to
616 use their local devserver. Default: False
617 @param repair: Whether or not we are in repair mode. This adds special
618 cases for repairing a machine like starting update_engine.
619 Setting repair to True sets force_update to True as well.
620 default: False
621 @raises autoupdater.ChromiumOSError
622
623 """
Dan Shi7458bf62013-06-10 12:50:16 -0700624 if update_url:
625 logging.debug('update url is set to %s', update_url)
626 else:
627 logging.debug('update url is not set, resolving...')
J. Richard Barnettee4af8b92013-05-01 13:16:12 -0700628 if self._parser.options.image:
629 requested_build = self._parser.options.image
630 if requested_build.startswith('http://'):
631 update_url = requested_build
Dan Shi7458bf62013-06-10 12:50:16 -0700632 logging.debug('update url is retrieved from requested_build'
633 ': %s', update_url)
J. Richard Barnettee4af8b92013-05-01 13:16:12 -0700634 else:
635 # Try to stage any build that does not start with
636 # http:// on the devservers defined in
637 # global_config.ini.
Dan Shi7458bf62013-06-10 12:50:16 -0700638 update_url = self._stage_image_for_update(requested_build)
639 logging.debug('Build staged, and update_url is set to: %s',
640 update_url)
J. Richard Barnettee4af8b92013-05-01 13:16:12 -0700641 elif repair:
642 update_url = self._stage_image_for_update()
Dan Shi7458bf62013-06-10 12:50:16 -0700643 logging.debug('Build staged, and update_url is set to: %s',
644 update_url)
Scott Zawalskieadbf702013-03-14 09:23:06 -0400645 else:
J. Richard Barnettee4af8b92013-05-01 13:16:12 -0700646 raise autoupdater.ChromiumOSError(
647 'Update failed. No update URL provided.')
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500648
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500649 if repair:
Dan Shi0f466e82013-02-22 15:44:58 -0800650 # In case the system is in a bad state, we always reboot the machine
651 # before machine_install.
Chris Sosab76e0ee2013-05-22 16:55:41 -0700652 self.reboot(timeout=self.REBOOT_TIMEOUT, wait=True)
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500653 self.run('stop update-engine; start update-engine')
654 force_update = True
Dan Shi0f466e82013-02-22 15:44:58 -0800655
Chris Sosaa3ac2152012-05-23 22:23:13 -0700656 updater = autoupdater.ChromiumOSUpdater(update_url, host=self,
Chris Sosa72312602013-04-16 15:01:56 -0700657 local_devserver=local_devserver)
Dan Shi0f466e82013-02-22 15:44:58 -0800658 updated = False
Scott Zawalskieadbf702013-03-14 09:23:06 -0400659 # Remove cros-version and job_repo_url host attribute from host.
660 self.clear_cros_version_labels_and_job_repo_url()
Dan Shi0f466e82013-02-22 15:44:58 -0800661 # If the DUT is already running the same build, try stateful update
662 # first. Stateful update does not update kernel and tends to run much
663 # faster than a full reimage.
664 try:
Chris Sosab76e0ee2013-05-22 16:55:41 -0700665 updated = self._try_stateful_update(
666 update_url, force_update, updater)
Dan Shi0f466e82013-02-22 15:44:58 -0800667 if updated:
668 logging.info('DUT is updated with stateful update.')
669 except Exception as e:
670 logging.exception(e)
Ilja H. Friedel04be2bd2014-05-07 21:29:59 -0700671 logging.warning('Failed to stateful update DUT, force to update.')
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700672
Dan Shi0f466e82013-02-22 15:44:58 -0800673 inactive_kernel = None
674 # Do a full update if stateful update is not applicable or failed.
675 if not updated:
676 # In case the system is in a bad state, we always reboot the
677 # machine before machine_install.
Chris Sosab76e0ee2013-05-22 16:55:41 -0700678 self.reboot(timeout=self.REBOOT_TIMEOUT, wait=True)
Chris Sosab7612bc2013-03-21 10:32:37 -0700679
680 # TODO(sosa): Remove temporary hack to get rid of bricked machines
681 # that can't update due to a corrupted policy.
682 self.run('rm -rf /var/lib/whitelist')
683 self.run('touch /var/lib/whitelist')
684 self.run('chmod -w /var/lib/whitelist')
Scott Zawalskib550d5a2013-03-22 09:23:59 -0400685 self.run('stop update-engine; start update-engine')
Chris Sosab7612bc2013-03-21 10:32:37 -0700686
Dan Shi0f466e82013-02-22 15:44:58 -0800687 if updater.run_update(force_update):
688 updated = True
689 # Figure out active and inactive kernel.
690 active_kernel, inactive_kernel = updater.get_kernel_state()
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700691
Dan Shi0f466e82013-02-22 15:44:58 -0800692 # Ensure inactive kernel has higher priority than active.
693 if (updater.get_kernel_priority(inactive_kernel)
694 < updater.get_kernel_priority(active_kernel)):
695 raise autoupdater.ChromiumOSError(
696 'Update failed. The priority of the inactive kernel'
697 ' partition is less than that of the active kernel'
698 ' partition.')
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700699
Dan Shi0f466e82013-02-22 15:44:58 -0800700 # Updater has returned successfully; reboot the host.
Chris Sosab76e0ee2013-05-22 16:55:41 -0700701 self.reboot(timeout=self.REBOOT_TIMEOUT, wait=True)
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700702
Dan Shi0f466e82013-02-22 15:44:58 -0800703 if updated:
704 self._post_update_processing(updater, inactive_kernel)
Scott Zawalskieadbf702013-03-14 09:23:06 -0400705 image_name = autoupdater.url_to_image_name(update_url)
Dan Shie9309262013-06-19 22:50:21 -0700706 self.add_cros_version_labels_and_job_repo_url(image_name)
Simran Basi13fa1ba2013-03-04 10:56:47 -0800707
J. Richard Barnette45e93de2012-04-11 17:24:15 -0700708 # Clean up any old autotest directories which may be lying around.
709 for path in global_config.global_config.get_config_value(
710 'AUTOSERV', 'client_autodir_paths', type=list):
711 self.run('rm -rf ' + path)
712
713
Dan Shi10e992b2013-08-30 11:02:59 -0700714 def show_update_engine_log(self):
715 """Output update engine log."""
716 logging.debug('Dumping %s', constants.UPDATE_ENGINE_LOG)
717 self.run('cat %s' % constants.UPDATE_ENGINE_LOG)
718
719
Richard Barnette82c35912012-11-20 10:09:10 -0800720 def _get_board_from_afe(self):
721 """Retrieve this host's board from its labels in the AFE.
722
723 Looks for a host label of the form "board:<board>", and
724 returns the "<board>" part of the label. `None` is returned
725 if there is not a single, unique label matching the pattern.
726
727 @returns board from label, or `None`.
728 """
Dan Shia1ecd5c2013-06-06 11:21:31 -0700729 return server_utils.get_board_from_afe(self.hostname, self._AFE)
Simran Basi833814b2013-01-29 13:13:43 -0800730
731
732 def get_build(self):
733 """Retrieve the current build for this Host from the AFE.
734
735 Looks through this host's labels in the AFE to determine its build.
736
737 @returns The current build or None if it could not find it or if there
738 were multiple build labels assigned to this host.
739 """
Dan Shia1ecd5c2013-06-06 11:21:31 -0700740 return server_utils.get_build_from_afe(self.hostname, self._AFE)
Richard Barnette82c35912012-11-20 10:09:10 -0800741
742
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500743 def _install_repair(self):
744 """Attempt to repair this host using upate-engine.
745
746 If the host is up, try installing the DUT with a stable
747 "repair" version of Chrome OS as defined in the global_config
748 under CROS.stable_cros_version.
749
Scott Zawalski62bacae2013-03-05 10:40:32 -0500750 @raises AutoservRepairMethodNA if the DUT is not reachable.
751 @raises ChromiumOSError if the install failed for some reason.
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500752
753 """
754 if not self.is_up():
Scott Zawalski62bacae2013-03-05 10:40:32 -0500755 raise error.AutoservRepairMethodNA('DUT unreachable for install.')
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500756 logging.info('Attempting to reimage machine to repair image.')
757 try:
758 self.machine_install(repair=True)
Fang Dengd0672f32013-03-18 17:18:09 -0700759 except autoupdater.ChromiumOSError as e:
760 logging.exception(e)
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500761 logging.info('Repair via install failed.')
Scott Zawalski62bacae2013-03-05 10:40:32 -0500762 raise
Scott Zawalski89c44dd2013-02-26 09:28:02 -0500763
764
Dan Shi2c88eed2013-11-12 10:18:38 -0800765 def _install_repair_with_powerwash(self):
Dan Shi9cc48452013-11-12 12:39:26 -0800766 """Attempt to powerwash first then repair this host using update-engine.
Dan Shi2c88eed2013-11-12 10:18:38 -0800767
Dan Shi9cc48452013-11-12 12:39:26 -0800768 update-engine may fail due to a bad image. In such case, powerwash
769 may help to cleanup the DUT for update-engine to work again.
Dan Shi2c88eed2013-11-12 10:18:38 -0800770
771 @raises AutoservRepairMethodNA if the DUT is not reachable.
772 @raises ChromiumOSError if the install failed for some reason.
773
774 """
775 if not self.is_up():
776 raise error.AutoservRepairMethodNA('DUT unreachable for install.')
777
778 logging.info('Attempting to powerwash the DUT.')
779 self.run('echo "fast safe" > '
780 '/mnt/stateful_partition/factory_install_reset')
781 self.reboot(timeout=self.POWERWASH_BOOT_TIMEOUT, wait=True)
782 if not self.is_up():
Dan Shi9cc48452013-11-12 12:39:26 -0800783 logging.error('Powerwash failed. DUT did not come back after '
Dan Shi2c88eed2013-11-12 10:18:38 -0800784 'reboot.')
785 raise error.AutoservRepairFailure(
786 'DUT failed to boot from powerwash after %d seconds' %
787 self.POWERWASH_BOOT_TIMEOUT)
788
789 logging.info('Powerwash succeeded.')
790 self._install_repair()
791
792
beepsf079cfb2013-09-18 17:49:51 -0700793 def servo_install(self, image_url=None, usb_boot_timeout=USB_BOOT_TIMEOUT,
794 install_timeout=INSTALL_TIMEOUT):
Scott Zawalski62bacae2013-03-05 10:40:32 -0500795 """
796 Re-install the OS on the DUT by:
797 1) installing a test image on a USB storage device attached to the Servo
798 board,
Richard Barnette03a0c132012-11-05 12:40:35 -0800799 2) booting that image in recovery mode, and then
J. Richard Barnette31b2e312013-04-04 16:05:22 -0700800 3) installing the image with chromeos-install.
801
Scott Zawalski62bacae2013-03-05 10:40:32 -0500802 @param image_url: If specified use as the url to install on the DUT.
803 otherwise boot the currently staged image on the USB stick.
beepsf079cfb2013-09-18 17:49:51 -0700804 @param usb_boot_timeout: The usb_boot_timeout to use during reimage.
805 Factory images need a longer usb_boot_timeout than regular
806 cros images.
807 @param install_timeout: The timeout to use when installing the chromeos
808 image. Factory images need a longer install_timeout.
Richard Barnette03a0c132012-11-05 12:40:35 -0800809
Scott Zawalski62bacae2013-03-05 10:40:32 -0500810 @raises AutoservError if the image fails to boot.
Richard Barnette03a0c132012-11-05 12:40:35 -0800811 """
beepsf079cfb2013-09-18 17:49:51 -0700812
813 usb_boot_timer_key = ('servo_install.usb_boot_timeout_%s'
814 % usb_boot_timeout)
815 logging.info('Downloading image to USB, then booting from it. Usb boot '
816 'timeout = %s', usb_boot_timeout)
817 timer = stats.Timer(usb_boot_timer_key)
818 timer.start()
J. Richard Barnette31b2e312013-04-04 16:05:22 -0700819 self.servo.install_recovery_image(image_url)
beepsf079cfb2013-09-18 17:49:51 -0700820 if not self.wait_up(timeout=usb_boot_timeout):
Scott Zawalski62bacae2013-03-05 10:40:32 -0500821 raise error.AutoservRepairFailure(
822 'DUT failed to boot from USB after %d seconds' %
beepsf079cfb2013-09-18 17:49:51 -0700823 usb_boot_timeout)
824 timer.stop()
Scott Zawalski62bacae2013-03-05 10:40:32 -0500825
beepsf079cfb2013-09-18 17:49:51 -0700826 install_timer_key = ('servo_install.install_timeout_%s'
827 % install_timeout)
828 timer = stats.Timer(install_timer_key)
829 timer.start()
830 logging.info('Installing image through chromeos-install.')
831 self.run('chromeos-install --yes', timeout=install_timeout)
832 timer.stop()
833
834 logging.info('Power cycling DUT through servo.')
Richard Barnette03a0c132012-11-05 12:40:35 -0800835 self.servo.power_long_press()
Fang Dengafb88142013-05-30 17:44:31 -0700836 self.servo.switch_usbkey('off')
J. Richard Barnettefbcc7122013-07-24 18:24:59 -0700837 # We *must* use power_on() here; on Parrot it's how we get
838 # out of recovery mode.
839 self.servo.get_power_state_controller().power_on()
beepsf079cfb2013-09-18 17:49:51 -0700840
841 logging.info('Waiting for DUT to come back up.')
Richard Barnette03a0c132012-11-05 12:40:35 -0800842 if not self.wait_up(timeout=self.BOOT_TIMEOUT):
843 raise error.AutoservError('DUT failed to reboot installed '
844 'test image after %d seconds' %
Scott Zawalski62bacae2013-03-05 10:40:32 -0500845 self.BOOT_TIMEOUT)
846
847
J. Richard Barnettee4af8b92013-05-01 13:16:12 -0700848 def _servo_repair_reinstall(self):
Scott Zawalski62bacae2013-03-05 10:40:32 -0500849 """Reinstall the DUT utilizing servo and a test image.
850
851 Re-install the OS on the DUT by:
852 1) installing a test image on a USB storage device attached to the Servo
853 board,
854 2) booting that image in recovery mode, and then
855 3) installing the image with chromeos-install.
856
Scott Zawalski62bacae2013-03-05 10:40:32 -0500857 @raises AutoservRepairMethodNA if the device does not have servo
858 support.
859
860 """
861 if not self.servo:
862 raise error.AutoservRepairMethodNA('Repair Reinstall NA: '
863 'DUT has no servo support.')
864
865 logging.info('Attempting to recovery servo enabled device with '
866 'servo_repair_reinstall')
867
J. Richard Barnettee4af8b92013-05-01 13:16:12 -0700868 image_url = self.stage_image_for_servo()
Scott Zawalski62bacae2013-03-05 10:40:32 -0500869 self.servo_install(image_url)
870
871
872 def _servo_repair_power(self):
873 """Attempt to repair DUT using an attached Servo.
874
875 Attempt to power on the DUT via power_long_press.
876
877 @raises AutoservRepairMethodNA if the device does not have servo
878 support.
879 @raises AutoservRepairFailure if the repair fails for any reason.
880 """
881 if not self.servo:
882 raise error.AutoservRepairMethodNA('Repair Power NA: '
883 'DUT has no servo support.')
884
885 logging.info('Attempting to recover servo enabled device by '
886 'powering it off and on.')
887 self.servo.get_power_state_controller().power_off()
888 self.servo.get_power_state_controller().power_on()
889 if self.wait_up(self.BOOT_TIMEOUT):
890 return
891
892 raise error.AutoservRepairFailure('DUT did not boot after long_press.')
Richard Barnette03a0c132012-11-05 12:40:35 -0800893
894
Richard Barnette82c35912012-11-20 10:09:10 -0800895 def _powercycle_to_repair(self):
896 """Utilize the RPM Infrastructure to bring the host back up.
897
898 If the host is not up/repaired after the first powercycle we utilize
899 auto fallback to the last good install by powercycling and rebooting the
900 host 6 times.
Scott Zawalski62bacae2013-03-05 10:40:32 -0500901
902 @raises AutoservRepairMethodNA if the device does not support remote
903 power.
904 @raises AutoservRepairFailure if the repair fails for any reason.
905
Richard Barnette82c35912012-11-20 10:09:10 -0800906 """
Scott Zawalski62bacae2013-03-05 10:40:32 -0500907 if not self.has_power():
908 raise error.AutoservRepairMethodNA('Device does not support power.')
909
Richard Barnette82c35912012-11-20 10:09:10 -0800910 logging.info('Attempting repair via RPM powercycle.')
911 failed_cycles = 0
912 self.power_cycle()
913 while not self.wait_up(timeout=self.BOOT_TIMEOUT):
914 failed_cycles += 1
915 if failed_cycles >= self._MAX_POWER_CYCLE_ATTEMPTS:
Scott Zawalski62bacae2013-03-05 10:40:32 -0500916 raise error.AutoservRepairFailure(
917 'Powercycled host %s %d times; device did not come back'
918 ' online.' % (self.hostname, failed_cycles))
Richard Barnette82c35912012-11-20 10:09:10 -0800919 self.power_cycle()
920 if failed_cycles == 0:
921 logging.info('Powercycling was successful first time.')
922 else:
923 logging.info('Powercycling was successful after %d failures.',
924 failed_cycles)
925
926
Prashanth B4d8184f2014-05-05 12:22:02 -0700927 def check_device(self):
928 """Check if a device is ssh-able, and if so, clean and verify it.
929
930 @raise AutoservSSHTimeout: If the ssh ping times out.
931 @raise AutoservSshPermissionDeniedError: If ssh ping fails due to
932 permissions.
933 @raise AutoservSshPingHostError: For other AutoservRunErrors during
934 ssh_ping.
935 @raises AutoservError: As appropriate, during cleanup and verify.
936 """
937 self.ssh_ping()
938 self.cleanup()
939 self.verify()
940
941
Richard Barnette82c35912012-11-20 10:09:10 -0800942 def repair_full(self):
943 """Repair a host for repair level NO_PROTECTION.
944
945 This overrides the base class function for repair; it does
946 not call back to the parent class, but instead offers a
947 simplified implementation based on the capabilities in the
948 Chrome OS test lab.
949
Fang Deng5d518f42013-08-02 14:04:32 -0700950 It first verifies and repairs servo if it is a DUT in CrOS
Fang Deng03590af2013-10-07 17:34:20 -0700951 lab and a servo is attached.
Fang Deng5d518f42013-08-02 14:04:32 -0700952
Prashanth B4d8184f2014-05-05 12:22:02 -0700953 If `self.check_device()` fails, the following procedures are
J. Richard Barnettefde55fc2013-03-15 17:47:01 -0700954 attempted:
955 1. Try to re-install to a known stable image using
956 auto-update.
Scott Zawalski62bacae2013-03-05 10:40:32 -0500957 2. If there's a servo for the DUT, try to power the DUT off and
958 on.
959 3. If there's a servo for the DUT, try to re-install via
J. Richard Barnettefde55fc2013-03-15 17:47:01 -0700960 the servo.
Scott Zawalski62bacae2013-03-05 10:40:32 -0500961 4. If the DUT can be power-cycled via RPM, try to repair
Richard Barnette82c35912012-11-20 10:09:10 -0800962 by power-cycling.
963
964 As with the parent method, the last operation performed on
Prashanth B4d8184f2014-05-05 12:22:02 -0700965 the DUT must be to call `self.check_device()`; If that call fails the
966 exception it raises is passed back to the caller.
J. Richard Barnettefde55fc2013-03-15 17:47:01 -0700967
Scott Zawalski62bacae2013-03-05 10:40:32 -0500968 @raises AutoservRepairTotalFailure if the repair process fails to
969 fix the DUT.
Fang Deng5d518f42013-08-02 14:04:32 -0700970 @raises ServoHostRepairTotalFailure if the repair process fails to
971 fix the servo host if one is attached to the DUT.
972 @raises AutoservSshPermissionDeniedError if it is unable
973 to ssh to the servo host due to permission error.
974
Richard Barnette82c35912012-11-20 10:09:10 -0800975 """
Dan Shi4d478522014-02-14 13:46:32 -0800976 if self._servo_host and not self.servo:
Fang Deng03590af2013-10-07 17:34:20 -0700977 try:
Dan Shi4d478522014-02-14 13:46:32 -0800978 self._servo_host.repair_full()
Fang Deng03590af2013-10-07 17:34:20 -0700979 except Exception as e:
Fang Deng03590af2013-10-07 17:34:20 -0700980 logging.error('Could not create a healthy servo: %s', e)
Dan Shi4d478522014-02-14 13:46:32 -0800981 self.servo = self._servo_host.get_servo()
Fang Deng5d518f42013-08-02 14:04:32 -0700982
Scott Zawalski62bacae2013-03-05 10:40:32 -0500983 # TODO(scottz): This should use something similar to label_decorator,
984 # but needs to be populated in order so DUTs are repaired with the
985 # least amount of effort.
Dan Shi849a1c42014-03-05 11:10:43 -0800986 repair_funcs = [self._servo_repair_power,
987 self._install_repair,
Dan Shi2c88eed2013-11-12 10:18:38 -0800988 self._install_repair_with_powerwash,
J. Richard Barnettee4af8b92013-05-01 13:16:12 -0700989 self._servo_repair_reinstall,
Scott Zawalski62bacae2013-03-05 10:40:32 -0500990 self._powercycle_to_repair]
991 errors = []
Simran Basie6130932013-10-01 14:07:52 -0700992 board = self._get_board_from_afe()
Scott Zawalski62bacae2013-03-05 10:40:32 -0500993 for repair_func in repair_funcs:
994 try:
995 repair_func()
Prashanth B4d8184f2014-05-05 12:22:02 -0700996 self.check_device()
Simran Basie6130932013-10-01 14:07:52 -0700997 stats.Counter(
998 '%s.SUCCEEDED' % repair_func.__name__).increment()
999 if board:
1000 stats.Counter(
Dan Shib87c3aa2014-02-12 15:40:31 -08001001 '%s.%s.SUCCEEDED' % (repair_func.__name__,
Simran Basie6130932013-10-01 14:07:52 -07001002 board)).increment()
Scott Zawalski62bacae2013-03-05 10:40:32 -05001003 return
Simran Basie6130932013-10-01 14:07:52 -07001004 except error.AutoservRepairMethodNA as e:
1005 stats.Counter(
1006 '%s.RepairNA' % repair_func.__name__).increment()
1007 if board:
1008 stats.Counter(
Dan Shib87c3aa2014-02-12 15:40:31 -08001009 '%s.%s.RepairNA' % (repair_func.__name__,
Simran Basie6130932013-10-01 14:07:52 -07001010 board)).increment()
Ilja H. Friedel04be2bd2014-05-07 21:29:59 -07001011 logging.warning('Repair function NA: %s', e)
Simran Basie6130932013-10-01 14:07:52 -07001012 errors.append(str(e))
Scott Zawalski62bacae2013-03-05 10:40:32 -05001013 except Exception as e:
Simran Basie6130932013-10-01 14:07:52 -07001014 stats.Counter(
1015 '%s.FAILED' % repair_func.__name__).increment()
1016 if board:
1017 stats.Counter(
Dan Shib87c3aa2014-02-12 15:40:31 -08001018 '%s.%s.FAILED' % (repair_func.__name__,
Simran Basie6130932013-10-01 14:07:52 -07001019 board)).increment()
Ilja H. Friedel04be2bd2014-05-07 21:29:59 -07001020 logging.warning('Failed to repair device: %s', e)
Scott Zawalski62bacae2013-03-05 10:40:32 -05001021 errors.append(str(e))
Scott Zawalski89c44dd2013-02-26 09:28:02 -05001022
Simran Basie6130932013-10-01 14:07:52 -07001023 stats.Counter('Full_Repair_Failed').increment()
1024 if board:
1025 stats.Counter(
1026 'Full_Repair_Failed.%s' % board).increment()
Scott Zawalski62bacae2013-03-05 10:40:32 -05001027 raise error.AutoservRepairTotalFailure(
1028 'All attempts at repairing the device failed:\n%s' %
1029 '\n'.join(errors))
Richard Barnette82c35912012-11-20 10:09:10 -08001030
1031
J. Richard Barnette1d78b012012-05-15 13:56:30 -07001032 def close(self):
beeps32a63082013-08-22 14:02:29 -07001033 self.rpc_disconnect_all()
Fang Deng0ca40e22013-08-27 17:47:44 -07001034 super(CrosHost, self).close()
J. Richard Barnette1d78b012012-05-15 13:56:30 -07001035
1036
Simran Basi5e6339a2013-03-21 11:34:32 -07001037 def _cleanup_poweron(self):
1038 """Special cleanup method to make sure hosts always get power back."""
1039 afe = frontend_wrappers.RetryingAFE(timeout_min=5, delay_sec=10)
1040 hosts = afe.get_hosts(hostname=self.hostname)
1041 if not hosts or not (self._RPM_OUTLET_CHANGED in
1042 hosts[0].attributes):
1043 return
1044 logging.debug('This host has recently interacted with the RPM'
1045 ' Infrastructure. Ensuring power is on.')
1046 try:
1047 self.power_on()
1048 except rpm_client.RemotePowerException:
1049 # If cleanup has completed but there was an issue with the RPM
1050 # Infrastructure, log an error message rather than fail cleanup
1051 logging.error('Failed to turn Power On for this host after '
1052 'cleanup through the RPM Infrastructure.')
1053 afe.set_host_attribute(self._RPM_OUTLET_CHANGED, None,
1054 hostname=self.hostname)
1055
1056
beepsc87ff602013-07-31 21:53:00 -07001057 def _is_factory_image(self):
1058 """Checks if the image on the DUT is a factory image.
1059
1060 @return: True if the image on the DUT is a factory image.
1061 False otherwise.
1062 """
1063 result = self.run('[ -f /root/.factory_test ]', ignore_status=True)
1064 return result.exit_status == 0
1065
1066
1067 def _restart_ui(self):
J. Richard Barnette84890bd2014-02-21 11:05:47 -08001068 """Restart the Chrome UI.
beepsc87ff602013-07-31 21:53:00 -07001069
1070 @raises: FactoryImageCheckerException for factory images, since
1071 we cannot attempt to restart ui on them.
1072 error.AutoservRunError for any other type of error that
1073 occurs while restarting ui.
1074 """
1075 if self._is_factory_image():
1076 raise FactoryImageCheckerException('Cannot restart ui on factory '
1077 'images')
1078
J. Richard Barnette84890bd2014-02-21 11:05:47 -08001079 # TODO(jrbarnette): The command to stop/start the ui job
1080 # should live inside cros_ui, too. However that would seem
1081 # to imply interface changes to the existing start()/restart()
1082 # functions, which is a bridge too far (for now).
1083 prompt = cros_ui.get_login_prompt_state(self)
1084 self.run('stop ui; start ui')
1085 cros_ui.wait_for_chrome_ready(prompt, self)
beepsc87ff602013-07-31 21:53:00 -07001086
1087
1088 def cleanup(self):
Richard Barnette82c35912012-11-20 10:09:10 -08001089 self.run('rm -f %s' % constants.CLEANUP_LOGS_PAUSED_FILE)
Scott Zawalskiddbc31e2012-11-15 11:29:01 -05001090 try:
beepsc87ff602013-07-31 21:53:00 -07001091 self._restart_ui()
1092 except (error.AutotestRunError, error.AutoservRunError,
1093 FactoryImageCheckerException):
Ilja H. Friedel04be2bd2014-05-07 21:29:59 -07001094 logging.warning('Unable to restart ui, rebooting device.')
Scott Zawalskiddbc31e2012-11-15 11:29:01 -05001095 # Since restarting the UI fails fall back to normal Autotest
1096 # cleanup routines, i.e. reboot the machine.
Fang Deng0ca40e22013-08-27 17:47:44 -07001097 super(CrosHost, self).cleanup()
Simran Basi5e6339a2013-03-21 11:34:32 -07001098 # Check if the rpm outlet was manipulated.
Simran Basid5e5e272012-09-24 15:23:59 -07001099 if self.has_power():
Simran Basi5e6339a2013-03-21 11:34:32 -07001100 self._cleanup_poweron()
J. Richard Barnette45e93de2012-04-11 17:24:15 -07001101
1102
Yu-Ju Honga2be94a2012-07-31 09:48:52 -07001103 def reboot(self, **dargs):
1104 """
1105 This function reboots the site host. The more generic
1106 RemoteHost.reboot() performs sync and sleeps for 5
1107 seconds. This is not necessary for Chrome OS devices as the
1108 sync should be finished in a short time during the reboot
1109 command.
1110 """
Tom Wai-Hong Tamf5cd1d42012-08-13 12:04:08 +08001111 if 'reboot_cmd' not in dargs:
Doug Anderson7d5aeb22014-02-27 15:12:17 -08001112 reboot_timeout = dargs.get('reboot_timeout', 10)
1113 dargs['reboot_cmd'] = ('((reboot & sleep %d; reboot -f &)'
1114 ' </dev/null >/dev/null 2>&1 &)' %
1115 reboot_timeout)
Yu-Ju Honga2be94a2012-07-31 09:48:52 -07001116 # Enable fastsync to avoid running extra sync commands before reboot.
Tom Wai-Hong Tamf5cd1d42012-08-13 12:04:08 +08001117 if 'fastsync' not in dargs:
1118 dargs['fastsync'] = True
Fang Deng0ca40e22013-08-27 17:47:44 -07001119 super(CrosHost, self).reboot(**dargs)
Yu-Ju Honga2be94a2012-07-31 09:48:52 -07001120
1121
J. Richard Barnette45e93de2012-04-11 17:24:15 -07001122 def verify_software(self):
Richard Barnetteb2bc13c2013-01-08 17:32:51 -08001123 """Verify working software on a Chrome OS system.
J. Richard Barnette45e93de2012-04-11 17:24:15 -07001124
Richard Barnetteb2bc13c2013-01-08 17:32:51 -08001125 Tests for the following conditions:
1126 1. All conditions tested by the parent version of this
1127 function.
1128 2. Sufficient space in /mnt/stateful_partition.
Fang Deng6b05f5b2013-03-20 13:42:11 -07001129 3. Sufficient space in /mnt/stateful_partition/encrypted.
1130 4. update_engine answers a simple status request over DBus.
J. Richard Barnette45e93de2012-04-11 17:24:15 -07001131
J. Richard Barnette45e93de2012-04-11 17:24:15 -07001132 """
Fang Deng0ca40e22013-08-27 17:47:44 -07001133 super(CrosHost, self).verify_software()
J. Richard Barnette45e93de2012-04-11 17:24:15 -07001134 self.check_diskspace(
1135 '/mnt/stateful_partition',
1136 global_config.global_config.get_config_value(
Fang Deng6b05f5b2013-03-20 13:42:11 -07001137 'SERVER', 'gb_diskspace_required', type=float,
1138 default=20.0))
1139 self.check_diskspace(
1140 '/mnt/stateful_partition/encrypted',
1141 global_config.global_config.get_config_value(
1142 'SERVER', 'gb_encrypted_diskspace_required', type=float,
1143 default=0.1))
beepsc87ff602013-07-31 21:53:00 -07001144
Prashanth B5d0a0512014-04-25 12:26:08 -07001145 services_status = self.run('status system-services').stdout
1146 if services_status != 'system-services start/running\n':
1147 raise error.AutoservError('Chrome failed to reach login. '
1148 'System services not running.')
1149
beepsc87ff602013-07-31 21:53:00 -07001150 # Factory images don't run update engine,
1151 # goofy controls dbus on these DUTs.
1152 if not self._is_factory_image():
1153 self.run('update_engine_client --status')
Scott Zawalskifbca4a92013-03-04 15:56:42 -05001154 # Makes sure python is present, loads and can use built in functions.
1155 # We have seen cases where importing cPickle fails with undefined
1156 # symbols in cPickle.so.
1157 self.run('python -c "import cPickle"')
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001158
1159
Fang Deng96667ca2013-08-01 17:46:18 -07001160 def make_ssh_command(self, user='root', port=22, opts='', hosts_file=None,
1161 connect_timeout=None, alive_interval=None):
1162 """Override default make_ssh_command to use options tuned for Chrome OS.
1163
1164 Tuning changes:
1165 - ConnectTimeout=30; maximum of 30 seconds allowed for an SSH
1166 connection failure. Consistency with remote_access.sh.
1167
1168 - ServerAliveInterval=180; which causes SSH to ping connection every
1169 180 seconds. In conjunction with ServerAliveCountMax ensures
1170 that if the connection dies, Autotest will bail out quickly.
1171 Originally tried 60 secs, but saw frequent job ABORTS where
1172 the test completed successfully.
1173
1174 - ServerAliveCountMax=3; consistency with remote_access.sh.
1175
1176 - ConnectAttempts=4; reduce flakiness in connection errors;
1177 consistency with remote_access.sh.
1178
1179 - UserKnownHostsFile=/dev/null; we don't care about the keys.
1180 Host keys change with every new installation, don't waste
1181 memory/space saving them.
1182
1183 - SSH protocol forced to 2; needed for ServerAliveInterval.
1184
1185 @param user User name to use for the ssh connection.
1186 @param port Port on the target host to use for ssh connection.
1187 @param opts Additional options to the ssh command.
1188 @param hosts_file Ignored.
1189 @param connect_timeout Ignored.
1190 @param alive_interval Ignored.
1191 """
Aviv Keshetc5947fa2013-09-04 14:06:29 -07001192 base_command = ('/usr/bin/ssh -a -x %s %s %s'
1193 ' -o StrictHostKeyChecking=no'
Fang Deng96667ca2013-08-01 17:46:18 -07001194 ' -o UserKnownHostsFile=/dev/null -o BatchMode=yes'
1195 ' -o ConnectTimeout=30 -o ServerAliveInterval=180'
1196 ' -o ServerAliveCountMax=3 -o ConnectionAttempts=4'
1197 ' -o Protocol=2 -l %s -p %d')
Aviv Keshetc5947fa2013-09-04 14:06:29 -07001198 return base_command % (self._ssh_verbosity_flag, self._ssh_options,
1199 opts, user, port)
Fang Deng96667ca2013-08-01 17:46:18 -07001200
1201
beeps32a63082013-08-22 14:02:29 -07001202 def _create_ssh_tunnel(self, port, local_port):
1203 """Create an ssh tunnel from local_port to port.
1204
1205 @param port: remote port on the host.
1206 @param local_port: local forwarding port.
1207
1208 @return: the tunnel process.
1209 """
1210 # Chrome OS on the target closes down most external ports
1211 # for security. We could open the port, but doing that
1212 # would conflict with security tests that check that only
1213 # expected ports are open. So, to get to the port on the
1214 # target we use an ssh tunnel.
1215 tunnel_options = '-n -N -q -L %d:localhost:%d' % (local_port, port)
1216 ssh_cmd = self.make_ssh_command(opts=tunnel_options)
1217 tunnel_cmd = '%s %s' % (ssh_cmd, self.hostname)
1218 logging.debug('Full tunnel command: %s', tunnel_cmd)
1219 tunnel_proc = subprocess.Popen(tunnel_cmd, shell=True, close_fds=True)
1220 logging.debug('Started ssh tunnel, local = %d'
1221 ' remote = %d, pid = %d',
1222 local_port, port, tunnel_proc.pid)
1223 return tunnel_proc
1224
1225
Christopher Wileydd181852013-10-10 19:56:58 -07001226 def _setup_rpc(self, port, command_name, remote_pid=None):
beeps32a63082013-08-22 14:02:29 -07001227 """Sets up a tunnel process and performs rpc connection book keeping.
1228
1229 This method assumes that xmlrpc and jsonrpc never conflict, since
1230 we can only either have an xmlrpc or a jsonrpc server listening on
1231 a remote port. As such, it enforces a single proxy->remote port
1232 policy, i.e if one starts a jsonrpc proxy/server from port A->B,
1233 and then tries to start an xmlrpc proxy forwarded to the same port,
1234 the xmlrpc proxy will override the jsonrpc tunnel process, however:
1235
1236 1. None of the methods on the xmlrpc proxy will work because
1237 the server listening on B is jsonrpc.
1238
1239 2. The xmlrpc client cannot initiate a termination of the JsonRPC
1240 server, as the only use case currently is goofy, which is tied to
1241 the factory image. It is much easier to handle a failed xmlrpc
1242 call on the client than it is to terminate goofy in this scenario,
1243 as doing the latter might leave the DUT in a hard to recover state.
1244
1245 With the current implementation newer rpc proxy connections will
1246 terminate the tunnel processes of older rpc connections tunneling
1247 to the same remote port. If methods are invoked on the client
1248 after this has happened they will fail with connection closed errors.
1249
1250 @param port: The remote forwarding port.
1251 @param command_name: The name of the remote process, to terminate
1252 using pkill.
1253
1254 @return A url that we can use to initiate the rpc connection.
1255 """
1256 self.rpc_disconnect(port)
1257 local_port = utils.get_unused_port()
1258 tunnel_proc = self._create_ssh_tunnel(port, local_port)
Christopher Wileydd181852013-10-10 19:56:58 -07001259 self._rpc_proxy_map[port] = (command_name, tunnel_proc, remote_pid)
beeps32a63082013-08-22 14:02:29 -07001260 return self._RPC_PROXY_URL % local_port
1261
1262
Christopher Wileyd78249a2013-03-01 13:05:31 -08001263 def xmlrpc_connect(self, command, port, command_name=None,
Yusuf Mohsinallyfff89d62013-11-18 16:34:07 -08001264 ready_test_name=None, timeout_seconds=10,
1265 logfile='/dev/null'):
J. Richard Barnette1d78b012012-05-15 13:56:30 -07001266 """Connect to an XMLRPC server on the host.
1267
1268 The `command` argument should be a simple shell command that
1269 starts an XMLRPC server on the given `port`. The command
1270 must not daemonize, and must terminate cleanly on SIGTERM.
1271 The command is started in the background on the host, and a
1272 local XMLRPC client for the server is created and returned
1273 to the caller.
1274
1275 Note that the process of creating an XMLRPC client makes no
1276 attempt to connect to the remote server; the caller is
1277 responsible for determining whether the server is running
1278 correctly, and is ready to serve requests.
1279
Christopher Wileyd78249a2013-03-01 13:05:31 -08001280 Optionally, the caller can pass ready_test_name, a string
1281 containing the name of a method to call on the proxy. This
1282 method should take no parameters and return successfully only
1283 when the server is ready to process client requests. When
1284 ready_test_name is set, xmlrpc_connect will block until the
1285 proxy is ready, and throw a TestError if the server isn't
1286 ready by timeout_seconds.
1287
beeps32a63082013-08-22 14:02:29 -07001288 If a server is already running on the remote port, this
1289 method will kill it and disconnect the tunnel process
1290 associated with the connection before establishing a new one,
1291 by consulting the rpc_proxy_map in rpc_disconnect.
1292
J. Richard Barnette1d78b012012-05-15 13:56:30 -07001293 @param command Shell command to start the server.
1294 @param port Port number on which the server is expected to
1295 be serving.
J. Richard Barnette7214e0b2013-02-06 15:20:49 -08001296 @param command_name String to use as input to `pkill` to
1297 terminate the XMLRPC server on the host.
Christopher Wileyd78249a2013-03-01 13:05:31 -08001298 @param ready_test_name String containing the name of a
1299 method defined on the XMLRPC server.
1300 @param timeout_seconds Number of seconds to wait
1301 for the server to become 'ready.' Will throw a
1302 TestFail error if server is not ready in time.
Yusuf Mohsinallyfff89d62013-11-18 16:34:07 -08001303 @param logfile Logfile to send output when running
1304 'command' argument.
Yusuf Mohsinally8d19e3c2013-11-21 14:25:45 -08001305
J. Richard Barnette1d78b012012-05-15 13:56:30 -07001306 """
Christopher Wileyc14f06a2013-10-16 13:55:39 -07001307 # Clean up any existing state. If the caller is willing
1308 # to believe their server is down, we ought to clean up
1309 # any tunnels we might have sitting around.
1310 self.rpc_disconnect(port)
J. Richard Barnette1d78b012012-05-15 13:56:30 -07001311 # Start the server on the host. Redirection in the command
1312 # below is necessary, because 'ssh' won't terminate until
1313 # background child processes close stdin, stdout, and
1314 # stderr.
Yusuf Mohsinallyfff89d62013-11-18 16:34:07 -08001315 remote_cmd = '%s </dev/null >%s 2>&1 & echo $!' % (command, logfile)
Christopher Wileydd181852013-10-10 19:56:58 -07001316 remote_pid = self.run(remote_cmd).stdout.rstrip('\n')
J. Richard Barnette1d78b012012-05-15 13:56:30 -07001317 logging.debug('Started XMLRPC server on host %s, pid = %s',
1318 self.hostname, remote_pid)
1319
Christopher Wileydd181852013-10-10 19:56:58 -07001320 # Tunnel through SSH to be able to reach that remote port.
1321 rpc_url = self._setup_rpc(port, command_name, remote_pid=remote_pid)
Christopher Wileyd78249a2013-03-01 13:05:31 -08001322 proxy = xmlrpclib.ServerProxy(rpc_url, allow_none=True)
Christopher Wileydd181852013-10-10 19:56:58 -07001323
Christopher Wileyd78249a2013-03-01 13:05:31 -08001324 if ready_test_name is not None:
J. Richard Barnette13eb7c02013-03-07 12:06:29 -08001325 # retry.retry logs each attempt; calculate delay_sec to
1326 # keep log spam to a dull roar.
Christopher Wiley0ed712b2013-04-09 15:25:12 -07001327 @retry.retry((socket.error,
1328 xmlrpclib.ProtocolError,
1329 httplib.BadStatusLine),
Chris Sosa65425082013-10-16 13:26:22 -07001330 timeout_min=timeout_seconds / 60.0,
1331 delay_sec=min(max(timeout_seconds / 20.0, 0.1), 1))
Christopher Wileyd78249a2013-03-01 13:05:31 -08001332 def ready_test():
1333 """ Call proxy.ready_test_name(). """
1334 getattr(proxy, ready_test_name)()
1335 successful = False
1336 try:
1337 logging.info('Waiting %d seconds for XMLRPC server '
1338 'to start.', timeout_seconds)
1339 ready_test()
1340 successful = True
Christopher Wileyd78249a2013-03-01 13:05:31 -08001341 finally:
1342 if not successful:
1343 logging.error('Failed to start XMLRPC server.')
beeps32a63082013-08-22 14:02:29 -07001344 self.rpc_disconnect(port)
Christopher Wileyd78249a2013-03-01 13:05:31 -08001345 logging.info('XMLRPC server started successfully.')
1346 return proxy
J. Richard Barnette1d78b012012-05-15 13:56:30 -07001347
J. Richard Barnette1d78b012012-05-15 13:56:30 -07001348
Jason Abeleb6f924f2013-11-13 16:01:54 -08001349 def syslog(self, message, tag='autotest'):
1350 """Logs a message to syslog on host.
1351
1352 @param message String message to log into syslog
1353 @param tag String tag prefix for syslog
1354
1355 """
1356 self.run('logger -t "%s" "%s"' % (tag, message))
1357
1358
beeps32a63082013-08-22 14:02:29 -07001359 def jsonrpc_connect(self, port):
1360 """Creates a jsonrpc proxy connection through an ssh tunnel.
1361
1362 This method exists to facilitate communication with goofy (which is
1363 the default system manager on all factory images) and as such, leaves
1364 most of the rpc server sanity checking to the caller. Unlike
1365 xmlrpc_connect, this method does not facilitate the creation of a remote
1366 jsonrpc server, as the only clients of this code are factory tests,
1367 for which the goofy system manager is built in to the image and starts
1368 when the target boots.
1369
1370 One can theoretically create multiple jsonrpc proxies all forwarded
1371 to the same remote port, provided the remote port has an rpc server
1372 listening. However, in doing so we stand the risk of leaking an
1373 existing tunnel process, so we always disconnect any older tunnels
1374 we might have through rpc_disconnect.
1375
1376 @param port: port on the remote host that is serving this proxy.
1377
1378 @return: The client proxy.
1379 """
1380 if not jsonrpclib:
1381 logging.warning('Jsonrpclib could not be imported. Check that '
1382 'site-packages contains jsonrpclib.')
1383 return None
1384
1385 proxy = jsonrpclib.jsonrpc.ServerProxy(self._setup_rpc(port, None))
1386
1387 logging.info('Established a jsonrpc connection through port %s.', port)
1388 return proxy
1389
1390
1391 def rpc_disconnect(self, port):
1392 """Disconnect from an RPC server on the host.
1393
1394 Terminates the remote RPC server previously started for
J. Richard Barnette1d78b012012-05-15 13:56:30 -07001395 the given `port`. Also closes the local ssh tunnel created
1396 for the connection to the host. This function does not
beeps32a63082013-08-22 14:02:29 -07001397 directly alter the state of a previously returned RPC
J. Richard Barnette1d78b012012-05-15 13:56:30 -07001398 client object; however disconnection will cause all
1399 subsequent calls to methods on the object to fail.
1400
1401 This function does nothing if requested to disconnect a port
beeps32a63082013-08-22 14:02:29 -07001402 that was not previously connected via _setup_rpc.
J. Richard Barnette1d78b012012-05-15 13:56:30 -07001403
1404 @param port Port number passed to a previous call to
beeps32a63082013-08-22 14:02:29 -07001405 `_setup_rpc()`.
J. Richard Barnette1d78b012012-05-15 13:56:30 -07001406 """
beeps32a63082013-08-22 14:02:29 -07001407 if port not in self._rpc_proxy_map:
J. Richard Barnette1d78b012012-05-15 13:56:30 -07001408 return
Christopher Wileydd181852013-10-10 19:56:58 -07001409 remote_name, tunnel_proc, remote_pid = self._rpc_proxy_map[port]
J. Richard Barnette1d78b012012-05-15 13:56:30 -07001410 if remote_name:
1411 # We use 'pkill' to find our target process rather than
1412 # a PID, because the host may have rebooted since
1413 # connecting, and we don't want to kill an innocent
1414 # process with the same PID.
1415 #
1416 # 'pkill' helpfully exits with status 1 if no target
1417 # process is found, for which run() will throw an
Simran Basid5e5e272012-09-24 15:23:59 -07001418 # exception. We don't want that, so we the ignore
J. Richard Barnette1d78b012012-05-15 13:56:30 -07001419 # status.
1420 self.run("pkill -f '%s'" % remote_name, ignore_status=True)
Christopher Wileydd181852013-10-10 19:56:58 -07001421 if remote_pid:
1422 logging.info('Waiting for RPC server "%s" shutdown',
1423 remote_name)
1424 start_time = time.time()
1425 while (time.time() - start_time <
1426 self._RPC_SHUTDOWN_TIMEOUT_SECONDS):
1427 running_processes = self.run(
1428 "pgrep -f '%s'" % remote_name,
1429 ignore_status=True).stdout.split()
1430 if not remote_pid in running_processes:
1431 logging.info('Shut down RPC server.')
1432 break
1433 time.sleep(self._RPC_SHUTDOWN_POLLING_PERIOD_SECONDS)
1434 else:
1435 raise error.TestError('Failed to shutdown RPC server %s' %
1436 remote_name)
J. Richard Barnette1d78b012012-05-15 13:56:30 -07001437
1438 if tunnel_proc.poll() is None:
1439 tunnel_proc.terminate()
1440 logging.debug('Terminated tunnel, pid %d', tunnel_proc.pid)
1441 else:
1442 logging.debug('Tunnel pid %d terminated early, status %d',
1443 tunnel_proc.pid, tunnel_proc.returncode)
beeps32a63082013-08-22 14:02:29 -07001444 del self._rpc_proxy_map[port]
J. Richard Barnette1d78b012012-05-15 13:56:30 -07001445
1446
beeps32a63082013-08-22 14:02:29 -07001447 def rpc_disconnect_all(self):
1448 """Disconnect all known RPC proxy ports."""
1449 for port in self._rpc_proxy_map.keys():
1450 self.rpc_disconnect(port)
J. Richard Barnette1d78b012012-05-15 13:56:30 -07001451
1452
J. Richard Barnetteb6de7e32013-02-14 13:28:04 -08001453 def _ping_check_status(self, status):
1454 """Ping the host once, and return whether it has a given status.
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001455
J. Richard Barnetteb6de7e32013-02-14 13:28:04 -08001456 @param status Check the ping status against this value.
1457 @return True iff `status` and the result of ping are the same
1458 (i.e. both True or both False).
1459
1460 """
1461 ping_val = utils.ping(self.hostname, tries=1, deadline=1)
1462 return not (status ^ (ping_val == 0))
1463
1464 def _ping_wait_for_status(self, status, timeout):
1465 """Wait for the host to have a given status (UP or DOWN).
1466
1467 Status is checked by polling. Polling will not last longer
1468 than the number of seconds in `timeout`. The polling
1469 interval will be long enough that only approximately
1470 _PING_WAIT_COUNT polling cycles will be executed, subject
1471 to a maximum interval of about one minute.
1472
1473 @param status Waiting will stop immediately if `ping` of the
1474 host returns this status.
1475 @param timeout Poll for at most this many seconds.
1476 @return True iff the host status from `ping` matched the
1477 requested status at the time of return.
1478
1479 """
1480 # _ping_check_status() takes about 1 second, hence the
1481 # "- 1" in the formula below.
1482 poll_interval = min(int(timeout / self._PING_WAIT_COUNT), 60) - 1
1483 end_time = time.time() + timeout
1484 while time.time() <= end_time:
1485 if self._ping_check_status(status):
1486 return True
1487 if poll_interval > 0:
1488 time.sleep(poll_interval)
1489
1490 # The last thing we did was sleep(poll_interval), so it may
1491 # have been too long since the last `ping`. Check one more
1492 # time, just to be sure.
1493 return self._ping_check_status(status)
1494
1495 def ping_wait_up(self, timeout):
1496 """Wait for the host to respond to `ping`.
1497
1498 N.B. This method is not a reliable substitute for
1499 `wait_up()`, because a host that responds to ping will not
1500 necessarily respond to ssh. This method should only be used
1501 if the target DUT can be considered functional even if it
1502 can't be reached via ssh.
1503
1504 @param timeout Minimum time to allow before declaring the
1505 host to be non-responsive.
1506 @return True iff the host answered to ping before the timeout.
1507
1508 """
1509 return self._ping_wait_for_status(self._PING_STATUS_UP, timeout)
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001510
Andrew Bresticker678c0c72013-01-22 10:44:09 -08001511 def ping_wait_down(self, timeout):
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001512 """Wait until the host no longer responds to `ping`.
1513
J. Richard Barnetteb6de7e32013-02-14 13:28:04 -08001514 This function can be used as a slightly faster version of
1515 `wait_down()`, by avoiding potentially long ssh timeouts.
1516
1517 @param timeout Minimum time to allow for the host to become
1518 non-responsive.
1519 @return True iff the host quit answering ping before the
1520 timeout.
1521
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001522 """
J. Richard Barnetteb6de7e32013-02-14 13:28:04 -08001523 return self._ping_wait_for_status(self._PING_STATUS_DOWN, timeout)
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001524
Tom Wai-Hong Tamfced4f62014-04-17 10:56:30 +08001525 def test_wait_for_sleep(self, sleep_timeout=None):
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001526 """Wait for the client to enter low-power sleep mode.
1527
1528 The test for "is asleep" can't distinguish a system that is
1529 powered off; to confirm that the unit was asleep, it is
1530 necessary to force resume, and then call
1531 `test_wait_for_resume()`.
1532
1533 This function is expected to be called from a test as part
1534 of a sequence like the following:
1535
1536 ~~~~~~~~
1537 boot_id = host.get_boot_id()
1538 # trigger sleep on the host
1539 host.test_wait_for_sleep()
1540 # trigger resume on the host
1541 host.test_wait_for_resume(boot_id)
1542 ~~~~~~~~
1543
Tom Wai-Hong Tamfced4f62014-04-17 10:56:30 +08001544 @param sleep_timeout time limit in seconds to allow the host sleep.
1545
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001546 @exception TestFail The host did not go to sleep within
1547 the allowed time.
1548 """
Tom Wai-Hong Tamfced4f62014-04-17 10:56:30 +08001549 if sleep_timeout is None:
1550 sleep_timeout = self.SLEEP_TIMEOUT
1551
1552 if not self.ping_wait_down(timeout=sleep_timeout):
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001553 raise error.TestFail(
Tom Wai-Hong Tamfced4f62014-04-17 10:56:30 +08001554 'client failed to sleep after %d seconds' % sleep_timeout)
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001555
1556
Tom Wai-Hong Tamfced4f62014-04-17 10:56:30 +08001557 def test_wait_for_resume(self, old_boot_id, resume_timeout=None):
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001558 """Wait for the client to resume from low-power sleep mode.
1559
1560 The `old_boot_id` parameter should be the value from
1561 `get_boot_id()` obtained prior to entering sleep mode. A
1562 `TestFail` exception is raised if the boot id changes.
1563
1564 See @ref test_wait_for_sleep for more on this function's
1565 usage.
1566
J. Richard Barnette7214e0b2013-02-06 15:20:49 -08001567 @param old_boot_id A boot id value obtained before the
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001568 target host went to sleep.
Tom Wai-Hong Tamfced4f62014-04-17 10:56:30 +08001569 @param resume_timeout time limit in seconds to allow the host up.
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001570
1571 @exception TestFail The host did not respond within the
1572 allowed time.
1573 @exception TestFail The host responded, but the boot id test
1574 indicated a reboot rather than a sleep
1575 cycle.
1576 """
Tom Wai-Hong Tamfced4f62014-04-17 10:56:30 +08001577 if resume_timeout is None:
1578 resume_timeout = self.RESUME_TIMEOUT
1579
1580 if not self.wait_up(timeout=resume_timeout):
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001581 raise error.TestFail(
1582 'client failed to resume from sleep after %d seconds' %
Tom Wai-Hong Tamfced4f62014-04-17 10:56:30 +08001583 resume_timeout)
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001584 else:
1585 new_boot_id = self.get_boot_id()
1586 if new_boot_id != old_boot_id:
1587 raise error.TestFail(
1588 'client rebooted, but sleep was expected'
1589 ' (old boot %s, new boot %s)'
1590 % (old_boot_id, new_boot_id))
1591
1592
1593 def test_wait_for_shutdown(self):
1594 """Wait for the client to shut down.
1595
1596 The test for "has shut down" can't distinguish a system that
1597 is merely asleep; to confirm that the unit was down, it is
1598 necessary to force boot, and then call test_wait_for_boot().
1599
1600 This function is expected to be called from a test as part
1601 of a sequence like the following:
1602
1603 ~~~~~~~~
1604 boot_id = host.get_boot_id()
1605 # trigger shutdown on the host
1606 host.test_wait_for_shutdown()
1607 # trigger boot on the host
1608 host.test_wait_for_boot(boot_id)
1609 ~~~~~~~~
1610
1611 @exception TestFail The host did not shut down within the
1612 allowed time.
1613 """
Andrew Bresticker678c0c72013-01-22 10:44:09 -08001614 if not self.ping_wait_down(timeout=self.SHUTDOWN_TIMEOUT):
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001615 raise error.TestFail(
1616 'client failed to shut down after %d seconds' %
J. Richard Barnetteeb69d722012-06-18 17:29:44 -07001617 self.SHUTDOWN_TIMEOUT)
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001618
1619
1620 def test_wait_for_boot(self, old_boot_id=None):
1621 """Wait for the client to boot from cold power.
1622
1623 The `old_boot_id` parameter should be the value from
1624 `get_boot_id()` obtained prior to shutting down. A
1625 `TestFail` exception is raised if the boot id does not
1626 change. The boot id test is omitted if `old_boot_id` is not
1627 specified.
1628
1629 See @ref test_wait_for_shutdown for more on this function's
1630 usage.
1631
J. Richard Barnette7214e0b2013-02-06 15:20:49 -08001632 @param old_boot_id A boot id value obtained before the
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001633 shut down.
1634
1635 @exception TestFail The host did not respond within the
1636 allowed time.
1637 @exception TestFail The host responded, but the boot id test
1638 indicated that there was no reboot.
1639 """
J. Richard Barnetteeb69d722012-06-18 17:29:44 -07001640 if not self.wait_up(timeout=self.REBOOT_TIMEOUT):
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001641 raise error.TestFail(
1642 'client failed to reboot after %d seconds' %
J. Richard Barnetteeb69d722012-06-18 17:29:44 -07001643 self.REBOOT_TIMEOUT)
J. Richard Barnette134ec2c2012-04-25 12:59:37 -07001644 elif old_boot_id:
1645 if self.get_boot_id() == old_boot_id:
1646 raise error.TestFail(
1647 'client is back up, but did not reboot'
1648 ' (boot %s)' % old_boot_id)
Simran Basid5e5e272012-09-24 15:23:59 -07001649
1650
1651 @staticmethod
1652 def check_for_rpm_support(hostname):
1653 """For a given hostname, return whether or not it is powered by an RPM.
1654
Simran Basi1df55112013-09-06 11:25:09 -07001655 @param hostname: hostname to check for rpm support.
1656
Simran Basid5e5e272012-09-24 15:23:59 -07001657 @return None if this host does not follows the defined naming format
1658 for RPM powered DUT's in the lab. If it does follow the format,
1659 it returns a regular expression MatchObject instead.
1660 """
Fang Deng0ca40e22013-08-27 17:47:44 -07001661 return re.match(CrosHost._RPM_HOSTNAME_REGEX, hostname)
Simran Basid5e5e272012-09-24 15:23:59 -07001662
1663
1664 def has_power(self):
1665 """For this host, return whether or not it is powered by an RPM.
1666
1667 @return True if this host is in the CROS lab and follows the defined
1668 naming format.
1669 """
Fang Deng0ca40e22013-08-27 17:47:44 -07001670 return CrosHost.check_for_rpm_support(self.hostname)
Simran Basid5e5e272012-09-24 15:23:59 -07001671
1672
Ismail Noorbasha07fdb612013-02-14 14:13:31 -08001673 def _set_power(self, state, power_method):
1674 """Sets the power to the host via RPM, Servo or manual.
1675
1676 @param state Specifies which power state to set to DUT
1677 @param power_method Specifies which method of power control to
1678 use. By default "RPM" will be used. Valid values
1679 are the strings "RPM", "manual", "servoj10".
1680
1681 """
1682 ACCEPTABLE_STATES = ['ON', 'OFF']
1683
1684 if state.upper() not in ACCEPTABLE_STATES:
1685 raise error.TestError('State must be one of: %s.'
1686 % (ACCEPTABLE_STATES,))
1687
1688 if power_method == self.POWER_CONTROL_SERVO:
1689 logging.info('Setting servo port J10 to %s', state)
1690 self.servo.set('prtctl3_pwren', state.lower())
1691 time.sleep(self._USB_POWER_TIMEOUT)
1692 elif power_method == self.POWER_CONTROL_MANUAL:
1693 logging.info('You have %d seconds to set the AC power to %s.',
1694 self._POWER_CYCLE_TIMEOUT, state)
1695 time.sleep(self._POWER_CYCLE_TIMEOUT)
1696 else:
1697 if not self.has_power():
1698 raise error.TestFail('DUT does not have RPM connected.')
Simran Basi5e6339a2013-03-21 11:34:32 -07001699 afe = frontend_wrappers.RetryingAFE(timeout_min=5, delay_sec=10)
1700 afe.set_host_attribute(self._RPM_OUTLET_CHANGED, True,
1701 hostname=self.hostname)
Simran Basi1df55112013-09-06 11:25:09 -07001702 rpm_client.set_power(self.hostname, state.upper(), timeout_mins=5)
Simran Basid5e5e272012-09-24 15:23:59 -07001703
1704
Ismail Noorbasha07fdb612013-02-14 14:13:31 -08001705 def power_off(self, power_method=POWER_CONTROL_RPM):
1706 """Turn off power to this host via RPM, Servo or manual.
1707
1708 @param power_method Specifies which method of power control to
1709 use. By default "RPM" will be used. Valid values
1710 are the strings "RPM", "manual", "servoj10".
1711
1712 """
1713 self._set_power('OFF', power_method)
Simran Basid5e5e272012-09-24 15:23:59 -07001714
1715
Ismail Noorbasha07fdb612013-02-14 14:13:31 -08001716 def power_on(self, power_method=POWER_CONTROL_RPM):
1717 """Turn on power to this host via RPM, Servo or manual.
1718
1719 @param power_method Specifies which method of power control to
1720 use. By default "RPM" will be used. Valid values
1721 are the strings "RPM", "manual", "servoj10".
1722
1723 """
1724 self._set_power('ON', power_method)
1725
1726
1727 def power_cycle(self, power_method=POWER_CONTROL_RPM):
1728 """Cycle power to this host by turning it OFF, then ON.
1729
1730 @param power_method Specifies which method of power control to
1731 use. By default "RPM" will be used. Valid values
1732 are the strings "RPM", "manual", "servoj10".
1733
1734 """
1735 if power_method in (self.POWER_CONTROL_SERVO,
1736 self.POWER_CONTROL_MANUAL):
1737 self.power_off(power_method=power_method)
1738 time.sleep(self._POWER_CYCLE_TIMEOUT)
1739 self.power_on(power_method=power_method)
1740 else:
1741 rpm_client.set_power(self.hostname, 'CYCLE')
Simran Basic6f1f7a2012-10-16 10:47:46 -07001742
1743
1744 def get_platform(self):
1745 """Determine the correct platform label for this host.
1746
1747 @returns a string representing this host's platform.
1748 """
1749 crossystem = utils.Crossystem(self)
1750 crossystem.init()
1751 # Extract fwid value and use the leading part as the platform id.
1752 # fwid generally follow the format of {platform}.{firmware version}
1753 # Example: Alex.X.YYY.Z or Google_Alex.X.YYY.Z
1754 platform = crossystem.fwid().split('.')[0].lower()
1755 # Newer platforms start with 'Google_' while the older ones do not.
1756 return platform.replace('google_', '')
1757
1758
Hung-ying Tyanb1328032014-04-01 14:18:54 +08001759 def get_architecture(self):
1760 """Determine the correct architecture label for this host.
1761
1762 @returns a string representing this host's architecture.
1763 """
1764 crossystem = utils.Crossystem(self)
1765 crossystem.init()
1766 return crossystem.arch()
1767
1768
Luis Lozano40b7d0d2014-01-17 15:12:06 -08001769 def get_chrome_version(self):
1770 """Gets the Chrome version number and milestone as strings.
1771
1772 Invokes "chrome --version" to get the version number and milestone.
1773
1774 @return A tuple (chrome_ver, milestone) where "chrome_ver" is the
1775 current Chrome version number as a string (in the form "W.X.Y.Z")
1776 and "milestone" is the first component of the version number
1777 (the "W" from "W.X.Y.Z"). If the version number cannot be parsed
1778 in the "W.X.Y.Z" format, the "chrome_ver" will be the full output
1779 of "chrome --version" and the milestone will be the empty string.
1780
1781 """
1782 version_string = self.run(constants.CHROME_VERSION_COMMAND).stdout
1783 return utils.parse_chrome_version(version_string)
1784
Aviv Keshet74c89a92013-02-04 15:18:30 -08001785 @label_decorator()
Simran Basic6f1f7a2012-10-16 10:47:46 -07001786 def get_board(self):
1787 """Determine the correct board label for this host.
1788
1789 @returns a string representing this host's board.
1790 """
1791 release_info = utils.parse_cmd_output('cat /etc/lsb-release',
1792 run_method=self.run)
1793 board = release_info['CHROMEOS_RELEASE_BOARD']
1794 # Devices in the lab generally have the correct board name but our own
1795 # development devices have {board_name}-signed-{key_type}. The board
1796 # name may also begin with 'x86-' which we need to keep.
Simran Basi833814b2013-01-29 13:13:43 -08001797 board_format_string = ds_constants.BOARD_PREFIX + '%s'
Simran Basic6f1f7a2012-10-16 10:47:46 -07001798 if 'x86' not in board:
Simran Basi833814b2013-01-29 13:13:43 -08001799 return board_format_string % board.split('-')[0]
1800 return board_format_string % '-'.join(board.split('-')[0:2])
Simran Basic6f1f7a2012-10-16 10:47:46 -07001801
1802
Aviv Keshet74c89a92013-02-04 15:18:30 -08001803 @label_decorator('lightsensor')
Simran Basic6f1f7a2012-10-16 10:47:46 -07001804 def has_lightsensor(self):
1805 """Determine the correct board label for this host.
1806
1807 @returns the string 'lightsensor' if this host has a lightsensor or
1808 None if it does not.
1809 """
1810 search_cmd = "find -L %s -maxdepth 4 | egrep '%s'" % (
Richard Barnette82c35912012-11-20 10:09:10 -08001811 self._LIGHTSENSOR_SEARCH_DIR, '|'.join(self._LIGHTSENSOR_FILES))
Simran Basic6f1f7a2012-10-16 10:47:46 -07001812 try:
1813 # Run the search cmd following the symlinks. Stderr_tee is set to
1814 # None as there can be a symlink loop, but the command will still
1815 # execute correctly with a few messages printed to stderr.
1816 self.run(search_cmd, stdout_tee=None, stderr_tee=None)
1817 return 'lightsensor'
1818 except error.AutoservRunError:
1819 # egrep exited with a return code of 1 meaning none of the possible
1820 # lightsensor files existed.
1821 return None
1822
1823
Aviv Keshet74c89a92013-02-04 15:18:30 -08001824 @label_decorator('bluetooth')
Simran Basic6f1f7a2012-10-16 10:47:46 -07001825 def has_bluetooth(self):
1826 """Determine the correct board label for this host.
1827
1828 @returns the string 'bluetooth' if this host has bluetooth or
1829 None if it does not.
1830 """
1831 try:
1832 self.run('test -d /sys/class/bluetooth/hci0')
1833 # test exited with a return code of 0.
1834 return 'bluetooth'
1835 except error.AutoservRunError:
1836 # test exited with a return code 1 meaning the directory did not
1837 # exist.
1838 return None
1839
1840
Ilja Friedel0ce0b602013-08-15 18:45:27 -07001841 @label_decorator('graphics')
1842 def get_graphics(self):
1843 """
1844 Determine the correct board label for this host.
1845
1846 @returns a string representing this host's graphics. For now ARM boards
1847 return graphics:gles while all other boards return graphics:gl. This
1848 may change over time, but for robustness reasons this should avoid
1849 executing code in actual graphics libraries (which may not be ready and
1850 is tested by graphics_GLAPICheck).
1851 """
1852 uname = self.run('uname -a').stdout.lower()
1853 if 'arm' in uname:
1854 return 'graphics:gles'
1855 return 'graphics:gl'
1856
1857
Bill Richardson4f595f52014-02-13 16:20:26 -08001858 @label_decorator('ec')
1859 def get_ec(self):
1860 """
1861 Determine the type of EC on this host.
1862
1863 @returns a string representing this host's embedded controller type.
1864 At present, it only returns "ec:cros", for Chrome OS ECs. Other types
1865 of EC (or none) don't return any strings, since no tests depend on
1866 those.
1867 """
1868 cmd = 'mosys ec info'
1869 # The output should look like these, so that the last field should
1870 # match our EC version scheme:
1871 #
1872 # stm | stm32f100 | snow_v1.3.139-375eb9f
1873 # ti | Unknown-10de | peppy_v1.5.114-5d52788
1874 #
1875 # Non-Chrome OS ECs will look like these:
1876 #
1877 # ENE | KB932 | 00BE107A00
1878 # ite | it8518 | 3.08
1879 #
1880 # And some systems don't have ECs at all (Lumpy, for example).
1881 regexp = r'^.*\|\s*(\S+_v\d+\.\d+\.\d+-[0-9a-f]+)\s*$'
1882
1883 ecinfo = self.run(command=cmd, ignore_status=True)
1884 if ecinfo.exit_status == 0:
1885 res = re.search(regexp, ecinfo.stdout)
1886 if res:
1887 logging.info("EC version is %s", res.groups()[0])
1888 return 'ec:cros'
1889 logging.info("%s got: %s", cmd, ecinfo.stdout)
1890 # Has an EC, but it's not a Chrome OS EC
1891 return None
1892 logging.info("%s exited with status %d", cmd, ecinfo.exit_status)
1893 # No EC present
1894 return None
1895
1896
Alec Berg31b932b2014-04-04 16:09:11 -07001897 @label_decorator('accels')
1898 def get_accels(self):
1899 """
1900 Determine the type of accelerometers on this host.
1901
1902 @returns a string representing this host's accelerometer type.
1903 At present, it only returns "accel:cros-ec", for accelerometers
1904 attached to a Chrome OS EC, or none, if no accelerometers.
1905 """
1906 # Check to make sure we have ectool
1907 rv = self.run('which ectool', ignore_status=True)
1908 if rv.exit_status:
1909 logging.info("No ectool cmd found, assuming no EC accelerometers")
1910 return None
1911
1912 # Check that the EC supports the motionsense command
1913 rv = self.run('ectool motionsense', ignore_status=True)
1914 if rv.exit_status:
1915 logging.info("EC does not support motionsense command "
1916 "assuming no EC accelerometers")
1917 return None
1918
1919 # Check that EC motion sensors are active
1920 active = self.run('ectool motionsense active').stdout.split('\n')
1921 if active[0] == "0":
1922 logging.info("Motion sense inactive, assuming no EC accelerometers")
1923 return None
1924
1925 logging.info("EC accelerometers found")
1926 return 'accel:cros-ec'
1927
1928
Tom Wai-Hong Tam3d6790d2014-04-14 16:15:47 +08001929 @label_decorator('chameleon')
1930 def has_chameleon(self):
1931 """Determine if a Chameleon connected to this host.
1932
1933 @returns the string 'chameleon' if this host has a Chameleon or
1934 None if it has not.
1935 """
1936 if self._chameleon_host:
1937 return 'chameleon'
1938 else:
1939 return None
1940
1941
Puthikorn Voravootivatfa011242014-03-14 18:45:11 -07001942 @label_decorator('storage')
1943 def get_storage(self):
1944 """
1945 Determine the type of boot device for this host.
1946
1947 Determine if the internal device is SCSI or dw_mmc device.
1948 Then check that it is SSD or HDD or eMMC or something else.
1949
1950 @returns a string representing this host's internal device type.
1951 'storage:ssd' when internal device is solid state drive
1952 'storage:hdd' when internal device is hard disk drive
1953 'storage:mmc' when internal device is mmc drive
1954 None When internal device is something else or
1955 when we are unable to determine the type
1956 """
1957 # The output should be /dev/mmcblk* for SD/eMMC or /dev/sd* for scsi
1958 rootdev_cmd = ' '.join(['. /usr/sbin/write_gpt.sh;',
1959 '. /usr/share/misc/chromeos-common.sh;',
1960 'load_base_vars;',
1961 'get_fixed_dst_drive'])
Puthikorn Voravootivat03c51682014-04-24 13:52:12 -07001962 rootdev = self.run(command=rootdev_cmd, ignore_status=True)
1963 if rootdev.exit_status:
1964 logging.info("Fail to run %s", rootdev_cmd)
1965 return None
Puthikorn Voravootivatfa011242014-03-14 18:45:11 -07001966 rootdev_str = rootdev.stdout.strip()
1967
1968 if not rootdev_str:
1969 return None
1970
1971 rootdev_base = os.path.basename(rootdev_str)
1972
1973 mmc_pattern = '/dev/mmcblk[0-9]'
1974 if re.match(mmc_pattern, rootdev_str):
1975 # Use type to determine if the internal device is eMMC or somthing
1976 # else. We can assume that MMC is always an internal device.
1977 type_cmd = 'cat /sys/block/%s/device/type' % rootdev_base
Puthikorn Voravootivat03c51682014-04-24 13:52:12 -07001978 type = self.run(command=type_cmd, ignore_status=True)
1979 if type.exit_status:
1980 logging.info("Fail to run %s", type_cmd)
1981 return None
Puthikorn Voravootivatfa011242014-03-14 18:45:11 -07001982 type_str = type.stdout.strip()
1983
1984 if type_str == 'MMC':
1985 return 'storage:mmc'
1986
1987 scsi_pattern = '/dev/sd[a-z]+'
1988 if re.match(scsi_pattern, rootdev.stdout):
1989 # Read symlink for /sys/block/sd* to determine if the internal
1990 # device is connected via ata or usb.
1991 link_cmd = 'readlink /sys/block/%s' % rootdev_base
Puthikorn Voravootivat03c51682014-04-24 13:52:12 -07001992 link = self.run(command=link_cmd, ignore_status=True)
1993 if link.exit_status:
1994 logging.info("Fail to run %s", link_cmd)
1995 return None
Puthikorn Voravootivatfa011242014-03-14 18:45:11 -07001996 link_str = link.stdout.strip()
1997 if 'usb' in link_str:
1998 return None
1999
2000 # Read rotation to determine if the internal device is ssd or hdd.
2001 rotate_cmd = str('cat /sys/block/%s/queue/rotational'
2002 % rootdev_base)
Puthikorn Voravootivat03c51682014-04-24 13:52:12 -07002003 rotate = self.run(command=rotate_cmd, ignore_status=True)
2004 if rotate.exit_status:
2005 logging.info("Fail to run %s", rotate_cmd)
2006 return None
Puthikorn Voravootivatfa011242014-03-14 18:45:11 -07002007 rotate_str = rotate.stdout.strip()
2008
2009 rotate_dict = {'0':'storage:ssd', '1':'storage:hdd'}
2010 return rotate_dict.get(rotate_str)
2011
2012 # All other internal device / error case will always fall here
2013 return None
2014
2015
Simran Basic6f1f7a2012-10-16 10:47:46 -07002016 def get_labels(self):
2017 """Return a list of labels for this given host.
2018
2019 This is the main way to retrieve all the automatic labels for a host
2020 as it will run through all the currently implemented label functions.
2021 """
2022 labels = []
Richard Barnette82c35912012-11-20 10:09:10 -08002023 for label_function in self._LABEL_FUNCTIONS:
Simran Basic6f1f7a2012-10-16 10:47:46 -07002024 label = label_function(self)
2025 if label:
2026 labels.append(label)
2027 return labels
Dan Shi85276d42014-04-08 22:11:45 -07002028
2029
2030 def is_boot_from_usb(self):
2031 """Check if DUT is boot from USB.
2032
2033 @return: True if DUT is boot from usb.
2034 """
2035 device = self.run('rootdev -s -d').stdout.strip()
2036 removable = int(self.run('cat /sys/block/%s/removable' %
2037 os.path.basename(device)).stdout.strip())
2038 return removable == 1