Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 1 | # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. |
| 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | # |
| 5 | # Expects to be run in an environment with sudo and no interactive password |
| 6 | # prompt, such as within the Chromium OS development chroot. |
| 7 | |
| 8 | |
| 9 | """This file provides core logic for servo verify/repair process.""" |
| 10 | |
| 11 | |
| 12 | import httplib |
| 13 | import logging |
| 14 | import socket |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 15 | import xmlrpclib |
Raul E Rangel | 52ca2e8 | 2018-07-03 14:10:14 -0600 | [diff] [blame] | 16 | import os |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 17 | |
| 18 | from autotest_lib.client.bin import utils |
Kevin Cheng | 5f2ba6c | 2016-09-28 10:20:05 -0700 | [diff] [blame] | 19 | from autotest_lib.client.common_lib import control_data |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 20 | from autotest_lib.client.common_lib import error |
beeps | 5e8c45a | 2013-12-17 22:05:11 -0800 | [diff] [blame] | 21 | from autotest_lib.client.common_lib import global_config |
Kevin Cheng | 5f2ba6c | 2016-09-28 10:20:05 -0700 | [diff] [blame] | 22 | from autotest_lib.client.common_lib import host_states |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 23 | from autotest_lib.client.common_lib import hosts |
Dan Shi | 0942b1d | 2015-03-31 11:07:00 -0700 | [diff] [blame] | 24 | from autotest_lib.client.common_lib import lsbrelease_utils |
beeps | 5e8c45a | 2013-12-17 22:05:11 -0800 | [diff] [blame] | 25 | from autotest_lib.client.common_lib.cros import dev_server |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 26 | from autotest_lib.client.common_lib.cros import retry |
Christopher Wiley | cef1f90 | 2014-06-19 11:11:23 -0700 | [diff] [blame] | 27 | from autotest_lib.client.common_lib.cros.network import ping_runner |
Hsinyu Chao | e0b08e6 | 2015-08-11 10:50:37 +0000 | [diff] [blame] | 28 | from autotest_lib.client.cros import constants as client_constants |
Richard Barnette | e519dcd | 2016-08-15 17:37:17 -0700 | [diff] [blame] | 29 | from autotest_lib.server import afe_utils |
Prathmesh Prabhu | cbd5ebb | 2018-08-28 17:04:50 -0700 | [diff] [blame] | 30 | from autotest_lib.server import site_utils as server_utils |
Richard Barnette | d31580e | 2018-05-14 19:58:00 +0000 | [diff] [blame] | 31 | from autotest_lib.server.cros import autoupdater |
Kevin Cheng | 5f2ba6c | 2016-09-28 10:20:05 -0700 | [diff] [blame] | 32 | from autotest_lib.server.cros.dynamic_suite import control_file_getter |
Richard Barnette | d31580e | 2018-05-14 19:58:00 +0000 | [diff] [blame] | 33 | from autotest_lib.server.cros.dynamic_suite import frontend_wrappers |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 34 | from autotest_lib.server.cros.servo import servo |
Richard Barnette | d31580e | 2018-05-14 19:58:00 +0000 | [diff] [blame] | 35 | from autotest_lib.server.hosts import servo_repair |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 36 | from autotest_lib.server.hosts import ssh_host |
Fang Deng | d4fe739 | 2013-09-20 12:18:21 -0700 | [diff] [blame] | 37 | from autotest_lib.site_utils.rpm_control_system import rpm_client |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 38 | |
Dan Shi | 5e2efb7 | 2017-02-07 11:40:23 -0800 | [diff] [blame] | 39 | try: |
| 40 | from chromite.lib import metrics |
| 41 | except ImportError: |
| 42 | metrics = utils.metrics_mock |
| 43 | |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 44 | |
Simran Basi | 0739d68 | 2015-02-25 16:22:56 -0800 | [diff] [blame] | 45 | # Names of the host attributes in the database that represent the values for |
| 46 | # the servo_host and servo_port for a servo connected to the DUT. |
| 47 | SERVO_HOST_ATTR = 'servo_host' |
| 48 | SERVO_PORT_ATTR = 'servo_port' |
Richard Barnette | e519dcd | 2016-08-15 17:37:17 -0700 | [diff] [blame] | 49 | SERVO_BOARD_ATTR = 'servo_board' |
Nick Sanders | 2f3c985 | 2018-10-24 12:10:24 -0700 | [diff] [blame] | 50 | # Model is inferred from host labels. |
| 51 | SERVO_MODEL_ATTR = 'servo_model' |
Kevin Cheng | 643ce8a | 2016-09-15 15:42:12 -0700 | [diff] [blame] | 52 | SERVO_SERIAL_ATTR = 'servo_serial' |
Prathmesh Prabhu | cba4429 | 2018-08-28 17:44:45 -0700 | [diff] [blame] | 53 | SERVO_ATTR_KEYS = ( |
| 54 | SERVO_BOARD_ATTR, |
| 55 | SERVO_HOST_ATTR, |
| 56 | SERVO_PORT_ATTR, |
| 57 | SERVO_SERIAL_ATTR, |
| 58 | ) |
Simran Basi | 0739d68 | 2015-02-25 16:22:56 -0800 | [diff] [blame] | 59 | |
Dan Shi | 3b2adf6 | 2015-09-02 17:46:54 -0700 | [diff] [blame] | 60 | _CONFIG = global_config.global_config |
xixuan | 6cf6d2f | 2016-01-29 15:29:00 -0800 | [diff] [blame] | 61 | ENABLE_SSH_TUNNEL_FOR_SERVO = _CONFIG.get_config_value( |
| 62 | 'CROS', 'enable_ssh_tunnel_for_servo', type=bool, default=False) |
Simran Basi | 0739d68 | 2015-02-25 16:22:56 -0800 | [diff] [blame] | 63 | |
Kevin Cheng | 5f2ba6c | 2016-09-28 10:20:05 -0700 | [diff] [blame] | 64 | AUTOTEST_BASE = _CONFIG.get_config_value( |
| 65 | 'SCHEDULER', 'drone_installation_directory', |
| 66 | default='/usr/local/autotest') |
| 67 | |
| 68 | _SERVO_HOST_REBOOT_TEST_NAME = 'servohost_Reboot' |
Kevin Cheng | 5526590 | 2016-10-19 12:46:50 -0700 | [diff] [blame] | 69 | _SERVO_HOST_FORCE_REBOOT_TEST_NAME = 'servohost_Reboot.force_reboot' |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 70 | |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 71 | class ServoHost(ssh_host.SSHHost): |
| 72 | """Host class for a host that controls a servo, e.g. beaglebone.""" |
| 73 | |
Raul E Rangel | 52ca2e8 | 2018-07-03 14:10:14 -0600 | [diff] [blame] | 74 | DEFAULT_PORT = int(os.getenv('SERVOD_PORT', '9999')) |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 75 | |
Dan Shi | e5b3c51 | 2014-08-21 12:12:09 -0700 | [diff] [blame] | 76 | # Timeout for initializing servo signals. |
Wai-Hong Tam | 37b6ed3 | 2017-09-19 15:52:39 -0700 | [diff] [blame] | 77 | INITIALIZE_SERVO_TIMEOUT_SECS = 60 |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 78 | |
xixuan | 6cf6d2f | 2016-01-29 15:29:00 -0800 | [diff] [blame] | 79 | # Ready test function |
| 80 | SERVO_READY_METHOD = 'get_version' |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 81 | |
Kevin Cheng | 5f2ba6c | 2016-09-28 10:20:05 -0700 | [diff] [blame] | 82 | REBOOT_CMD = 'sleep 1; reboot & sleep 10; reboot -f' |
| 83 | |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 84 | |
Richard Barnette | 17bfc6c | 2016-08-04 18:41:43 -0700 | [diff] [blame] | 85 | def _initialize(self, servo_host='localhost', |
Richard Barnette | e519dcd | 2016-08-15 17:37:17 -0700 | [diff] [blame] | 86 | servo_port=DEFAULT_PORT, servo_board=None, |
Nick Sanders | 2f3c985 | 2018-10-24 12:10:24 -0700 | [diff] [blame] | 87 | servo_model=None, servo_serial=None, is_in_lab=None, |
| 88 | *args, **dargs): |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 89 | """Initialize a ServoHost instance. |
| 90 | |
| 91 | A ServoHost instance represents a host that controls a servo. |
| 92 | |
| 93 | @param servo_host: Name of the host where the servod process |
| 94 | is running. |
Raul E Rangel | 52ca2e8 | 2018-07-03 14:10:14 -0600 | [diff] [blame] | 95 | @param servo_port: Port the servod process is listening on. Defaults |
| 96 | to the SERVOD_PORT environment variable if set, |
| 97 | otherwise 9999. |
Kevin Cheng | 5f2ba6c | 2016-09-28 10:20:05 -0700 | [diff] [blame] | 98 | @param servo_board: Board that the servo is connected to. |
Nick Sanders | 2f3c985 | 2018-10-24 12:10:24 -0700 | [diff] [blame] | 99 | @param servo_model: Model that the servo is connected to. |
Dan Shi | 4d47852 | 2014-02-14 13:46:32 -0800 | [diff] [blame] | 100 | @param is_in_lab: True if the servo host is in Cros Lab. Default is set |
| 101 | to None, for which utils.host_is_in_lab_zone will be |
| 102 | called to check if the servo host is in Cros lab. |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 103 | |
| 104 | """ |
| 105 | super(ServoHost, self)._initialize(hostname=servo_host, |
| 106 | *args, **dargs) |
Richard Barnette | 42f4db9 | 2018-08-23 15:05:15 -0700 | [diff] [blame] | 107 | self.servo_port = int(servo_port) |
Richard Barnette | e519dcd | 2016-08-15 17:37:17 -0700 | [diff] [blame] | 108 | self.servo_board = servo_board |
Nick Sanders | 2f3c985 | 2018-10-24 12:10:24 -0700 | [diff] [blame] | 109 | self.servo_model = servo_model |
Kevin Cheng | 643ce8a | 2016-09-15 15:42:12 -0700 | [diff] [blame] | 110 | self.servo_serial = servo_serial |
Richard Barnette | e519dcd | 2016-08-15 17:37:17 -0700 | [diff] [blame] | 111 | self._servo = None |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 112 | self._repair_strategy = ( |
| 113 | servo_repair.create_servo_repair_strategy()) |
Richard Barnette | e519dcd | 2016-08-15 17:37:17 -0700 | [diff] [blame] | 114 | self._is_localhost = (self.hostname == 'localhost') |
| 115 | if self._is_localhost: |
| 116 | self._is_in_lab = False |
| 117 | elif is_in_lab is None: |
Dan Shi | 4d47852 | 2014-02-14 13:46:32 -0800 | [diff] [blame] | 118 | self._is_in_lab = utils.host_is_in_lab_zone(self.hostname) |
| 119 | else: |
| 120 | self._is_in_lab = is_in_lab |
xixuan | 6cf6d2f | 2016-01-29 15:29:00 -0800 | [diff] [blame] | 121 | |
Richard Barnette | e519dcd | 2016-08-15 17:37:17 -0700 | [diff] [blame] | 122 | # Commands on the servo host must be run by the superuser. |
| 123 | # Our account on a remote host is root, but if our target is |
| 124 | # localhost then we might be running unprivileged. If so, |
| 125 | # `sudo` will have to be added to the commands. |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 126 | if self._is_localhost: |
| 127 | self._sudo_required = utils.system_output('id -u') != '0' |
| 128 | else: |
| 129 | self._sudo_required = False |
Richard Barnette | e519dcd | 2016-08-15 17:37:17 -0700 | [diff] [blame] | 130 | |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 131 | |
| 132 | def connect_servo(self): |
Kevin Cheng | 5f2ba6c | 2016-09-28 10:20:05 -0700 | [diff] [blame] | 133 | """Establish a connection to the servod server on this host. |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 134 | |
| 135 | Initializes `self._servo` and then verifies that all network |
| 136 | connections are working. This will create an ssh tunnel if |
| 137 | it's required. |
| 138 | |
| 139 | As a side effect of testing the connection, all signals on the |
| 140 | target servo are reset to default values, and the USB stick is |
| 141 | set to the neutral (off) position. |
| 142 | """ |
Kevin Cheng | 643ce8a | 2016-09-15 15:42:12 -0700 | [diff] [blame] | 143 | servo_obj = servo.Servo(servo_host=self, servo_serial=self.servo_serial) |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 144 | timeout, _ = retry.timeout( |
| 145 | servo_obj.initialize_dut, |
| 146 | timeout_sec=self.INITIALIZE_SERVO_TIMEOUT_SECS) |
| 147 | if timeout: |
| 148 | raise hosts.AutoservVerifyError( |
| 149 | 'Servo initialize timed out.') |
| 150 | self._servo = servo_obj |
| 151 | |
| 152 | |
| 153 | def disconnect_servo(self): |
Kevin Cheng | 5f2ba6c | 2016-09-28 10:20:05 -0700 | [diff] [blame] | 154 | """Disconnect our servo if it exists. |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 155 | |
| 156 | If we've previously successfully connected to our servo, |
| 157 | disconnect any established ssh tunnel, and set `self._servo` |
| 158 | back to `None`. |
| 159 | """ |
| 160 | if self._servo: |
| 161 | # N.B. This call is safe even without a tunnel: |
| 162 | # rpc_server_tracker.disconnect() silently ignores |
| 163 | # unknown ports. |
| 164 | self.rpc_server_tracker.disconnect(self.servo_port) |
| 165 | self._servo = None |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 166 | |
| 167 | |
| 168 | def is_in_lab(self): |
| 169 | """Check whether the servo host is a lab device. |
| 170 | |
| 171 | @returns: True if the servo host is in Cros Lab, otherwise False. |
| 172 | |
| 173 | """ |
| 174 | return self._is_in_lab |
| 175 | |
| 176 | |
| 177 | def is_localhost(self): |
| 178 | """Checks whether the servo host points to localhost. |
| 179 | |
| 180 | @returns: True if it points to localhost, otherwise False. |
| 181 | |
| 182 | """ |
| 183 | return self._is_localhost |
| 184 | |
| 185 | |
| 186 | def get_servod_server_proxy(self): |
Kevin Cheng | 5f2ba6c | 2016-09-28 10:20:05 -0700 | [diff] [blame] | 187 | """Return a proxy that can be used to communicate with servod server. |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 188 | |
| 189 | @returns: An xmlrpclib.ServerProxy that is connected to the servod |
| 190 | server on the host. |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 191 | """ |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 192 | if ENABLE_SSH_TUNNEL_FOR_SERVO and not self.is_localhost(): |
| 193 | return self.rpc_server_tracker.xmlrpc_connect( |
| 194 | None, self.servo_port, |
| 195 | ready_test_name=self.SERVO_READY_METHOD, |
Allen Li | 2b1a899 | 2018-11-27 14:17:18 -0800 | [diff] [blame] | 196 | timeout_seconds=60, |
Allen Li | 556f453 | 2018-12-03 18:11:23 -0800 | [diff] [blame] | 197 | request_timeout_seconds=3600) |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 198 | else: |
| 199 | remote = 'http://%s:%s' % (self.hostname, self.servo_port) |
| 200 | return xmlrpclib.ServerProxy(remote) |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 201 | |
| 202 | |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 203 | def is_cros_host(self): |
beeps | 5e8c45a | 2013-12-17 22:05:11 -0800 | [diff] [blame] | 204 | """Check if a servo host is running chromeos. |
| 205 | |
| 206 | @return: True if the servo host is running chromeos. |
| 207 | False if it isn't, or we don't have enough information. |
| 208 | """ |
| 209 | try: |
| 210 | result = self.run('grep -q CHROMEOS /etc/lsb-release', |
| 211 | ignore_status=True, timeout=10) |
| 212 | except (error.AutoservRunError, error.AutoservSSHTimeout): |
| 213 | return False |
| 214 | return result.exit_status == 0 |
| 215 | |
| 216 | |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 217 | def make_ssh_command(self, user='root', port=22, opts='', hosts_file=None, |
Dean Liao | e3e75f6 | 2017-11-14 10:36:43 +0800 | [diff] [blame] | 218 | connect_timeout=None, alive_interval=None, |
| 219 | alive_count_max=None, connection_attempts=None): |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 220 | """Override default make_ssh_command to use tuned options. |
| 221 | |
| 222 | Tuning changes: |
| 223 | - ConnectTimeout=30; maximum of 30 seconds allowed for an SSH |
| 224 | connection failure. Consistency with remote_access.py. |
| 225 | |
| 226 | - ServerAliveInterval=180; which causes SSH to ping connection every |
| 227 | 180 seconds. In conjunction with ServerAliveCountMax ensures |
| 228 | that if the connection dies, Autotest will bail out quickly. |
| 229 | |
| 230 | - ServerAliveCountMax=3; consistency with remote_access.py. |
| 231 | |
| 232 | - ConnectAttempts=4; reduce flakiness in connection errors; |
| 233 | consistency with remote_access.py. |
| 234 | |
| 235 | - UserKnownHostsFile=/dev/null; we don't care about the keys. |
| 236 | |
| 237 | - SSH protocol forced to 2; needed for ServerAliveInterval. |
| 238 | |
| 239 | @param user User name to use for the ssh connection. |
| 240 | @param port Port on the target host to use for ssh connection. |
| 241 | @param opts Additional options to the ssh command. |
| 242 | @param hosts_file Ignored. |
| 243 | @param connect_timeout Ignored. |
| 244 | @param alive_interval Ignored. |
Dean Liao | e3e75f6 | 2017-11-14 10:36:43 +0800 | [diff] [blame] | 245 | @param alive_count_max Ignored. |
| 246 | @param connection_attempts Ignored. |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 247 | |
| 248 | @returns: An ssh command with the requested settings. |
| 249 | |
| 250 | """ |
Dean Liao | e3e75f6 | 2017-11-14 10:36:43 +0800 | [diff] [blame] | 251 | options = ' '.join([opts, '-o Protocol=2']) |
| 252 | return super(ServoHost, self).make_ssh_command( |
| 253 | user=user, port=port, opts=options, hosts_file='/dev/null', |
| 254 | connect_timeout=30, alive_interval=180, alive_count_max=3, |
| 255 | connection_attempts=4) |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 256 | |
| 257 | |
| 258 | def _make_scp_cmd(self, sources, dest): |
| 259 | """Format scp command. |
| 260 | |
| 261 | Given a list of source paths and a destination path, produces the |
| 262 | appropriate scp command for encoding it. Remote paths must be |
| 263 | pre-encoded. Overrides _make_scp_cmd in AbstractSSHHost |
| 264 | to allow additional ssh options. |
| 265 | |
| 266 | @param sources: A list of source paths to copy from. |
| 267 | @param dest: Destination path to copy to. |
| 268 | |
| 269 | @returns: An scp command that copies |sources| on local machine to |
| 270 | |dest| on the remote servo host. |
| 271 | |
| 272 | """ |
| 273 | command = ('scp -rq %s -o BatchMode=yes -o StrictHostKeyChecking=no ' |
| 274 | '-o UserKnownHostsFile=/dev/null -P %d %s "%s"') |
| 275 | return command % (self.master_ssh_option, |
| 276 | self.port, ' '.join(sources), dest) |
| 277 | |
| 278 | |
| 279 | def run(self, command, timeout=3600, ignore_status=False, |
| 280 | stdout_tee=utils.TEE_TO_LOGS, stderr_tee=utils.TEE_TO_LOGS, |
Luigi Semenzato | bfbd1f3 | 2017-01-06 10:41:18 -0800 | [diff] [blame] | 281 | connect_timeout=30, ssh_failure_retry_ok=False, |
| 282 | options='', stdin=None, verbose=True, args=()): |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 283 | """Run a command on the servo host. |
| 284 | |
| 285 | Extends method `run` in SSHHost. If the servo host is a remote device, |
| 286 | it will call `run` in SSHost without changing anything. |
| 287 | If the servo host is 'localhost', it will call utils.system_output. |
| 288 | |
| 289 | @param command: The command line string. |
| 290 | @param timeout: Time limit in seconds before attempting to |
| 291 | kill the running process. The run() function |
| 292 | will take a few seconds longer than 'timeout' |
| 293 | to complete if it has to kill the process. |
| 294 | @param ignore_status: Do not raise an exception, no matter |
| 295 | what the exit code of the command is. |
| 296 | @param stdout_tee/stderr_tee: Where to tee the stdout/stderr. |
| 297 | @param connect_timeout: SSH connection timeout (in seconds) |
| 298 | Ignored if host is 'localhost'. |
| 299 | @param options: String with additional ssh command options |
| 300 | Ignored if host is 'localhost'. |
Luigi Semenzato | bfbd1f3 | 2017-01-06 10:41:18 -0800 | [diff] [blame] | 301 | @param ssh_failure_retry_ok: when True and ssh connection failure is |
| 302 | suspected, OK to retry command (but not |
| 303 | compulsory, and likely not needed here) |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 304 | @param stdin: Stdin to pass (a string) to the executed command. |
| 305 | @param verbose: Log the commands. |
| 306 | @param args: Sequence of strings to pass as arguments to command by |
| 307 | quoting them in " and escaping their contents if necessary. |
| 308 | |
| 309 | @returns: A utils.CmdResult object. |
| 310 | |
| 311 | @raises AutoservRunError if the command failed. |
| 312 | @raises AutoservSSHTimeout SSH connection has timed out. Only applies |
| 313 | when servo host is not 'localhost'. |
| 314 | |
| 315 | """ |
| 316 | run_args = {'command': command, 'timeout': timeout, |
| 317 | 'ignore_status': ignore_status, 'stdout_tee': stdout_tee, |
| 318 | 'stderr_tee': stderr_tee, 'stdin': stdin, |
| 319 | 'verbose': verbose, 'args': args} |
| 320 | if self.is_localhost(): |
| 321 | if self._sudo_required: |
Michael Tang | f9b3ada | 2016-11-18 16:01:05 -0800 | [diff] [blame] | 322 | run_args['command'] = 'sudo -n sh -c "%s"' % utils.sh_escape( |
| 323 | command) |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 324 | try: |
| 325 | return utils.run(**run_args) |
| 326 | except error.CmdError as e: |
| 327 | logging.error(e) |
| 328 | raise error.AutoservRunError('command execution error', |
| 329 | e.result_obj) |
| 330 | else: |
| 331 | run_args['connect_timeout'] = connect_timeout |
| 332 | run_args['options'] = options |
| 333 | return super(ServoHost, self).run(**run_args) |
| 334 | |
| 335 | |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 336 | def _get_release_version(self): |
Dan Shi | 0942b1d | 2015-03-31 11:07:00 -0700 | [diff] [blame] | 337 | """Get the value of attribute CHROMEOS_RELEASE_VERSION from lsb-release. |
| 338 | |
| 339 | @returns The version string in lsb-release, under attribute |
| 340 | CHROMEOS_RELEASE_VERSION. |
| 341 | """ |
| 342 | lsb_release_content = self.run( |
| 343 | 'cat "%s"' % client_constants.LSB_RELEASE).stdout.strip() |
| 344 | return lsbrelease_utils.get_chromeos_release_version( |
| 345 | lsb_release_content=lsb_release_content) |
| 346 | |
| 347 | |
Kevin Cheng | 5f2ba6c | 2016-09-28 10:20:05 -0700 | [diff] [blame] | 348 | def get_attached_duts(self, afe): |
| 349 | """Gather a list of duts that use this servo host. |
| 350 | |
| 351 | @param afe: afe instance. |
| 352 | |
| 353 | @returns list of duts. |
Richard Barnette | 3a7697f | 2016-04-20 11:33:27 -0700 | [diff] [blame] | 354 | """ |
Kevin Cheng | 5f2ba6c | 2016-09-28 10:20:05 -0700 | [diff] [blame] | 355 | return afe.get_hosts_by_attribute( |
| 356 | attribute=SERVO_HOST_ATTR, value=self.hostname) |
| 357 | |
| 358 | |
| 359 | def get_board(self): |
| 360 | """Determine the board for this servo host. |
| 361 | |
| 362 | @returns a string representing this servo host's board. |
| 363 | """ |
| 364 | return lsbrelease_utils.get_current_board( |
| 365 | lsb_release_content=self.run('cat /etc/lsb-release').stdout) |
| 366 | |
| 367 | |
| 368 | def _choose_dut_for_synchronized_reboot(self, dut_list, afe): |
| 369 | """Choose which dut to schedule servo host reboot job. |
| 370 | |
| 371 | We'll want a semi-deterministic way of selecting which host should be |
| 372 | scheduled for the servo host reboot job. For now we'll sort the |
| 373 | list with the expectation the dut list will stay consistent. |
| 374 | From there we'll grab the first dut that is available so we |
| 375 | don't schedule a job on a dut that will never run. |
| 376 | |
| 377 | @param dut_list: List of the dut hostnames to choose from. |
| 378 | @param afe: Instance of the AFE. |
| 379 | |
| 380 | @return hostname of dut to schedule job on. |
| 381 | """ |
| 382 | afe_hosts = afe.get_hosts(dut_list) |
| 383 | afe_hosts.sort() |
| 384 | for afe_host in afe_hosts: |
| 385 | if afe_host.status not in host_states.UNAVAILABLE_STATES: |
| 386 | return afe_host.hostname |
| 387 | # If they're all unavailable, just return the first sorted dut. |
| 388 | dut_list.sort() |
| 389 | return dut_list[0] |
| 390 | |
| 391 | |
| 392 | def _sync_job_scheduled_for_duts(self, dut_list, afe): |
| 393 | """Checks if a synchronized reboot has been scheduled for these duts. |
| 394 | |
| 395 | Grab all the host queue entries that aren't completed for the duts and |
| 396 | see if any of them have the expected job name. |
| 397 | |
| 398 | @param dut_list: List of duts to check on. |
| 399 | @param afe: Instance of the AFE. |
| 400 | |
| 401 | @returns True if the job is scheduled, False otherwise. |
| 402 | """ |
| 403 | afe_hosts = afe.get_hosts(dut_list) |
| 404 | for afe_host in afe_hosts: |
| 405 | hqes = afe.get_host_queue_entries(host=afe_host.id, complete=0) |
| 406 | for hqe in hqes: |
| 407 | job = afe.get_jobs(id=hqe.job.id) |
Kevin Cheng | 5526590 | 2016-10-19 12:46:50 -0700 | [diff] [blame] | 408 | if job and job[0].name in (_SERVO_HOST_REBOOT_TEST_NAME, |
| 409 | _SERVO_HOST_FORCE_REBOOT_TEST_NAME): |
Kevin Cheng | 5f2ba6c | 2016-09-28 10:20:05 -0700 | [diff] [blame] | 410 | return True |
| 411 | return False |
| 412 | |
| 413 | |
Kevin Cheng | 5526590 | 2016-10-19 12:46:50 -0700 | [diff] [blame] | 414 | def schedule_synchronized_reboot(self, dut_list, afe, force_reboot=False): |
Kevin Cheng | 5f2ba6c | 2016-09-28 10:20:05 -0700 | [diff] [blame] | 415 | """Schedule a job to reboot the servo host. |
| 416 | |
| 417 | When we schedule a job, it will create a ServoHost object which will |
| 418 | go through this entire flow of checking if a reboot is needed and |
| 419 | trying to schedule it. There is probably a better approach to setting |
| 420 | up a synchronized reboot but I'm coming up short on better ideas so I |
| 421 | apologize for this circus show. |
| 422 | |
Kevin Cheng | 5526590 | 2016-10-19 12:46:50 -0700 | [diff] [blame] | 423 | @param dut_list: List of duts that need to be locked. |
| 424 | @param afe: Instance of afe. |
| 425 | @param force_reboot: Boolean to indicate if a forced reboot should be |
| 426 | scheduled or not. |
Kevin Cheng | 5f2ba6c | 2016-09-28 10:20:05 -0700 | [diff] [blame] | 427 | """ |
| 428 | # If we've already scheduled job on a dut, we're done here. |
| 429 | if self._sync_job_scheduled_for_duts(dut_list, afe): |
| 430 | return |
| 431 | |
| 432 | # Looks like we haven't scheduled a job yet. |
Kevin Cheng | 5526590 | 2016-10-19 12:46:50 -0700 | [diff] [blame] | 433 | test = (_SERVO_HOST_REBOOT_TEST_NAME if not force_reboot |
| 434 | else _SERVO_HOST_FORCE_REBOOT_TEST_NAME) |
Kevin Cheng | 5f2ba6c | 2016-09-28 10:20:05 -0700 | [diff] [blame] | 435 | dut = self._choose_dut_for_synchronized_reboot(dut_list, afe) |
| 436 | getter = control_file_getter.FileSystemGetter([AUTOTEST_BASE]) |
Kevin Cheng | 5526590 | 2016-10-19 12:46:50 -0700 | [diff] [blame] | 437 | control_file = getter.get_control_file_contents_by_name(test) |
Kevin Cheng | 5f2ba6c | 2016-09-28 10:20:05 -0700 | [diff] [blame] | 438 | control_type = control_data.CONTROL_TYPE_NAMES.SERVER |
Kevin Cheng | 7958998 | 2016-10-25 13:26:04 -0700 | [diff] [blame] | 439 | try: |
| 440 | afe.create_job(control_file=control_file, name=test, |
| 441 | control_type=control_type, hosts=[dut]) |
| 442 | except Exception as e: |
| 443 | # Sometimes creating the job will raise an exception. We'll log it |
| 444 | # but we don't want to fail because of it. |
Aviv Keshet | 5ae0a00 | 2017-05-05 10:23:33 -0700 | [diff] [blame] | 445 | logging.exception('Scheduling reboot job failed due to Exception.') |
Kevin Cheng | 5f2ba6c | 2016-09-28 10:20:05 -0700 | [diff] [blame] | 446 | |
| 447 | |
| 448 | def reboot(self, *args, **dargs): |
| 449 | """Reboot using special servo host reboot command.""" |
| 450 | super(ServoHost, self).reboot(reboot_cmd=self.REBOOT_CMD, |
| 451 | *args, **dargs) |
| 452 | |
| 453 | |
| 454 | def _check_for_reboot(self, updater): |
| 455 | """Reboot this servo host if an upgrade is waiting. |
Richard Barnette | 3a7697f | 2016-04-20 11:33:27 -0700 | [diff] [blame] | 456 | |
| 457 | If the host has successfully downloaded and finalized a new |
| 458 | build, reboot. |
| 459 | |
| 460 | @param updater: a ChromiumOSUpdater instance for checking |
| 461 | whether reboot is needed. |
| 462 | @return Return a (status, build) tuple reflecting the |
| 463 | update_engine status and current build of the host |
| 464 | at the end of the call. |
| 465 | """ |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 466 | current_build_number = self._get_release_version() |
Richard Barnette | 3a7697f | 2016-04-20 11:33:27 -0700 | [diff] [blame] | 467 | status = updater.check_update_status() |
| 468 | if status == autoupdater.UPDATER_NEED_REBOOT: |
Kevin Cheng | 5f2ba6c | 2016-09-28 10:20:05 -0700 | [diff] [blame] | 469 | # Check if we need to schedule an organized reboot. |
Kevin Cheng | 7958998 | 2016-10-25 13:26:04 -0700 | [diff] [blame] | 470 | afe = frontend_wrappers.RetryingAFE( |
| 471 | timeout_min=5, delay_sec=10, |
Prathmesh Prabhu | cbd5ebb | 2018-08-28 17:04:50 -0700 | [diff] [blame] | 472 | server=server_utils.get_global_afe_hostname()) |
Kevin Cheng | 5f2ba6c | 2016-09-28 10:20:05 -0700 | [diff] [blame] | 473 | dut_list = self.get_attached_duts(afe) |
| 474 | logging.info('servo host has the following duts: %s', dut_list) |
| 475 | if len(dut_list) > 1: |
| 476 | logging.info('servo host has multiple duts, scheduling ' |
| 477 | 'synchronized reboot') |
| 478 | self.schedule_synchronized_reboot(dut_list, afe) |
| 479 | return status, current_build_number |
| 480 | |
| 481 | logging.info('Rebooting servo host %s from build %s', |
Richard Barnette | 3a7697f | 2016-04-20 11:33:27 -0700 | [diff] [blame] | 482 | self.hostname, current_build_number) |
| 483 | # Tell the reboot() call not to wait for completion. |
| 484 | # Otherwise, the call will log reboot failure if servo does |
| 485 | # not come back. The logged reboot failure will lead to |
| 486 | # test job failure. If the test does not require servo, we |
| 487 | # don't want servo failure to fail the test with error: |
| 488 | # `Host did not return from reboot` in status.log. |
Kevin Cheng | 5f2ba6c | 2016-09-28 10:20:05 -0700 | [diff] [blame] | 489 | self.reboot(fastsync=True, wait=False) |
Richard Barnette | 3a7697f | 2016-04-20 11:33:27 -0700 | [diff] [blame] | 490 | |
| 491 | # We told the reboot() call not to wait, but we need to wait |
| 492 | # for the reboot before we continue. Alas. The code from |
| 493 | # here below is basically a copy of Host.wait_for_restart(), |
| 494 | # with the logging bits ripped out, so that they can't cause |
| 495 | # the failure logging problem described above. |
| 496 | # |
| 497 | # The black stain that this has left on my soul can never be |
| 498 | # erased. |
| 499 | old_boot_id = self.get_boot_id() |
| 500 | if not self.wait_down(timeout=self.WAIT_DOWN_REBOOT_TIMEOUT, |
| 501 | warning_timer=self.WAIT_DOWN_REBOOT_WARNING, |
| 502 | old_boot_id=old_boot_id): |
| 503 | raise error.AutoservHostError( |
Kevin Cheng | 5f2ba6c | 2016-09-28 10:20:05 -0700 | [diff] [blame] | 504 | 'servo host %s failed to shut down.' % |
| 505 | self.hostname) |
Richard Barnette | 3a7697f | 2016-04-20 11:33:27 -0700 | [diff] [blame] | 506 | if self.wait_up(timeout=120): |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 507 | current_build_number = self._get_release_version() |
Richard Barnette | 3a7697f | 2016-04-20 11:33:27 -0700 | [diff] [blame] | 508 | status = updater.check_update_status() |
| 509 | logging.info('servo host %s back from reboot, with build %s', |
| 510 | self.hostname, current_build_number) |
| 511 | else: |
| 512 | raise error.AutoservHostError( |
Kevin Cheng | 5f2ba6c | 2016-09-28 10:20:05 -0700 | [diff] [blame] | 513 | 'servo host %s failed to come back from reboot.' % |
| 514 | self.hostname) |
Richard Barnette | 3a7697f | 2016-04-20 11:33:27 -0700 | [diff] [blame] | 515 | return status, current_build_number |
| 516 | |
| 517 | |
Richard Barnette | 3a7697f | 2016-04-20 11:33:27 -0700 | [diff] [blame] | 518 | def update_image(self, wait_for_update=False): |
beeps | 5e8c45a | 2013-12-17 22:05:11 -0800 | [diff] [blame] | 519 | """Update the image on the servo host, if needed. |
| 520 | |
J. Richard Barnette | 8489539 | 2015-04-30 12:31:01 -0700 | [diff] [blame] | 521 | This method recognizes the following cases: |
| 522 | * If the Host is not running Chrome OS, do nothing. |
| 523 | * If a previously triggered update is now complete, reboot |
| 524 | to the new version. |
| 525 | * If the host is processing a previously triggered update, |
| 526 | do nothing. |
| 527 | * If the host is running a version of Chrome OS different |
| 528 | from the default for servo Hosts, trigger an update, but |
| 529 | don't wait for it to complete. |
beeps | 5e8c45a | 2013-12-17 22:05:11 -0800 | [diff] [blame] | 530 | |
Richard Barnette | 3a7697f | 2016-04-20 11:33:27 -0700 | [diff] [blame] | 531 | @param wait_for_update If an update needs to be applied and |
| 532 | this is true, then don't return until the update is |
| 533 | downloaded and finalized, and the host rebooted. |
beeps | 5e8c45a | 2013-12-17 22:05:11 -0800 | [diff] [blame] | 534 | @raises dev_server.DevServerException: If all the devservers are down. |
| 535 | @raises site_utils.ParseBuildNameException: If the devserver returns |
| 536 | an invalid build name. |
beeps | 5e8c45a | 2013-12-17 22:05:11 -0800 | [diff] [blame] | 537 | @raises AutoservRunError: If the update_engine_client isn't present on |
| 538 | the host, and the host is a cros_host. |
J. Richard Barnette | 8489539 | 2015-04-30 12:31:01 -0700 | [diff] [blame] | 539 | |
beeps | 5e8c45a | 2013-12-17 22:05:11 -0800 | [diff] [blame] | 540 | """ |
Dan Shi | b795b5a | 2015-09-24 13:26:35 -0700 | [diff] [blame] | 541 | # servod could be running in a Ubuntu workstation. |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 542 | if not self.is_cros_host(): |
beeps | 5e8c45a | 2013-12-17 22:05:11 -0800 | [diff] [blame] | 543 | logging.info('Not attempting an update, either %s is not running ' |
| 544 | 'chromeos or we cannot find enough information about ' |
| 545 | 'the host.', self.hostname) |
| 546 | return |
| 547 | |
Dan Shi | b795b5a | 2015-09-24 13:26:35 -0700 | [diff] [blame] | 548 | if lsbrelease_utils.is_moblab(): |
| 549 | logging.info('Not attempting an update, %s is running moblab.', |
| 550 | self.hostname) |
| 551 | return |
| 552 | |
Richard Barnette | 383ef9c | 2016-12-13 11:56:49 -0800 | [diff] [blame] | 553 | target_build = afe_utils.get_stable_cros_image_name(self.get_board()) |
Prathmesh Prabhu | cbd5ebb | 2018-08-28 17:04:50 -0700 | [diff] [blame] | 554 | target_build_number = server_utils.ParseBuildName( |
J. Richard Barnette | 8489539 | 2015-04-30 12:31:01 -0700 | [diff] [blame] | 555 | target_build)[3] |
xixuan | fa2d92a | 2016-12-09 09:45:27 -0800 | [diff] [blame] | 556 | # For servo image staging, we want it as more widely distributed as |
| 557 | # possible, so that devservers' load can be evenly distributed. So use |
| 558 | # hostname instead of target_build as hash. |
| 559 | ds = dev_server.ImageServer.resolve(self.hostname, |
| 560 | hostname=self.hostname) |
J. Richard Barnette | 8489539 | 2015-04-30 12:31:01 -0700 | [diff] [blame] | 561 | url = ds.get_update_url(target_build) |
beeps | 5e8c45a | 2013-12-17 22:05:11 -0800 | [diff] [blame] | 562 | |
| 563 | updater = autoupdater.ChromiumOSUpdater(update_url=url, host=self) |
Richard Barnette | 3a7697f | 2016-04-20 11:33:27 -0700 | [diff] [blame] | 564 | status, current_build_number = self._check_for_reboot(updater) |
| 565 | update_pending = True |
beeps | 5e8c45a | 2013-12-17 22:05:11 -0800 | [diff] [blame] | 566 | if status in autoupdater.UPDATER_PROCESSING_UPDATE: |
| 567 | logging.info('servo host %s already processing an update, update ' |
| 568 | 'engine client status=%s', self.hostname, status) |
Allen Li | 66aa254 | 2017-06-26 15:26:27 -0700 | [diff] [blame] | 569 | elif status == autoupdater.UPDATER_NEED_REBOOT: |
| 570 | return |
J. Richard Barnette | 8489539 | 2015-04-30 12:31:01 -0700 | [diff] [blame] | 571 | elif current_build_number != target_build_number: |
beeps | 5e8c45a | 2013-12-17 22:05:11 -0800 | [diff] [blame] | 572 | logging.info('Using devserver url: %s to trigger update on ' |
| 573 | 'servo host %s, from %s to %s', url, self.hostname, |
J. Richard Barnette | 8489539 | 2015-04-30 12:31:01 -0700 | [diff] [blame] | 574 | current_build_number, target_build_number) |
beeps | 5e8c45a | 2013-12-17 22:05:11 -0800 | [diff] [blame] | 575 | try: |
J. Richard Barnette | 8489539 | 2015-04-30 12:31:01 -0700 | [diff] [blame] | 576 | ds.stage_artifacts(target_build, |
| 577 | artifacts=['full_payload']) |
| 578 | except Exception as e: |
| 579 | logging.error('Staging artifacts failed: %s', str(e)) |
| 580 | logging.error('Abandoning update for this cycle.') |
beeps | 5e8c45a | 2013-12-17 22:05:11 -0800 | [diff] [blame] | 581 | else: |
J. Richard Barnette | 8489539 | 2015-04-30 12:31:01 -0700 | [diff] [blame] | 582 | try: |
| 583 | updater.trigger_update() |
| 584 | except autoupdater.RootFSUpdateError as e: |
| 585 | trigger_download_status = 'failed with %s' % str(e) |
Aviv Keshet | 1183632 | 2016-11-22 11:32:01 -0800 | [diff] [blame] | 586 | metrics.Counter('chromeos/autotest/servo/' |
| 587 | 'rootfs_update_failed').increment() |
J. Richard Barnette | 8489539 | 2015-04-30 12:31:01 -0700 | [diff] [blame] | 588 | else: |
| 589 | trigger_download_status = 'passed' |
| 590 | logging.info('Triggered download and update %s for %s, ' |
| 591 | 'update engine currently in status %s', |
| 592 | trigger_download_status, self.hostname, |
| 593 | updater.check_update_status()) |
beeps | 5e8c45a | 2013-12-17 22:05:11 -0800 | [diff] [blame] | 594 | else: |
| 595 | logging.info('servo host %s does not require an update.', |
| 596 | self.hostname) |
Richard Barnette | 3a7697f | 2016-04-20 11:33:27 -0700 | [diff] [blame] | 597 | update_pending = False |
| 598 | |
| 599 | if update_pending and wait_for_update: |
| 600 | logging.info('Waiting for servo update to complete.') |
| 601 | self.run('update_engine_client --follow', ignore_status=True) |
beeps | 5e8c45a | 2013-12-17 22:05:11 -0800 | [diff] [blame] | 602 | |
| 603 | |
Richard Barnette | 1edbb16 | 2016-11-01 11:47:50 -0700 | [diff] [blame] | 604 | def verify(self, silent=False): |
| 605 | """Update the servo host and verify it's in a good state. |
| 606 | |
| 607 | @param silent If true, suppress logging in `status.log`. |
| 608 | """ |
Richard Barnette | abbdc25 | 2018-07-26 16:57:42 -0700 | [diff] [blame] | 609 | message = 'Beginning verify for servo host %s port %s serial %s' |
| 610 | message %= (self.hostname, self.servo_port, self.servo_serial) |
| 611 | self.record('INFO', None, None, message) |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 612 | try: |
Richard Barnette | 1edbb16 | 2016-11-01 11:47:50 -0700 | [diff] [blame] | 613 | self._repair_strategy.verify(self, silent) |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 614 | except: |
| 615 | self.disconnect_servo() |
| 616 | raise |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 617 | |
| 618 | |
Richard Barnette | 1edbb16 | 2016-11-01 11:47:50 -0700 | [diff] [blame] | 619 | def repair(self, silent=False): |
| 620 | """Attempt to repair servo host. |
| 621 | |
| 622 | @param silent If true, suppress logging in `status.log`. |
| 623 | """ |
Richard Barnette | abbdc25 | 2018-07-26 16:57:42 -0700 | [diff] [blame] | 624 | message = 'Beginning repair for servo host %s port %s serial %s' |
| 625 | message %= (self.hostname, self.servo_port, self.servo_serial) |
| 626 | self.record('INFO', None, None, message) |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 627 | try: |
Richard Barnette | 1edbb16 | 2016-11-01 11:47:50 -0700 | [diff] [blame] | 628 | self._repair_strategy.repair(self, silent) |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 629 | except: |
| 630 | self.disconnect_servo() |
| 631 | raise |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 632 | |
| 633 | |
Fang Deng | d4fe739 | 2013-09-20 12:18:21 -0700 | [diff] [blame] | 634 | def has_power(self): |
| 635 | """Return whether or not the servo host is powered by PoE.""" |
| 636 | # TODO(fdeng): See crbug.com/302791 |
| 637 | # For now, assume all servo hosts in the lab have power. |
| 638 | return self.is_in_lab() |
| 639 | |
| 640 | |
| 641 | def power_cycle(self): |
| 642 | """Cycle power to this host via PoE if it is a lab device. |
| 643 | |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 644 | @raises AutoservRepairError if it fails to power cycle the |
Fang Deng | d4fe739 | 2013-09-20 12:18:21 -0700 | [diff] [blame] | 645 | servo host. |
| 646 | |
| 647 | """ |
| 648 | if self.has_power(): |
| 649 | try: |
| 650 | rpm_client.set_power(self.hostname, 'CYCLE') |
| 651 | except (socket.error, xmlrpclib.Error, |
| 652 | httplib.BadStatusLine, |
| 653 | rpm_client.RemotePowerException) as e: |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 654 | raise hosts.AutoservRepairError( |
Fang Deng | d4fe739 | 2013-09-20 12:18:21 -0700 | [diff] [blame] | 655 | 'Power cycling %s failed: %s' % (self.hostname, e)) |
| 656 | else: |
| 657 | logging.info('Skipping power cycling, not a lab device.') |
| 658 | |
| 659 | |
Dan Shi | 4d47852 | 2014-02-14 13:46:32 -0800 | [diff] [blame] | 660 | def get_servo(self): |
| 661 | """Get the cached servo.Servo object. |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 662 | |
Dan Shi | 4d47852 | 2014-02-14 13:46:32 -0800 | [diff] [blame] | 663 | @return: a servo.Servo object. |
Fang Deng | 5d518f4 | 2013-08-02 14:04:32 -0700 | [diff] [blame] | 664 | """ |
Dan Shi | 4d47852 | 2014-02-14 13:46:32 -0800 | [diff] [blame] | 665 | return self._servo |
| 666 | |
| 667 | |
Congbin Guo | a1f9cba | 2018-07-03 11:36:59 -0700 | [diff] [blame] | 668 | def close(self): |
| 669 | """Stop UART logging and close the host object.""" |
| 670 | if self._servo: |
Congbin Guo | 2e5e2a2 | 2018-07-27 10:32:48 -0700 | [diff] [blame] | 671 | # In some cases when we run as lab-tools, the job object is None. |
| 672 | if self.job: |
| 673 | self._servo.dump_uart_streams(self.job.resultdir) |
Congbin Guo | a1f9cba | 2018-07-03 11:36:59 -0700 | [diff] [blame] | 674 | self._servo.close() |
| 675 | |
| 676 | super(ServoHost, self).close() |
| 677 | |
| 678 | |
Richard Barnette | ea3e460 | 2016-06-10 12:36:41 -0700 | [diff] [blame] | 679 | def make_servo_hostname(dut_hostname): |
| 680 | """Given a DUT's hostname, return the hostname of its servo. |
| 681 | |
| 682 | @param dut_hostname: hostname of a DUT. |
| 683 | |
| 684 | @return hostname of the DUT's servo. |
| 685 | |
| 686 | """ |
| 687 | host_parts = dut_hostname.split('.') |
| 688 | host_parts[0] = host_parts[0] + '-servo' |
| 689 | return '.'.join(host_parts) |
| 690 | |
| 691 | |
| 692 | def servo_host_is_up(servo_hostname): |
Kevin Cheng | 5f2ba6c | 2016-09-28 10:20:05 -0700 | [diff] [blame] | 693 | """Given a servo host name, return if it's up or not. |
Richard Barnette | ea3e460 | 2016-06-10 12:36:41 -0700 | [diff] [blame] | 694 | |
| 695 | @param servo_hostname: hostname of the servo host. |
| 696 | |
| 697 | @return True if it's up, False otherwise |
| 698 | """ |
| 699 | # Technically, this duplicates the SSH ping done early in the servo |
| 700 | # proxy initialization code. However, this ping ends in a couple |
| 701 | # seconds when if fails, rather than the 60 seconds it takes to decide |
| 702 | # that an SSH ping has timed out. Specifically, that timeout happens |
| 703 | # when our servo DNS name resolves, but there is no host at that IP. |
| 704 | logging.info('Pinging servo host at %s', servo_hostname) |
| 705 | ping_config = ping_runner.PingConfig( |
| 706 | servo_hostname, count=3, |
| 707 | ignore_result=True, ignore_status=True) |
| 708 | return ping_runner.PingRunner().ping(ping_config).received > 0 |
| 709 | |
| 710 | |
Richard Barnette | e519dcd | 2016-08-15 17:37:17 -0700 | [diff] [blame] | 711 | def _map_afe_board_to_servo_board(afe_board): |
| 712 | """Map a board we get from the AFE to a servo appropriate value. |
| 713 | |
| 714 | Many boards are identical to other boards for servo's purposes. |
| 715 | This function makes that mapping. |
| 716 | |
| 717 | @param afe_board string board name received from AFE. |
| 718 | @return board we expect servo to have. |
| 719 | |
| 720 | """ |
| 721 | KNOWN_SUFFIXES = ['-freon', '_freon', '_moblab', '-cheets'] |
| 722 | BOARD_MAP = {'gizmo': 'panther'} |
| 723 | mapped_board = afe_board |
| 724 | if afe_board in BOARD_MAP: |
| 725 | mapped_board = BOARD_MAP[afe_board] |
| 726 | else: |
| 727 | for suffix in KNOWN_SUFFIXES: |
| 728 | if afe_board.endswith(suffix): |
| 729 | mapped_board = afe_board[0:-len(suffix)] |
| 730 | break |
| 731 | if mapped_board != afe_board: |
| 732 | logging.info('Mapping AFE board=%s to %s', afe_board, mapped_board) |
| 733 | return mapped_board |
| 734 | |
| 735 | |
Prathmesh Prabhu | b481023 | 2018-09-07 13:24:08 -0700 | [diff] [blame] | 736 | def get_servo_args_for_host(dut_host): |
Kevin Cheng | 5f2ba6c | 2016-09-28 10:20:05 -0700 | [diff] [blame] | 737 | """Return servo data associated with a given DUT. |
Richard Barnette | ea3e460 | 2016-06-10 12:36:41 -0700 | [diff] [blame] | 738 | |
Richard Barnette | ea3e460 | 2016-06-10 12:36:41 -0700 | [diff] [blame] | 739 | @param dut_host Instance of `Host` on which to find the servo |
| 740 | attributes. |
Prathmesh Prabhu | f605dd3 | 2018-08-28 17:09:04 -0700 | [diff] [blame] | 741 | @return `servo_args` dict with host and an optional port. |
Richard Barnette | ea3e460 | 2016-06-10 12:36:41 -0700 | [diff] [blame] | 742 | """ |
Prathmesh Prabhu | cba4429 | 2018-08-28 17:44:45 -0700 | [diff] [blame] | 743 | info = dut_host.host_info_store.get() |
| 744 | servo_args = {k: v for k, v in info.attributes.iteritems() |
| 745 | if k in SERVO_ATTR_KEYS} |
Richard Barnette | ea3e460 | 2016-06-10 12:36:41 -0700 | [diff] [blame] | 746 | |
Prathmesh Prabhu | cba4429 | 2018-08-28 17:44:45 -0700 | [diff] [blame] | 747 | if SERVO_PORT_ATTR in servo_args: |
| 748 | try: |
| 749 | servo_args[SERVO_PORT_ATTR] = int(servo_args[SERVO_PORT_ATTR]) |
| 750 | except ValueError: |
| 751 | logging.error('servo port is not an int: %s', |
| 752 | servo_args[SERVO_PORT_ATTR]) |
| 753 | # Reset servo_args because we don't want to use an invalid port. |
| 754 | servo_args.pop(SERVO_HOST_ATTR, None) |
| 755 | |
| 756 | if info.board: |
| 757 | servo_args[SERVO_BOARD_ATTR] = _map_afe_board_to_servo_board(info.board) |
Nick Sanders | 2f3c985 | 2018-10-24 12:10:24 -0700 | [diff] [blame] | 758 | if info.model: |
| 759 | servo_args[SERVO_MODEL_ATTR] = info.model |
Prathmesh Prabhu | 6f5f636 | 2018-09-05 17:20:31 -0700 | [diff] [blame] | 760 | return servo_args if SERVO_HOST_ATTR in servo_args else None |
Richard Barnette | ea3e460 | 2016-06-10 12:36:41 -0700 | [diff] [blame] | 761 | |
| 762 | |
Prathmesh Prabhu | efb1b48 | 2018-08-28 17:15:05 -0700 | [diff] [blame] | 763 | def _tweak_args_for_ssp_moblab(servo_args): |
| 764 | if servo_args[SERVO_HOST_ATTR] in ['localhost', '127.0.0.1']: |
| 765 | servo_args[SERVO_HOST_ATTR] = _CONFIG.get_config_value( |
| 766 | 'SSP', 'host_container_ip', type=str, default=None) |
| 767 | |
| 768 | |
Dan Shi | 023aae3 | 2016-05-25 11:13:01 -0700 | [diff] [blame] | 769 | def create_servo_host(dut, servo_args, try_lab_servo=False, |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 770 | try_servo_repair=False): |
Kevin Cheng | 5f2ba6c | 2016-09-28 10:20:05 -0700 | [diff] [blame] | 771 | """Create a ServoHost object for a given DUT, if appropriate. |
Dan Shi | 4d47852 | 2014-02-14 13:46:32 -0800 | [diff] [blame] | 772 | |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 773 | This function attempts to create and verify or repair a `ServoHost` |
| 774 | object for a servo connected to the given `dut`, subject to various |
| 775 | constraints imposed by the parameters: |
| 776 | * When the `servo_args` parameter is not `None`, a servo |
| 777 | host must be created, and must be checked with `repair()`. |
| 778 | * Otherwise, if a servo exists in the lab and `try_lab_servo` is |
| 779 | true: |
| 780 | * If `try_servo_repair` is true, then create a servo host and |
| 781 | check it with `repair()`. |
| 782 | * Otherwise, if the servo responds to `ping` then create a |
| 783 | servo host and check it with `verify()`. |
Fang Deng | e545abb | 2014-12-30 18:43:47 -0800 | [diff] [blame] | 784 | |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 785 | In cases where `servo_args` was not `None`, repair failure |
| 786 | exceptions are passed back to the caller; otherwise, exceptions |
Richard Barnette | 07c2e1d | 2016-10-26 14:24:28 -0700 | [diff] [blame] | 787 | are logged and then discarded. Note that this only happens in cases |
| 788 | where we're called from a test (not special task) control file that |
| 789 | has an explicit dependency on servo. In that case, we require that |
| 790 | repair not write to `status.log`, so as to avoid polluting test |
| 791 | results. |
| 792 | |
| 793 | TODO(jrbarnette): The special handling for servo in test control |
| 794 | files is a thorn in my flesh; I dearly hope to see it cut out before |
| 795 | my retirement. |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 796 | |
| 797 | Parameters for a servo host consist of a host name, port number, and |
| 798 | DUT board, and are determined from one of these sources, in order of |
| 799 | priority: |
Richard Barnette | ea3e460 | 2016-06-10 12:36:41 -0700 | [diff] [blame] | 800 | * Servo attributes from the `dut` parameter take precedence over |
| 801 | all other sources of information. |
| 802 | * If a DNS entry for the servo based on the DUT hostname exists in |
| 803 | the CrOS lab network, that hostname is used with the default |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 804 | port and the DUT's board. |
Richard Barnette | ea3e460 | 2016-06-10 12:36:41 -0700 | [diff] [blame] | 805 | * If no other options are found, the parameters will be taken |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 806 | from the `servo_args` dict passed in from the caller. |
Richard Barnette | ea3e460 | 2016-06-10 12:36:41 -0700 | [diff] [blame] | 807 | |
| 808 | @param dut An instance of `Host` from which to take |
| 809 | servo parameters (if available). |
| 810 | @param servo_args A dictionary with servo parameters to use if |
| 811 | they can't be found from `dut`. If this |
| 812 | argument is supplied, unrepaired exceptions |
| 813 | from `verify()` will be passed back to the |
| 814 | caller. |
| 815 | @param try_lab_servo If not true, servo host creation will be |
| 816 | skipped unless otherwise required by the |
| 817 | caller. |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 818 | @param try_servo_repair If true, check a servo host with |
| 819 | `repair()` instead of `verify()`. |
Dan Shi | 4d47852 | 2014-02-14 13:46:32 -0800 | [diff] [blame] | 820 | |
| 821 | @returns: A ServoHost object or None. See comments above. |
| 822 | |
| 823 | """ |
Richard Barnette | 07c2e1d | 2016-10-26 14:24:28 -0700 | [diff] [blame] | 824 | servo_dependency = servo_args is not None |
Richard Barnette | 07c2e1d | 2016-10-26 14:24:28 -0700 | [diff] [blame] | 825 | if dut is not None and (try_lab_servo or servo_dependency): |
Prathmesh Prabhu | b481023 | 2018-09-07 13:24:08 -0700 | [diff] [blame] | 826 | servo_args_override = get_servo_args_for_host(dut) |
Richard Barnette | ea3e460 | 2016-06-10 12:36:41 -0700 | [diff] [blame] | 827 | if servo_args_override is not None: |
Prathmesh Prabhu | efb1b48 | 2018-08-28 17:15:05 -0700 | [diff] [blame] | 828 | if utils.in_moblab_ssp(): |
| 829 | _tweak_args_for_ssp_moblab(servo_args_override) |
Prathmesh Prabhu | 88bf605 | 2018-08-28 16:21:26 -0700 | [diff] [blame] | 830 | logging.debug( |
| 831 | 'Overriding provided servo_args (%s) with arguments' |
| 832 | ' determined from the host (%s)', |
| 833 | servo_args, |
| 834 | servo_args_override, |
| 835 | ) |
Richard Barnette | ea3e460 | 2016-06-10 12:36:41 -0700 | [diff] [blame] | 836 | servo_args = servo_args_override |
Prathmesh Prabhu | cba4429 | 2018-08-28 17:44:45 -0700 | [diff] [blame] | 837 | |
Richard Barnette | ea3e460 | 2016-06-10 12:36:41 -0700 | [diff] [blame] | 838 | if servo_args is None: |
Prathmesh Prabhu | 88bf605 | 2018-08-28 16:21:26 -0700 | [diff] [blame] | 839 | logging.debug('No servo_args provided, and failed to find overrides.') |
Richard Barnette | ea3e460 | 2016-06-10 12:36:41 -0700 | [diff] [blame] | 840 | return None |
Prathmesh Prabhu | cba4429 | 2018-08-28 17:44:45 -0700 | [diff] [blame] | 841 | if SERVO_HOST_ATTR not in servo_args: |
| 842 | logging.debug('%s attribute missing from servo_args: %s', |
| 843 | SERVO_HOST_ATTR, servo_args) |
| 844 | return None |
Richard Barnette | 07c2e1d | 2016-10-26 14:24:28 -0700 | [diff] [blame] | 845 | if (not servo_dependency and not try_servo_repair and |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 846 | not servo_host_is_up(servo_args[SERVO_HOST_ATTR])): |
Prathmesh Prabhu | 88bf605 | 2018-08-28 16:21:26 -0700 | [diff] [blame] | 847 | logging.debug('ServoHost is not up.') |
Dan Shi | bbb0cb6 | 2014-03-24 17:50:57 -0700 | [diff] [blame] | 848 | return None |
Prathmesh Prabhu | 88bf605 | 2018-08-28 16:21:26 -0700 | [diff] [blame] | 849 | |
Prathmesh Prabhu | f605dd3 | 2018-08-28 17:09:04 -0700 | [diff] [blame] | 850 | newhost = ServoHost( |
| 851 | is_in_lab=(servo_args |
| 852 | and server_utils.host_in_lab( |
| 853 | servo_args[SERVO_HOST_ATTR])), |
| 854 | **servo_args |
| 855 | ) |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 856 | # Note that the logic of repair() includes everything done |
| 857 | # by verify(). It's sufficient to call one or the other; |
| 858 | # we don't need both. |
Richard Barnette | 07c2e1d | 2016-10-26 14:24:28 -0700 | [diff] [blame] | 859 | if servo_dependency: |
| 860 | newhost.repair(silent=True) |
Prathmesh Prabhu | 88bf605 | 2018-08-28 16:21:26 -0700 | [diff] [blame] | 861 | return newhost |
| 862 | |
| 863 | if try_servo_repair: |
| 864 | try: |
| 865 | newhost.repair() |
| 866 | except Exception: |
| 867 | logging.exception('servo repair failed for %s', newhost.hostname) |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 868 | else: |
| 869 | try: |
Prathmesh Prabhu | 88bf605 | 2018-08-28 16:21:26 -0700 | [diff] [blame] | 870 | newhost.verify() |
Kevin Cheng | 5f2ba6c | 2016-09-28 10:20:05 -0700 | [diff] [blame] | 871 | except Exception: |
Prathmesh Prabhu | 88bf605 | 2018-08-28 16:21:26 -0700 | [diff] [blame] | 872 | logging.exception('servo verify failed for %s', newhost.hostname) |
Richard Barnette | 9a26ad6 | 2016-06-10 12:03:08 -0700 | [diff] [blame] | 873 | return newhost |