blob: 7f779b3bdf602f849c5a2968daf92f54b67279ad [file] [log] [blame]
Fang Deng5d518f42013-08-02 14:04:32 -07001# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4#
5# Expects to be run in an environment with sudo and no interactive password
6# prompt, such as within the Chromium OS development chroot.
7
8
9"""This file provides core logic for servo verify/repair process."""
10
11
12import httplib
13import logging
14import socket
Fang Deng5d518f42013-08-02 14:04:32 -070015import xmlrpclib
Raul E Rangel52ca2e82018-07-03 14:10:14 -060016import os
Fang Deng5d518f42013-08-02 14:04:32 -070017
18from autotest_lib.client.bin import utils
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -070019from autotest_lib.client.common_lib import control_data
Fang Deng5d518f42013-08-02 14:04:32 -070020from autotest_lib.client.common_lib import error
beeps5e8c45a2013-12-17 22:05:11 -080021from autotest_lib.client.common_lib import global_config
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -070022from autotest_lib.client.common_lib import host_states
Richard Barnette9a26ad62016-06-10 12:03:08 -070023from autotest_lib.client.common_lib import hosts
Dan Shi0942b1d2015-03-31 11:07:00 -070024from autotest_lib.client.common_lib import lsbrelease_utils
beeps5e8c45a2013-12-17 22:05:11 -080025from autotest_lib.client.common_lib.cros import dev_server
Fang Deng5d518f42013-08-02 14:04:32 -070026from autotest_lib.client.common_lib.cros import retry
Christopher Wileycef1f902014-06-19 11:11:23 -070027from autotest_lib.client.common_lib.cros.network import ping_runner
Hsinyu Chaoe0b08e62015-08-11 10:50:37 +000028from autotest_lib.client.cros import constants as client_constants
Richard Barnettee519dcd2016-08-15 17:37:17 -070029from autotest_lib.server import afe_utils
Prathmesh Prabhucbd5ebb2018-08-28 17:04:50 -070030from autotest_lib.server import site_utils as server_utils
Richard Barnetted31580e2018-05-14 19:58:00 +000031from autotest_lib.server.cros import autoupdater
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -070032from autotest_lib.server.cros.dynamic_suite import control_file_getter
Richard Barnetted31580e2018-05-14 19:58:00 +000033from autotest_lib.server.cros.dynamic_suite import frontend_wrappers
Richard Barnette9a26ad62016-06-10 12:03:08 -070034from autotest_lib.server.cros.servo import servo
Richard Barnetted31580e2018-05-14 19:58:00 +000035from autotest_lib.server.hosts import servo_repair
Fang Deng5d518f42013-08-02 14:04:32 -070036from autotest_lib.server.hosts import ssh_host
Fang Dengd4fe7392013-09-20 12:18:21 -070037from autotest_lib.site_utils.rpm_control_system import rpm_client
Fang Deng5d518f42013-08-02 14:04:32 -070038
Dan Shi5e2efb72017-02-07 11:40:23 -080039try:
40 from chromite.lib import metrics
41except ImportError:
42 metrics = utils.metrics_mock
43
Fang Deng5d518f42013-08-02 14:04:32 -070044
Simran Basi0739d682015-02-25 16:22:56 -080045# Names of the host attributes in the database that represent the values for
46# the servo_host and servo_port for a servo connected to the DUT.
47SERVO_HOST_ATTR = 'servo_host'
48SERVO_PORT_ATTR = 'servo_port'
Richard Barnettee519dcd2016-08-15 17:37:17 -070049SERVO_BOARD_ATTR = 'servo_board'
Nick Sanders2f3c9852018-10-24 12:10:24 -070050# Model is inferred from host labels.
51SERVO_MODEL_ATTR = 'servo_model'
Kevin Cheng643ce8a2016-09-15 15:42:12 -070052SERVO_SERIAL_ATTR = 'servo_serial'
Prathmesh Prabhucba44292018-08-28 17:44:45 -070053SERVO_ATTR_KEYS = (
54 SERVO_BOARD_ATTR,
55 SERVO_HOST_ATTR,
56 SERVO_PORT_ATTR,
57 SERVO_SERIAL_ATTR,
58)
Simran Basi0739d682015-02-25 16:22:56 -080059
Dan Shi3b2adf62015-09-02 17:46:54 -070060_CONFIG = global_config.global_config
xixuan6cf6d2f2016-01-29 15:29:00 -080061ENABLE_SSH_TUNNEL_FOR_SERVO = _CONFIG.get_config_value(
62 'CROS', 'enable_ssh_tunnel_for_servo', type=bool, default=False)
Simran Basi0739d682015-02-25 16:22:56 -080063
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -070064AUTOTEST_BASE = _CONFIG.get_config_value(
65 'SCHEDULER', 'drone_installation_directory',
66 default='/usr/local/autotest')
67
68_SERVO_HOST_REBOOT_TEST_NAME = 'servohost_Reboot'
Kevin Cheng55265902016-10-19 12:46:50 -070069_SERVO_HOST_FORCE_REBOOT_TEST_NAME = 'servohost_Reboot.force_reboot'
Fang Deng5d518f42013-08-02 14:04:32 -070070
Fang Deng5d518f42013-08-02 14:04:32 -070071class ServoHost(ssh_host.SSHHost):
72 """Host class for a host that controls a servo, e.g. beaglebone."""
73
Raul E Rangel52ca2e82018-07-03 14:10:14 -060074 DEFAULT_PORT = int(os.getenv('SERVOD_PORT', '9999'))
Richard Barnette9a26ad62016-06-10 12:03:08 -070075
Dan Shie5b3c512014-08-21 12:12:09 -070076 # Timeout for initializing servo signals.
Wai-Hong Tam37b6ed32017-09-19 15:52:39 -070077 INITIALIZE_SERVO_TIMEOUT_SECS = 60
Richard Barnette9a26ad62016-06-10 12:03:08 -070078
xixuan6cf6d2f2016-01-29 15:29:00 -080079 # Ready test function
80 SERVO_READY_METHOD = 'get_version'
Fang Deng5d518f42013-08-02 14:04:32 -070081
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -070082 REBOOT_CMD = 'sleep 1; reboot & sleep 10; reboot -f'
83
Fang Deng5d518f42013-08-02 14:04:32 -070084
Richard Barnette17bfc6c2016-08-04 18:41:43 -070085 def _initialize(self, servo_host='localhost',
Richard Barnettee519dcd2016-08-15 17:37:17 -070086 servo_port=DEFAULT_PORT, servo_board=None,
Nick Sanders2f3c9852018-10-24 12:10:24 -070087 servo_model=None, servo_serial=None, is_in_lab=None,
88 *args, **dargs):
Fang Deng5d518f42013-08-02 14:04:32 -070089 """Initialize a ServoHost instance.
90
91 A ServoHost instance represents a host that controls a servo.
92
93 @param servo_host: Name of the host where the servod process
94 is running.
Raul E Rangel52ca2e82018-07-03 14:10:14 -060095 @param servo_port: Port the servod process is listening on. Defaults
96 to the SERVOD_PORT environment variable if set,
97 otherwise 9999.
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -070098 @param servo_board: Board that the servo is connected to.
Nick Sanders2f3c9852018-10-24 12:10:24 -070099 @param servo_model: Model that the servo is connected to.
Dan Shi4d478522014-02-14 13:46:32 -0800100 @param is_in_lab: True if the servo host is in Cros Lab. Default is set
101 to None, for which utils.host_is_in_lab_zone will be
102 called to check if the servo host is in Cros lab.
Fang Deng5d518f42013-08-02 14:04:32 -0700103
104 """
105 super(ServoHost, self)._initialize(hostname=servo_host,
106 *args, **dargs)
Richard Barnette42f4db92018-08-23 15:05:15 -0700107 self.servo_port = int(servo_port)
Richard Barnettee519dcd2016-08-15 17:37:17 -0700108 self.servo_board = servo_board
Nick Sanders2f3c9852018-10-24 12:10:24 -0700109 self.servo_model = servo_model
Kevin Cheng643ce8a2016-09-15 15:42:12 -0700110 self.servo_serial = servo_serial
Richard Barnettee519dcd2016-08-15 17:37:17 -0700111 self._servo = None
Richard Barnette9a26ad62016-06-10 12:03:08 -0700112 self._repair_strategy = (
113 servo_repair.create_servo_repair_strategy())
Richard Barnettee519dcd2016-08-15 17:37:17 -0700114 self._is_localhost = (self.hostname == 'localhost')
115 if self._is_localhost:
116 self._is_in_lab = False
117 elif is_in_lab is None:
Dan Shi4d478522014-02-14 13:46:32 -0800118 self._is_in_lab = utils.host_is_in_lab_zone(self.hostname)
119 else:
120 self._is_in_lab = is_in_lab
xixuan6cf6d2f2016-01-29 15:29:00 -0800121
Richard Barnettee519dcd2016-08-15 17:37:17 -0700122 # Commands on the servo host must be run by the superuser.
123 # Our account on a remote host is root, but if our target is
124 # localhost then we might be running unprivileged. If so,
125 # `sudo` will have to be added to the commands.
Fang Deng5d518f42013-08-02 14:04:32 -0700126 if self._is_localhost:
127 self._sudo_required = utils.system_output('id -u') != '0'
128 else:
129 self._sudo_required = False
Richard Barnettee519dcd2016-08-15 17:37:17 -0700130
Richard Barnette9a26ad62016-06-10 12:03:08 -0700131
132 def connect_servo(self):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700133 """Establish a connection to the servod server on this host.
Richard Barnette9a26ad62016-06-10 12:03:08 -0700134
135 Initializes `self._servo` and then verifies that all network
136 connections are working. This will create an ssh tunnel if
137 it's required.
138
139 As a side effect of testing the connection, all signals on the
140 target servo are reset to default values, and the USB stick is
141 set to the neutral (off) position.
142 """
Kevin Cheng643ce8a2016-09-15 15:42:12 -0700143 servo_obj = servo.Servo(servo_host=self, servo_serial=self.servo_serial)
Richard Barnette9a26ad62016-06-10 12:03:08 -0700144 timeout, _ = retry.timeout(
145 servo_obj.initialize_dut,
146 timeout_sec=self.INITIALIZE_SERVO_TIMEOUT_SECS)
147 if timeout:
148 raise hosts.AutoservVerifyError(
149 'Servo initialize timed out.')
150 self._servo = servo_obj
151
152
153 def disconnect_servo(self):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700154 """Disconnect our servo if it exists.
Richard Barnette9a26ad62016-06-10 12:03:08 -0700155
156 If we've previously successfully connected to our servo,
157 disconnect any established ssh tunnel, and set `self._servo`
158 back to `None`.
159 """
160 if self._servo:
161 # N.B. This call is safe even without a tunnel:
162 # rpc_server_tracker.disconnect() silently ignores
163 # unknown ports.
164 self.rpc_server_tracker.disconnect(self.servo_port)
165 self._servo = None
Fang Deng5d518f42013-08-02 14:04:32 -0700166
167
168 def is_in_lab(self):
169 """Check whether the servo host is a lab device.
170
171 @returns: True if the servo host is in Cros Lab, otherwise False.
172
173 """
174 return self._is_in_lab
175
176
177 def is_localhost(self):
178 """Checks whether the servo host points to localhost.
179
180 @returns: True if it points to localhost, otherwise False.
181
182 """
183 return self._is_localhost
184
185
186 def get_servod_server_proxy(self):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700187 """Return a proxy that can be used to communicate with servod server.
Fang Deng5d518f42013-08-02 14:04:32 -0700188
189 @returns: An xmlrpclib.ServerProxy that is connected to the servod
190 server on the host.
Fang Deng5d518f42013-08-02 14:04:32 -0700191 """
Richard Barnette9a26ad62016-06-10 12:03:08 -0700192 if ENABLE_SSH_TUNNEL_FOR_SERVO and not self.is_localhost():
193 return self.rpc_server_tracker.xmlrpc_connect(
194 None, self.servo_port,
195 ready_test_name=self.SERVO_READY_METHOD,
Allen Li2b1a8992018-11-27 14:17:18 -0800196 timeout_seconds=60,
Allen Li556f4532018-12-03 18:11:23 -0800197 request_timeout_seconds=3600)
Richard Barnette9a26ad62016-06-10 12:03:08 -0700198 else:
199 remote = 'http://%s:%s' % (self.hostname, self.servo_port)
200 return xmlrpclib.ServerProxy(remote)
Fang Deng5d518f42013-08-02 14:04:32 -0700201
202
Richard Barnette9a26ad62016-06-10 12:03:08 -0700203 def is_cros_host(self):
beeps5e8c45a2013-12-17 22:05:11 -0800204 """Check if a servo host is running chromeos.
205
206 @return: True if the servo host is running chromeos.
207 False if it isn't, or we don't have enough information.
208 """
209 try:
210 result = self.run('grep -q CHROMEOS /etc/lsb-release',
211 ignore_status=True, timeout=10)
212 except (error.AutoservRunError, error.AutoservSSHTimeout):
213 return False
214 return result.exit_status == 0
215
216
Fang Deng5d518f42013-08-02 14:04:32 -0700217 def make_ssh_command(self, user='root', port=22, opts='', hosts_file=None,
Dean Liaoe3e75f62017-11-14 10:36:43 +0800218 connect_timeout=None, alive_interval=None,
219 alive_count_max=None, connection_attempts=None):
Fang Deng5d518f42013-08-02 14:04:32 -0700220 """Override default make_ssh_command to use tuned options.
221
222 Tuning changes:
223 - ConnectTimeout=30; maximum of 30 seconds allowed for an SSH
224 connection failure. Consistency with remote_access.py.
225
226 - ServerAliveInterval=180; which causes SSH to ping connection every
227 180 seconds. In conjunction with ServerAliveCountMax ensures
228 that if the connection dies, Autotest will bail out quickly.
229
230 - ServerAliveCountMax=3; consistency with remote_access.py.
231
232 - ConnectAttempts=4; reduce flakiness in connection errors;
233 consistency with remote_access.py.
234
235 - UserKnownHostsFile=/dev/null; we don't care about the keys.
236
237 - SSH protocol forced to 2; needed for ServerAliveInterval.
238
239 @param user User name to use for the ssh connection.
240 @param port Port on the target host to use for ssh connection.
241 @param opts Additional options to the ssh command.
242 @param hosts_file Ignored.
243 @param connect_timeout Ignored.
244 @param alive_interval Ignored.
Dean Liaoe3e75f62017-11-14 10:36:43 +0800245 @param alive_count_max Ignored.
246 @param connection_attempts Ignored.
Fang Deng5d518f42013-08-02 14:04:32 -0700247
248 @returns: An ssh command with the requested settings.
249
250 """
Dean Liaoe3e75f62017-11-14 10:36:43 +0800251 options = ' '.join([opts, '-o Protocol=2'])
252 return super(ServoHost, self).make_ssh_command(
253 user=user, port=port, opts=options, hosts_file='/dev/null',
254 connect_timeout=30, alive_interval=180, alive_count_max=3,
255 connection_attempts=4)
Fang Deng5d518f42013-08-02 14:04:32 -0700256
257
258 def _make_scp_cmd(self, sources, dest):
259 """Format scp command.
260
261 Given a list of source paths and a destination path, produces the
262 appropriate scp command for encoding it. Remote paths must be
263 pre-encoded. Overrides _make_scp_cmd in AbstractSSHHost
264 to allow additional ssh options.
265
266 @param sources: A list of source paths to copy from.
267 @param dest: Destination path to copy to.
268
269 @returns: An scp command that copies |sources| on local machine to
270 |dest| on the remote servo host.
271
272 """
273 command = ('scp -rq %s -o BatchMode=yes -o StrictHostKeyChecking=no '
274 '-o UserKnownHostsFile=/dev/null -P %d %s "%s"')
275 return command % (self.master_ssh_option,
276 self.port, ' '.join(sources), dest)
277
278
279 def run(self, command, timeout=3600, ignore_status=False,
280 stdout_tee=utils.TEE_TO_LOGS, stderr_tee=utils.TEE_TO_LOGS,
Luigi Semenzatobfbd1f32017-01-06 10:41:18 -0800281 connect_timeout=30, ssh_failure_retry_ok=False,
282 options='', stdin=None, verbose=True, args=()):
Fang Deng5d518f42013-08-02 14:04:32 -0700283 """Run a command on the servo host.
284
285 Extends method `run` in SSHHost. If the servo host is a remote device,
286 it will call `run` in SSHost without changing anything.
287 If the servo host is 'localhost', it will call utils.system_output.
288
289 @param command: The command line string.
290 @param timeout: Time limit in seconds before attempting to
291 kill the running process. The run() function
292 will take a few seconds longer than 'timeout'
293 to complete if it has to kill the process.
294 @param ignore_status: Do not raise an exception, no matter
295 what the exit code of the command is.
296 @param stdout_tee/stderr_tee: Where to tee the stdout/stderr.
297 @param connect_timeout: SSH connection timeout (in seconds)
298 Ignored if host is 'localhost'.
299 @param options: String with additional ssh command options
300 Ignored if host is 'localhost'.
Luigi Semenzatobfbd1f32017-01-06 10:41:18 -0800301 @param ssh_failure_retry_ok: when True and ssh connection failure is
302 suspected, OK to retry command (but not
303 compulsory, and likely not needed here)
Fang Deng5d518f42013-08-02 14:04:32 -0700304 @param stdin: Stdin to pass (a string) to the executed command.
305 @param verbose: Log the commands.
306 @param args: Sequence of strings to pass as arguments to command by
307 quoting them in " and escaping their contents if necessary.
308
309 @returns: A utils.CmdResult object.
310
311 @raises AutoservRunError if the command failed.
312 @raises AutoservSSHTimeout SSH connection has timed out. Only applies
313 when servo host is not 'localhost'.
314
315 """
316 run_args = {'command': command, 'timeout': timeout,
317 'ignore_status': ignore_status, 'stdout_tee': stdout_tee,
318 'stderr_tee': stderr_tee, 'stdin': stdin,
319 'verbose': verbose, 'args': args}
320 if self.is_localhost():
321 if self._sudo_required:
Michael Tangf9b3ada2016-11-18 16:01:05 -0800322 run_args['command'] = 'sudo -n sh -c "%s"' % utils.sh_escape(
323 command)
Fang Deng5d518f42013-08-02 14:04:32 -0700324 try:
325 return utils.run(**run_args)
326 except error.CmdError as e:
327 logging.error(e)
328 raise error.AutoservRunError('command execution error',
329 e.result_obj)
330 else:
331 run_args['connect_timeout'] = connect_timeout
332 run_args['options'] = options
333 return super(ServoHost, self).run(**run_args)
334
335
Richard Barnette9a26ad62016-06-10 12:03:08 -0700336 def _get_release_version(self):
Dan Shi0942b1d2015-03-31 11:07:00 -0700337 """Get the value of attribute CHROMEOS_RELEASE_VERSION from lsb-release.
338
339 @returns The version string in lsb-release, under attribute
340 CHROMEOS_RELEASE_VERSION.
341 """
342 lsb_release_content = self.run(
343 'cat "%s"' % client_constants.LSB_RELEASE).stdout.strip()
344 return lsbrelease_utils.get_chromeos_release_version(
345 lsb_release_content=lsb_release_content)
346
347
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700348 def get_attached_duts(self, afe):
349 """Gather a list of duts that use this servo host.
350
351 @param afe: afe instance.
352
353 @returns list of duts.
Richard Barnette3a7697f2016-04-20 11:33:27 -0700354 """
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700355 return afe.get_hosts_by_attribute(
356 attribute=SERVO_HOST_ATTR, value=self.hostname)
357
358
359 def get_board(self):
360 """Determine the board for this servo host.
361
362 @returns a string representing this servo host's board.
363 """
364 return lsbrelease_utils.get_current_board(
365 lsb_release_content=self.run('cat /etc/lsb-release').stdout)
366
367
368 def _choose_dut_for_synchronized_reboot(self, dut_list, afe):
369 """Choose which dut to schedule servo host reboot job.
370
371 We'll want a semi-deterministic way of selecting which host should be
372 scheduled for the servo host reboot job. For now we'll sort the
373 list with the expectation the dut list will stay consistent.
374 From there we'll grab the first dut that is available so we
375 don't schedule a job on a dut that will never run.
376
377 @param dut_list: List of the dut hostnames to choose from.
378 @param afe: Instance of the AFE.
379
380 @return hostname of dut to schedule job on.
381 """
382 afe_hosts = afe.get_hosts(dut_list)
383 afe_hosts.sort()
384 for afe_host in afe_hosts:
385 if afe_host.status not in host_states.UNAVAILABLE_STATES:
386 return afe_host.hostname
387 # If they're all unavailable, just return the first sorted dut.
388 dut_list.sort()
389 return dut_list[0]
390
391
392 def _sync_job_scheduled_for_duts(self, dut_list, afe):
393 """Checks if a synchronized reboot has been scheduled for these duts.
394
395 Grab all the host queue entries that aren't completed for the duts and
396 see if any of them have the expected job name.
397
398 @param dut_list: List of duts to check on.
399 @param afe: Instance of the AFE.
400
401 @returns True if the job is scheduled, False otherwise.
402 """
403 afe_hosts = afe.get_hosts(dut_list)
404 for afe_host in afe_hosts:
405 hqes = afe.get_host_queue_entries(host=afe_host.id, complete=0)
406 for hqe in hqes:
407 job = afe.get_jobs(id=hqe.job.id)
Kevin Cheng55265902016-10-19 12:46:50 -0700408 if job and job[0].name in (_SERVO_HOST_REBOOT_TEST_NAME,
409 _SERVO_HOST_FORCE_REBOOT_TEST_NAME):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700410 return True
411 return False
412
413
Kevin Cheng55265902016-10-19 12:46:50 -0700414 def schedule_synchronized_reboot(self, dut_list, afe, force_reboot=False):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700415 """Schedule a job to reboot the servo host.
416
417 When we schedule a job, it will create a ServoHost object which will
418 go through this entire flow of checking if a reboot is needed and
419 trying to schedule it. There is probably a better approach to setting
420 up a synchronized reboot but I'm coming up short on better ideas so I
421 apologize for this circus show.
422
Kevin Cheng55265902016-10-19 12:46:50 -0700423 @param dut_list: List of duts that need to be locked.
424 @param afe: Instance of afe.
425 @param force_reboot: Boolean to indicate if a forced reboot should be
426 scheduled or not.
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700427 """
428 # If we've already scheduled job on a dut, we're done here.
429 if self._sync_job_scheduled_for_duts(dut_list, afe):
430 return
431
432 # Looks like we haven't scheduled a job yet.
Kevin Cheng55265902016-10-19 12:46:50 -0700433 test = (_SERVO_HOST_REBOOT_TEST_NAME if not force_reboot
434 else _SERVO_HOST_FORCE_REBOOT_TEST_NAME)
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700435 dut = self._choose_dut_for_synchronized_reboot(dut_list, afe)
436 getter = control_file_getter.FileSystemGetter([AUTOTEST_BASE])
Kevin Cheng55265902016-10-19 12:46:50 -0700437 control_file = getter.get_control_file_contents_by_name(test)
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700438 control_type = control_data.CONTROL_TYPE_NAMES.SERVER
Kevin Cheng79589982016-10-25 13:26:04 -0700439 try:
440 afe.create_job(control_file=control_file, name=test,
441 control_type=control_type, hosts=[dut])
442 except Exception as e:
443 # Sometimes creating the job will raise an exception. We'll log it
444 # but we don't want to fail because of it.
Aviv Keshet5ae0a002017-05-05 10:23:33 -0700445 logging.exception('Scheduling reboot job failed due to Exception.')
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700446
447
448 def reboot(self, *args, **dargs):
449 """Reboot using special servo host reboot command."""
450 super(ServoHost, self).reboot(reboot_cmd=self.REBOOT_CMD,
451 *args, **dargs)
452
453
454 def _check_for_reboot(self, updater):
455 """Reboot this servo host if an upgrade is waiting.
Richard Barnette3a7697f2016-04-20 11:33:27 -0700456
457 If the host has successfully downloaded and finalized a new
458 build, reboot.
459
460 @param updater: a ChromiumOSUpdater instance for checking
461 whether reboot is needed.
Richard Barnette3a7697f2016-04-20 11:33:27 -0700462 """
Prathmesh Prabhu460905d2018-12-27 13:10:00 -0800463 if updater.check_update_status() != autoupdater.UPDATER_NEED_REBOOT:
464 return
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700465
Prathmesh Prabhu460905d2018-12-27 13:10:00 -0800466 # Check if we need to schedule an organized reboot.
467 afe = frontend_wrappers.RetryingAFE(
468 timeout_min=5, delay_sec=10,
469 server=server_utils.get_global_afe_hostname())
470 dut_list = self.get_attached_duts(afe)
471 logging.info('servo host has the following duts: %s', dut_list)
472 if len(dut_list) > 1:
473 logging.info('servo host has multiple duts, scheduling '
474 'synchronized reboot')
475 self.schedule_synchronized_reboot(dut_list, afe)
476 return
Richard Barnette3a7697f2016-04-20 11:33:27 -0700477
Prathmesh Prabhu460905d2018-12-27 13:10:00 -0800478 logging.info('Rebooting servo host %s from build %s',
479 self.hostname, self._get_release_version())
480 # Tell the reboot() call not to wait for completion.
481 # Otherwise, the call will log reboot failure if servo does
482 # not come back. The logged reboot failure will lead to
483 # test job failure. If the test does not require servo, we
484 # don't want servo failure to fail the test with error:
485 # `Host did not return from reboot` in status.log.
486 self.reboot(fastsync=True, wait=False)
487
488 # We told the reboot() call not to wait, but we need to wait
489 # for the reboot before we continue. Alas. The code from
490 # here below is basically a copy of Host.wait_for_restart(),
491 # with the logging bits ripped out, so that they can't cause
492 # the failure logging problem described above.
493 #
494 # The black stain that this has left on my soul can never be
495 # erased.
496 old_boot_id = self.get_boot_id()
497 if not self.wait_down(timeout=self.WAIT_DOWN_REBOOT_TIMEOUT,
498 warning_timer=self.WAIT_DOWN_REBOOT_WARNING,
499 old_boot_id=old_boot_id):
500 raise error.AutoservHostError(
501 'servo host %s failed to shut down.' %
502 self.hostname)
503 if self.wait_up(timeout=120):
504 logging.info('servo host %s back from reboot, with build %s',
505 self.hostname, self._get_release_version())
506 else:
507 raise error.AutoservHostError(
508 'servo host %s failed to come back from reboot.' %
509 self.hostname)
Richard Barnette3a7697f2016-04-20 11:33:27 -0700510
511
Richard Barnette3a7697f2016-04-20 11:33:27 -0700512 def update_image(self, wait_for_update=False):
beeps5e8c45a2013-12-17 22:05:11 -0800513 """Update the image on the servo host, if needed.
514
J. Richard Barnette84895392015-04-30 12:31:01 -0700515 This method recognizes the following cases:
516 * If the Host is not running Chrome OS, do nothing.
517 * If a previously triggered update is now complete, reboot
518 to the new version.
519 * If the host is processing a previously triggered update,
520 do nothing.
521 * If the host is running a version of Chrome OS different
522 from the default for servo Hosts, trigger an update, but
523 don't wait for it to complete.
beeps5e8c45a2013-12-17 22:05:11 -0800524
Richard Barnette3a7697f2016-04-20 11:33:27 -0700525 @param wait_for_update If an update needs to be applied and
526 this is true, then don't return until the update is
527 downloaded and finalized, and the host rebooted.
beeps5e8c45a2013-12-17 22:05:11 -0800528 @raises dev_server.DevServerException: If all the devservers are down.
529 @raises site_utils.ParseBuildNameException: If the devserver returns
530 an invalid build name.
beeps5e8c45a2013-12-17 22:05:11 -0800531 @raises AutoservRunError: If the update_engine_client isn't present on
532 the host, and the host is a cros_host.
J. Richard Barnette84895392015-04-30 12:31:01 -0700533
beeps5e8c45a2013-12-17 22:05:11 -0800534 """
Dan Shib795b5a2015-09-24 13:26:35 -0700535 # servod could be running in a Ubuntu workstation.
Richard Barnette9a26ad62016-06-10 12:03:08 -0700536 if not self.is_cros_host():
beeps5e8c45a2013-12-17 22:05:11 -0800537 logging.info('Not attempting an update, either %s is not running '
538 'chromeos or we cannot find enough information about '
539 'the host.', self.hostname)
540 return
541
Dan Shib795b5a2015-09-24 13:26:35 -0700542 if lsbrelease_utils.is_moblab():
543 logging.info('Not attempting an update, %s is running moblab.',
544 self.hostname)
545 return
546
Richard Barnette383ef9c2016-12-13 11:56:49 -0800547 target_build = afe_utils.get_stable_cros_image_name(self.get_board())
Prathmesh Prabhucbd5ebb2018-08-28 17:04:50 -0700548 target_build_number = server_utils.ParseBuildName(
J. Richard Barnette84895392015-04-30 12:31:01 -0700549 target_build)[3]
xixuanfa2d92a2016-12-09 09:45:27 -0800550 # For servo image staging, we want it as more widely distributed as
551 # possible, so that devservers' load can be evenly distributed. So use
552 # hostname instead of target_build as hash.
553 ds = dev_server.ImageServer.resolve(self.hostname,
554 hostname=self.hostname)
J. Richard Barnette84895392015-04-30 12:31:01 -0700555 url = ds.get_update_url(target_build)
beeps5e8c45a2013-12-17 22:05:11 -0800556
557 updater = autoupdater.ChromiumOSUpdater(update_url=url, host=self)
Prathmesh Prabhuec8414f2018-12-27 13:08:47 -0800558 self._check_for_reboot(updater)
559 current_build_number = self._get_release_version()
560 status = updater.check_update_status()
Richard Barnette3a7697f2016-04-20 11:33:27 -0700561 update_pending = True
beeps5e8c45a2013-12-17 22:05:11 -0800562 if status in autoupdater.UPDATER_PROCESSING_UPDATE:
563 logging.info('servo host %s already processing an update, update '
564 'engine client status=%s', self.hostname, status)
Allen Li66aa2542017-06-26 15:26:27 -0700565 elif status == autoupdater.UPDATER_NEED_REBOOT:
566 return
J. Richard Barnette84895392015-04-30 12:31:01 -0700567 elif current_build_number != target_build_number:
beeps5e8c45a2013-12-17 22:05:11 -0800568 logging.info('Using devserver url: %s to trigger update on '
569 'servo host %s, from %s to %s', url, self.hostname,
J. Richard Barnette84895392015-04-30 12:31:01 -0700570 current_build_number, target_build_number)
beeps5e8c45a2013-12-17 22:05:11 -0800571 try:
J. Richard Barnette84895392015-04-30 12:31:01 -0700572 ds.stage_artifacts(target_build,
573 artifacts=['full_payload'])
574 except Exception as e:
575 logging.error('Staging artifacts failed: %s', str(e))
576 logging.error('Abandoning update for this cycle.')
beeps5e8c45a2013-12-17 22:05:11 -0800577 else:
J. Richard Barnette84895392015-04-30 12:31:01 -0700578 try:
579 updater.trigger_update()
580 except autoupdater.RootFSUpdateError as e:
581 trigger_download_status = 'failed with %s' % str(e)
Aviv Keshet11836322016-11-22 11:32:01 -0800582 metrics.Counter('chromeos/autotest/servo/'
583 'rootfs_update_failed').increment()
J. Richard Barnette84895392015-04-30 12:31:01 -0700584 else:
585 trigger_download_status = 'passed'
586 logging.info('Triggered download and update %s for %s, '
587 'update engine currently in status %s',
588 trigger_download_status, self.hostname,
589 updater.check_update_status())
beeps5e8c45a2013-12-17 22:05:11 -0800590 else:
591 logging.info('servo host %s does not require an update.',
592 self.hostname)
Richard Barnette3a7697f2016-04-20 11:33:27 -0700593 update_pending = False
594
595 if update_pending and wait_for_update:
596 logging.info('Waiting for servo update to complete.')
597 self.run('update_engine_client --follow', ignore_status=True)
beeps5e8c45a2013-12-17 22:05:11 -0800598
599
Richard Barnette1edbb162016-11-01 11:47:50 -0700600 def verify(self, silent=False):
601 """Update the servo host and verify it's in a good state.
602
603 @param silent If true, suppress logging in `status.log`.
604 """
Richard Barnetteabbdc252018-07-26 16:57:42 -0700605 message = 'Beginning verify for servo host %s port %s serial %s'
606 message %= (self.hostname, self.servo_port, self.servo_serial)
607 self.record('INFO', None, None, message)
Richard Barnette9a26ad62016-06-10 12:03:08 -0700608 try:
Richard Barnette1edbb162016-11-01 11:47:50 -0700609 self._repair_strategy.verify(self, silent)
Richard Barnette9a26ad62016-06-10 12:03:08 -0700610 except:
611 self.disconnect_servo()
612 raise
Fang Deng5d518f42013-08-02 14:04:32 -0700613
614
Richard Barnette1edbb162016-11-01 11:47:50 -0700615 def repair(self, silent=False):
616 """Attempt to repair servo host.
617
618 @param silent If true, suppress logging in `status.log`.
619 """
Richard Barnetteabbdc252018-07-26 16:57:42 -0700620 message = 'Beginning repair for servo host %s port %s serial %s'
621 message %= (self.hostname, self.servo_port, self.servo_serial)
622 self.record('INFO', None, None, message)
Richard Barnette9a26ad62016-06-10 12:03:08 -0700623 try:
Richard Barnette1edbb162016-11-01 11:47:50 -0700624 self._repair_strategy.repair(self, silent)
Richard Barnette9a26ad62016-06-10 12:03:08 -0700625 except:
626 self.disconnect_servo()
627 raise
Fang Deng5d518f42013-08-02 14:04:32 -0700628
629
Fang Dengd4fe7392013-09-20 12:18:21 -0700630 def has_power(self):
631 """Return whether or not the servo host is powered by PoE."""
632 # TODO(fdeng): See crbug.com/302791
633 # For now, assume all servo hosts in the lab have power.
634 return self.is_in_lab()
635
636
637 def power_cycle(self):
638 """Cycle power to this host via PoE if it is a lab device.
639
Richard Barnette9a26ad62016-06-10 12:03:08 -0700640 @raises AutoservRepairError if it fails to power cycle the
Fang Dengd4fe7392013-09-20 12:18:21 -0700641 servo host.
642
643 """
644 if self.has_power():
645 try:
646 rpm_client.set_power(self.hostname, 'CYCLE')
647 except (socket.error, xmlrpclib.Error,
648 httplib.BadStatusLine,
649 rpm_client.RemotePowerException) as e:
Richard Barnette9a26ad62016-06-10 12:03:08 -0700650 raise hosts.AutoservRepairError(
Fang Dengd4fe7392013-09-20 12:18:21 -0700651 'Power cycling %s failed: %s' % (self.hostname, e))
652 else:
653 logging.info('Skipping power cycling, not a lab device.')
654
655
Dan Shi4d478522014-02-14 13:46:32 -0800656 def get_servo(self):
657 """Get the cached servo.Servo object.
Fang Deng5d518f42013-08-02 14:04:32 -0700658
Dan Shi4d478522014-02-14 13:46:32 -0800659 @return: a servo.Servo object.
Fang Deng5d518f42013-08-02 14:04:32 -0700660 """
Dan Shi4d478522014-02-14 13:46:32 -0800661 return self._servo
662
663
Congbin Guoa1f9cba2018-07-03 11:36:59 -0700664 def close(self):
665 """Stop UART logging and close the host object."""
666 if self._servo:
Congbin Guo2e5e2a22018-07-27 10:32:48 -0700667 # In some cases when we run as lab-tools, the job object is None.
668 if self.job:
669 self._servo.dump_uart_streams(self.job.resultdir)
Congbin Guoa1f9cba2018-07-03 11:36:59 -0700670 self._servo.close()
671
672 super(ServoHost, self).close()
673
674
Richard Barnetteea3e4602016-06-10 12:36:41 -0700675def make_servo_hostname(dut_hostname):
676 """Given a DUT's hostname, return the hostname of its servo.
677
678 @param dut_hostname: hostname of a DUT.
679
680 @return hostname of the DUT's servo.
681
682 """
683 host_parts = dut_hostname.split('.')
684 host_parts[0] = host_parts[0] + '-servo'
685 return '.'.join(host_parts)
686
687
688def servo_host_is_up(servo_hostname):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700689 """Given a servo host name, return if it's up or not.
Richard Barnetteea3e4602016-06-10 12:36:41 -0700690
691 @param servo_hostname: hostname of the servo host.
692
693 @return True if it's up, False otherwise
694 """
695 # Technically, this duplicates the SSH ping done early in the servo
696 # proxy initialization code. However, this ping ends in a couple
697 # seconds when if fails, rather than the 60 seconds it takes to decide
698 # that an SSH ping has timed out. Specifically, that timeout happens
699 # when our servo DNS name resolves, but there is no host at that IP.
700 logging.info('Pinging servo host at %s', servo_hostname)
701 ping_config = ping_runner.PingConfig(
702 servo_hostname, count=3,
703 ignore_result=True, ignore_status=True)
704 return ping_runner.PingRunner().ping(ping_config).received > 0
705
706
Richard Barnettee519dcd2016-08-15 17:37:17 -0700707def _map_afe_board_to_servo_board(afe_board):
708 """Map a board we get from the AFE to a servo appropriate value.
709
710 Many boards are identical to other boards for servo's purposes.
711 This function makes that mapping.
712
713 @param afe_board string board name received from AFE.
714 @return board we expect servo to have.
715
716 """
717 KNOWN_SUFFIXES = ['-freon', '_freon', '_moblab', '-cheets']
718 BOARD_MAP = {'gizmo': 'panther'}
719 mapped_board = afe_board
720 if afe_board in BOARD_MAP:
721 mapped_board = BOARD_MAP[afe_board]
722 else:
723 for suffix in KNOWN_SUFFIXES:
724 if afe_board.endswith(suffix):
725 mapped_board = afe_board[0:-len(suffix)]
726 break
727 if mapped_board != afe_board:
728 logging.info('Mapping AFE board=%s to %s', afe_board, mapped_board)
729 return mapped_board
730
731
Prathmesh Prabhub4810232018-09-07 13:24:08 -0700732def get_servo_args_for_host(dut_host):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700733 """Return servo data associated with a given DUT.
Richard Barnetteea3e4602016-06-10 12:36:41 -0700734
Richard Barnetteea3e4602016-06-10 12:36:41 -0700735 @param dut_host Instance of `Host` on which to find the servo
736 attributes.
Prathmesh Prabhuf605dd32018-08-28 17:09:04 -0700737 @return `servo_args` dict with host and an optional port.
Richard Barnetteea3e4602016-06-10 12:36:41 -0700738 """
Prathmesh Prabhucba44292018-08-28 17:44:45 -0700739 info = dut_host.host_info_store.get()
740 servo_args = {k: v for k, v in info.attributes.iteritems()
741 if k in SERVO_ATTR_KEYS}
Richard Barnetteea3e4602016-06-10 12:36:41 -0700742
Prathmesh Prabhucba44292018-08-28 17:44:45 -0700743 if SERVO_PORT_ATTR in servo_args:
744 try:
745 servo_args[SERVO_PORT_ATTR] = int(servo_args[SERVO_PORT_ATTR])
746 except ValueError:
747 logging.error('servo port is not an int: %s',
748 servo_args[SERVO_PORT_ATTR])
749 # Reset servo_args because we don't want to use an invalid port.
750 servo_args.pop(SERVO_HOST_ATTR, None)
751
752 if info.board:
753 servo_args[SERVO_BOARD_ATTR] = _map_afe_board_to_servo_board(info.board)
Nick Sanders2f3c9852018-10-24 12:10:24 -0700754 if info.model:
755 servo_args[SERVO_MODEL_ATTR] = info.model
Prathmesh Prabhu6f5f6362018-09-05 17:20:31 -0700756 return servo_args if SERVO_HOST_ATTR in servo_args else None
Richard Barnetteea3e4602016-06-10 12:36:41 -0700757
758
Prathmesh Prabhuefb1b482018-08-28 17:15:05 -0700759def _tweak_args_for_ssp_moblab(servo_args):
760 if servo_args[SERVO_HOST_ATTR] in ['localhost', '127.0.0.1']:
761 servo_args[SERVO_HOST_ATTR] = _CONFIG.get_config_value(
762 'SSP', 'host_container_ip', type=str, default=None)
763
764
Dan Shi023aae32016-05-25 11:13:01 -0700765def create_servo_host(dut, servo_args, try_lab_servo=False,
Richard Barnette9a26ad62016-06-10 12:03:08 -0700766 try_servo_repair=False):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700767 """Create a ServoHost object for a given DUT, if appropriate.
Dan Shi4d478522014-02-14 13:46:32 -0800768
Richard Barnette9a26ad62016-06-10 12:03:08 -0700769 This function attempts to create and verify or repair a `ServoHost`
770 object for a servo connected to the given `dut`, subject to various
771 constraints imposed by the parameters:
772 * When the `servo_args` parameter is not `None`, a servo
773 host must be created, and must be checked with `repair()`.
774 * Otherwise, if a servo exists in the lab and `try_lab_servo` is
775 true:
776 * If `try_servo_repair` is true, then create a servo host and
777 check it with `repair()`.
778 * Otherwise, if the servo responds to `ping` then create a
779 servo host and check it with `verify()`.
Fang Denge545abb2014-12-30 18:43:47 -0800780
Richard Barnette9a26ad62016-06-10 12:03:08 -0700781 In cases where `servo_args` was not `None`, repair failure
782 exceptions are passed back to the caller; otherwise, exceptions
Richard Barnette07c2e1d2016-10-26 14:24:28 -0700783 are logged and then discarded. Note that this only happens in cases
784 where we're called from a test (not special task) control file that
785 has an explicit dependency on servo. In that case, we require that
786 repair not write to `status.log`, so as to avoid polluting test
787 results.
788
789 TODO(jrbarnette): The special handling for servo in test control
790 files is a thorn in my flesh; I dearly hope to see it cut out before
791 my retirement.
Richard Barnette9a26ad62016-06-10 12:03:08 -0700792
793 Parameters for a servo host consist of a host name, port number, and
794 DUT board, and are determined from one of these sources, in order of
795 priority:
Richard Barnetteea3e4602016-06-10 12:36:41 -0700796 * Servo attributes from the `dut` parameter take precedence over
797 all other sources of information.
798 * If a DNS entry for the servo based on the DUT hostname exists in
799 the CrOS lab network, that hostname is used with the default
Richard Barnette9a26ad62016-06-10 12:03:08 -0700800 port and the DUT's board.
Richard Barnetteea3e4602016-06-10 12:36:41 -0700801 * If no other options are found, the parameters will be taken
Richard Barnette9a26ad62016-06-10 12:03:08 -0700802 from the `servo_args` dict passed in from the caller.
Richard Barnetteea3e4602016-06-10 12:36:41 -0700803
804 @param dut An instance of `Host` from which to take
805 servo parameters (if available).
806 @param servo_args A dictionary with servo parameters to use if
807 they can't be found from `dut`. If this
808 argument is supplied, unrepaired exceptions
809 from `verify()` will be passed back to the
810 caller.
811 @param try_lab_servo If not true, servo host creation will be
812 skipped unless otherwise required by the
813 caller.
Richard Barnette9a26ad62016-06-10 12:03:08 -0700814 @param try_servo_repair If true, check a servo host with
815 `repair()` instead of `verify()`.
Dan Shi4d478522014-02-14 13:46:32 -0800816
817 @returns: A ServoHost object or None. See comments above.
818
819 """
Richard Barnette07c2e1d2016-10-26 14:24:28 -0700820 servo_dependency = servo_args is not None
Richard Barnette07c2e1d2016-10-26 14:24:28 -0700821 if dut is not None and (try_lab_servo or servo_dependency):
Prathmesh Prabhub4810232018-09-07 13:24:08 -0700822 servo_args_override = get_servo_args_for_host(dut)
Richard Barnetteea3e4602016-06-10 12:36:41 -0700823 if servo_args_override is not None:
Prathmesh Prabhuefb1b482018-08-28 17:15:05 -0700824 if utils.in_moblab_ssp():
825 _tweak_args_for_ssp_moblab(servo_args_override)
Prathmesh Prabhu88bf6052018-08-28 16:21:26 -0700826 logging.debug(
827 'Overriding provided servo_args (%s) with arguments'
828 ' determined from the host (%s)',
829 servo_args,
830 servo_args_override,
831 )
Richard Barnetteea3e4602016-06-10 12:36:41 -0700832 servo_args = servo_args_override
Prathmesh Prabhucba44292018-08-28 17:44:45 -0700833
Richard Barnetteea3e4602016-06-10 12:36:41 -0700834 if servo_args is None:
Prathmesh Prabhu88bf6052018-08-28 16:21:26 -0700835 logging.debug('No servo_args provided, and failed to find overrides.')
Richard Barnetteea3e4602016-06-10 12:36:41 -0700836 return None
Prathmesh Prabhucba44292018-08-28 17:44:45 -0700837 if SERVO_HOST_ATTR not in servo_args:
838 logging.debug('%s attribute missing from servo_args: %s',
839 SERVO_HOST_ATTR, servo_args)
840 return None
Richard Barnette07c2e1d2016-10-26 14:24:28 -0700841 if (not servo_dependency and not try_servo_repair and
Richard Barnette9a26ad62016-06-10 12:03:08 -0700842 not servo_host_is_up(servo_args[SERVO_HOST_ATTR])):
Prathmesh Prabhu88bf6052018-08-28 16:21:26 -0700843 logging.debug('ServoHost is not up.')
Dan Shibbb0cb62014-03-24 17:50:57 -0700844 return None
Prathmesh Prabhu88bf6052018-08-28 16:21:26 -0700845
Prathmesh Prabhuf605dd32018-08-28 17:09:04 -0700846 newhost = ServoHost(
847 is_in_lab=(servo_args
848 and server_utils.host_in_lab(
849 servo_args[SERVO_HOST_ATTR])),
850 **servo_args
851 )
Richard Barnette9a26ad62016-06-10 12:03:08 -0700852 # Note that the logic of repair() includes everything done
853 # by verify(). It's sufficient to call one or the other;
854 # we don't need both.
Richard Barnette07c2e1d2016-10-26 14:24:28 -0700855 if servo_dependency:
856 newhost.repair(silent=True)
Prathmesh Prabhu88bf6052018-08-28 16:21:26 -0700857 return newhost
858
859 if try_servo_repair:
860 try:
861 newhost.repair()
862 except Exception:
863 logging.exception('servo repair failed for %s', newhost.hostname)
Richard Barnette9a26ad62016-06-10 12:03:08 -0700864 else:
865 try:
Prathmesh Prabhu88bf6052018-08-28 16:21:26 -0700866 newhost.verify()
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700867 except Exception:
Prathmesh Prabhu88bf6052018-08-28 16:21:26 -0700868 logging.exception('servo verify failed for %s', newhost.hostname)
Richard Barnette9a26ad62016-06-10 12:03:08 -0700869 return newhost