blob: 89672b68d74c869f5bb7e32ca467c59e6441ba95 [file] [log] [blame]
Fang Deng5d518f42013-08-02 14:04:32 -07001# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4#
5# Expects to be run in an environment with sudo and no interactive password
6# prompt, such as within the Chromium OS development chroot.
7
8
9"""This file provides core logic for servo verify/repair process."""
10
11
12import httplib
13import logging
14import socket
Kevin Cheng79589982016-10-25 13:26:04 -070015import traceback
Fang Deng5d518f42013-08-02 14:04:32 -070016import xmlrpclib
17
18from autotest_lib.client.bin import utils
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -070019from autotest_lib.client.common_lib import control_data
Fang Deng5d518f42013-08-02 14:04:32 -070020from autotest_lib.client.common_lib import error
beeps5e8c45a2013-12-17 22:05:11 -080021from autotest_lib.client.common_lib import global_config
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -070022from autotest_lib.client.common_lib import host_states
Richard Barnette9a26ad62016-06-10 12:03:08 -070023from autotest_lib.client.common_lib import hosts
Dan Shi0942b1d2015-03-31 11:07:00 -070024from autotest_lib.client.common_lib import lsbrelease_utils
beeps5e8c45a2013-12-17 22:05:11 -080025from autotest_lib.client.common_lib.cros import autoupdater
26from autotest_lib.client.common_lib.cros import dev_server
Fang Deng5d518f42013-08-02 14:04:32 -070027from autotest_lib.client.common_lib.cros import retry
Kevin Cheng79589982016-10-25 13:26:04 -070028from autotest_lib.client.common_lib.cros.graphite import autotest_es
Gabe Black1e1c41b2015-02-04 23:55:15 -080029from autotest_lib.client.common_lib.cros.graphite import autotest_stats
Christopher Wileycef1f902014-06-19 11:11:23 -070030from autotest_lib.client.common_lib.cros.network import ping_runner
Hsinyu Chaoe0b08e62015-08-11 10:50:37 +000031from autotest_lib.client.cros import constants as client_constants
Richard Barnettee519dcd2016-08-15 17:37:17 -070032from autotest_lib.server import afe_utils
beeps5e8c45a2013-12-17 22:05:11 -080033from autotest_lib.server import site_utils as server_site_utils
Cheng-Yi Chiang22612862015-08-20 20:39:57 +080034from autotest_lib.server.cros import dnsname_mangler
Simran Basi0739d682015-02-25 16:22:56 -080035from autotest_lib.server.cros.dynamic_suite import frontend_wrappers
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -070036from autotest_lib.server.cros.dynamic_suite import control_file_getter
Richard Barnette9a26ad62016-06-10 12:03:08 -070037from autotest_lib.server.cros.servo import servo
38from autotest_lib.server.hosts import servo_repair
Fang Deng5d518f42013-08-02 14:04:32 -070039from autotest_lib.server.hosts import ssh_host
Fang Dengd4fe7392013-09-20 12:18:21 -070040from autotest_lib.site_utils.rpm_control_system import rpm_client
Fang Deng5d518f42013-08-02 14:04:32 -070041
42
Simran Basi0739d682015-02-25 16:22:56 -080043# Names of the host attributes in the database that represent the values for
44# the servo_host and servo_port for a servo connected to the DUT.
45SERVO_HOST_ATTR = 'servo_host'
46SERVO_PORT_ATTR = 'servo_port'
Richard Barnettee519dcd2016-08-15 17:37:17 -070047SERVO_BOARD_ATTR = 'servo_board'
Kevin Cheng643ce8a2016-09-15 15:42:12 -070048SERVO_SERIAL_ATTR = 'servo_serial'
Simran Basi0739d682015-02-25 16:22:56 -080049
Dan Shi3b2adf62015-09-02 17:46:54 -070050_CONFIG = global_config.global_config
xixuan6cf6d2f2016-01-29 15:29:00 -080051ENABLE_SSH_TUNNEL_FOR_SERVO = _CONFIG.get_config_value(
52 'CROS', 'enable_ssh_tunnel_for_servo', type=bool, default=False)
Simran Basi0739d682015-02-25 16:22:56 -080053
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -070054AUTOTEST_BASE = _CONFIG.get_config_value(
55 'SCHEDULER', 'drone_installation_directory',
56 default='/usr/local/autotest')
57
58_SERVO_HOST_REBOOT_TEST_NAME = 'servohost_Reboot'
Kevin Cheng55265902016-10-19 12:46:50 -070059_SERVO_HOST_FORCE_REBOOT_TEST_NAME = 'servohost_Reboot.force_reboot'
Fang Deng5d518f42013-08-02 14:04:32 -070060
Fang Deng5d518f42013-08-02 14:04:32 -070061class ServoHost(ssh_host.SSHHost):
62 """Host class for a host that controls a servo, e.g. beaglebone."""
63
Richard Barnette9a26ad62016-06-10 12:03:08 -070064 DEFAULT_PORT = 9999
65
Dan Shie5b3c512014-08-21 12:12:09 -070066 # Timeout for initializing servo signals.
67 INITIALIZE_SERVO_TIMEOUT_SECS = 30
Richard Barnette9a26ad62016-06-10 12:03:08 -070068
xixuan6cf6d2f2016-01-29 15:29:00 -080069 # Ready test function
70 SERVO_READY_METHOD = 'get_version'
Fang Deng5d518f42013-08-02 14:04:32 -070071
Gabe Black1e1c41b2015-02-04 23:55:15 -080072 _timer = autotest_stats.Timer('servo_host')
Fang Dengd4fe7392013-09-20 12:18:21 -070073
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -070074 REBOOT_CMD = 'sleep 1; reboot & sleep 10; reboot -f'
75
Fang Deng5d518f42013-08-02 14:04:32 -070076
Richard Barnette17bfc6c2016-08-04 18:41:43 -070077 def _initialize(self, servo_host='localhost',
Richard Barnettee519dcd2016-08-15 17:37:17 -070078 servo_port=DEFAULT_PORT, servo_board=None,
Kevin Cheng643ce8a2016-09-15 15:42:12 -070079 servo_serial=None, is_in_lab=None, *args, **dargs):
Fang Deng5d518f42013-08-02 14:04:32 -070080 """Initialize a ServoHost instance.
81
82 A ServoHost instance represents a host that controls a servo.
83
84 @param servo_host: Name of the host where the servod process
85 is running.
86 @param servo_port: Port the servod process is listening on.
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -070087 @param servo_board: Board that the servo is connected to.
Dan Shi4d478522014-02-14 13:46:32 -080088 @param is_in_lab: True if the servo host is in Cros Lab. Default is set
89 to None, for which utils.host_is_in_lab_zone will be
90 called to check if the servo host is in Cros lab.
Fang Deng5d518f42013-08-02 14:04:32 -070091
92 """
93 super(ServoHost, self)._initialize(hostname=servo_host,
94 *args, **dargs)
Richard Barnettee519dcd2016-08-15 17:37:17 -070095 self.servo_port = servo_port
96 self.servo_board = servo_board
Kevin Cheng643ce8a2016-09-15 15:42:12 -070097 self.servo_serial = servo_serial
Richard Barnettee519dcd2016-08-15 17:37:17 -070098 self._servo = None
Richard Barnette9a26ad62016-06-10 12:03:08 -070099 self._repair_strategy = (
100 servo_repair.create_servo_repair_strategy())
Richard Barnettee519dcd2016-08-15 17:37:17 -0700101 self._is_localhost = (self.hostname == 'localhost')
102 if self._is_localhost:
103 self._is_in_lab = False
104 elif is_in_lab is None:
Dan Shi4d478522014-02-14 13:46:32 -0800105 self._is_in_lab = utils.host_is_in_lab_zone(self.hostname)
106 else:
107 self._is_in_lab = is_in_lab
xixuan6cf6d2f2016-01-29 15:29:00 -0800108
Richard Barnettee519dcd2016-08-15 17:37:17 -0700109 # Commands on the servo host must be run by the superuser.
110 # Our account on a remote host is root, but if our target is
111 # localhost then we might be running unprivileged. If so,
112 # `sudo` will have to be added to the commands.
Fang Deng5d518f42013-08-02 14:04:32 -0700113 if self._is_localhost:
114 self._sudo_required = utils.system_output('id -u') != '0'
115 else:
116 self._sudo_required = False
Richard Barnettee519dcd2016-08-15 17:37:17 -0700117
Richard Barnette9a26ad62016-06-10 12:03:08 -0700118
119 def connect_servo(self):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700120 """Establish a connection to the servod server on this host.
Richard Barnette9a26ad62016-06-10 12:03:08 -0700121
122 Initializes `self._servo` and then verifies that all network
123 connections are working. This will create an ssh tunnel if
124 it's required.
125
126 As a side effect of testing the connection, all signals on the
127 target servo are reset to default values, and the USB stick is
128 set to the neutral (off) position.
129 """
Kevin Cheng643ce8a2016-09-15 15:42:12 -0700130 servo_obj = servo.Servo(servo_host=self, servo_serial=self.servo_serial)
Richard Barnette9a26ad62016-06-10 12:03:08 -0700131 timeout, _ = retry.timeout(
132 servo_obj.initialize_dut,
133 timeout_sec=self.INITIALIZE_SERVO_TIMEOUT_SECS)
134 if timeout:
135 raise hosts.AutoservVerifyError(
136 'Servo initialize timed out.')
137 self._servo = servo_obj
138
139
140 def disconnect_servo(self):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700141 """Disconnect our servo if it exists.
Richard Barnette9a26ad62016-06-10 12:03:08 -0700142
143 If we've previously successfully connected to our servo,
144 disconnect any established ssh tunnel, and set `self._servo`
145 back to `None`.
146 """
147 if self._servo:
148 # N.B. This call is safe even without a tunnel:
149 # rpc_server_tracker.disconnect() silently ignores
150 # unknown ports.
151 self.rpc_server_tracker.disconnect(self.servo_port)
152 self._servo = None
Fang Deng5d518f42013-08-02 14:04:32 -0700153
154
155 def is_in_lab(self):
156 """Check whether the servo host is a lab device.
157
158 @returns: True if the servo host is in Cros Lab, otherwise False.
159
160 """
161 return self._is_in_lab
162
163
164 def is_localhost(self):
165 """Checks whether the servo host points to localhost.
166
167 @returns: True if it points to localhost, otherwise False.
168
169 """
170 return self._is_localhost
171
172
173 def get_servod_server_proxy(self):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700174 """Return a proxy that can be used to communicate with servod server.
Fang Deng5d518f42013-08-02 14:04:32 -0700175
176 @returns: An xmlrpclib.ServerProxy that is connected to the servod
177 server on the host.
Fang Deng5d518f42013-08-02 14:04:32 -0700178 """
Richard Barnette9a26ad62016-06-10 12:03:08 -0700179 if ENABLE_SSH_TUNNEL_FOR_SERVO and not self.is_localhost():
180 return self.rpc_server_tracker.xmlrpc_connect(
181 None, self.servo_port,
182 ready_test_name=self.SERVO_READY_METHOD,
183 timeout_seconds=60)
184 else:
185 remote = 'http://%s:%s' % (self.hostname, self.servo_port)
186 return xmlrpclib.ServerProxy(remote)
Fang Deng5d518f42013-08-02 14:04:32 -0700187
188
Richard Barnette9a26ad62016-06-10 12:03:08 -0700189 def is_cros_host(self):
beeps5e8c45a2013-12-17 22:05:11 -0800190 """Check if a servo host is running chromeos.
191
192 @return: True if the servo host is running chromeos.
193 False if it isn't, or we don't have enough information.
194 """
195 try:
196 result = self.run('grep -q CHROMEOS /etc/lsb-release',
197 ignore_status=True, timeout=10)
198 except (error.AutoservRunError, error.AutoservSSHTimeout):
199 return False
200 return result.exit_status == 0
201
202
Fang Deng5d518f42013-08-02 14:04:32 -0700203 def make_ssh_command(self, user='root', port=22, opts='', hosts_file=None,
204 connect_timeout=None, alive_interval=None):
205 """Override default make_ssh_command to use tuned options.
206
207 Tuning changes:
208 - ConnectTimeout=30; maximum of 30 seconds allowed for an SSH
209 connection failure. Consistency with remote_access.py.
210
211 - ServerAliveInterval=180; which causes SSH to ping connection every
212 180 seconds. In conjunction with ServerAliveCountMax ensures
213 that if the connection dies, Autotest will bail out quickly.
214
215 - ServerAliveCountMax=3; consistency with remote_access.py.
216
217 - ConnectAttempts=4; reduce flakiness in connection errors;
218 consistency with remote_access.py.
219
220 - UserKnownHostsFile=/dev/null; we don't care about the keys.
221
222 - SSH protocol forced to 2; needed for ServerAliveInterval.
223
224 @param user User name to use for the ssh connection.
225 @param port Port on the target host to use for ssh connection.
226 @param opts Additional options to the ssh command.
227 @param hosts_file Ignored.
228 @param connect_timeout Ignored.
229 @param alive_interval Ignored.
230
231 @returns: An ssh command with the requested settings.
232
233 """
234 base_command = ('/usr/bin/ssh -a -x %s -o StrictHostKeyChecking=no'
235 ' -o UserKnownHostsFile=/dev/null -o BatchMode=yes'
236 ' -o ConnectTimeout=30 -o ServerAliveInterval=180'
237 ' -o ServerAliveCountMax=3 -o ConnectionAttempts=4'
238 ' -o Protocol=2 -l %s -p %d')
239 return base_command % (opts, user, port)
240
241
242 def _make_scp_cmd(self, sources, dest):
243 """Format scp command.
244
245 Given a list of source paths and a destination path, produces the
246 appropriate scp command for encoding it. Remote paths must be
247 pre-encoded. Overrides _make_scp_cmd in AbstractSSHHost
248 to allow additional ssh options.
249
250 @param sources: A list of source paths to copy from.
251 @param dest: Destination path to copy to.
252
253 @returns: An scp command that copies |sources| on local machine to
254 |dest| on the remote servo host.
255
256 """
257 command = ('scp -rq %s -o BatchMode=yes -o StrictHostKeyChecking=no '
258 '-o UserKnownHostsFile=/dev/null -P %d %s "%s"')
259 return command % (self.master_ssh_option,
260 self.port, ' '.join(sources), dest)
261
262
263 def run(self, command, timeout=3600, ignore_status=False,
264 stdout_tee=utils.TEE_TO_LOGS, stderr_tee=utils.TEE_TO_LOGS,
265 connect_timeout=30, options='', stdin=None, verbose=True, args=()):
266 """Run a command on the servo host.
267
268 Extends method `run` in SSHHost. If the servo host is a remote device,
269 it will call `run` in SSHost without changing anything.
270 If the servo host is 'localhost', it will call utils.system_output.
271
272 @param command: The command line string.
273 @param timeout: Time limit in seconds before attempting to
274 kill the running process. The run() function
275 will take a few seconds longer than 'timeout'
276 to complete if it has to kill the process.
277 @param ignore_status: Do not raise an exception, no matter
278 what the exit code of the command is.
279 @param stdout_tee/stderr_tee: Where to tee the stdout/stderr.
280 @param connect_timeout: SSH connection timeout (in seconds)
281 Ignored if host is 'localhost'.
282 @param options: String with additional ssh command options
283 Ignored if host is 'localhost'.
284 @param stdin: Stdin to pass (a string) to the executed command.
285 @param verbose: Log the commands.
286 @param args: Sequence of strings to pass as arguments to command by
287 quoting them in " and escaping their contents if necessary.
288
289 @returns: A utils.CmdResult object.
290
291 @raises AutoservRunError if the command failed.
292 @raises AutoservSSHTimeout SSH connection has timed out. Only applies
293 when servo host is not 'localhost'.
294
295 """
296 run_args = {'command': command, 'timeout': timeout,
297 'ignore_status': ignore_status, 'stdout_tee': stdout_tee,
298 'stderr_tee': stderr_tee, 'stdin': stdin,
299 'verbose': verbose, 'args': args}
300 if self.is_localhost():
301 if self._sudo_required:
302 run_args['command'] = 'sudo -n %s' % command
303 try:
304 return utils.run(**run_args)
305 except error.CmdError as e:
306 logging.error(e)
307 raise error.AutoservRunError('command execution error',
308 e.result_obj)
309 else:
310 run_args['connect_timeout'] = connect_timeout
311 run_args['options'] = options
312 return super(ServoHost, self).run(**run_args)
313
314
Richard Barnette9a26ad62016-06-10 12:03:08 -0700315 def _get_release_version(self):
Dan Shi0942b1d2015-03-31 11:07:00 -0700316 """Get the value of attribute CHROMEOS_RELEASE_VERSION from lsb-release.
317
318 @returns The version string in lsb-release, under attribute
319 CHROMEOS_RELEASE_VERSION.
320 """
321 lsb_release_content = self.run(
322 'cat "%s"' % client_constants.LSB_RELEASE).stdout.strip()
323 return lsbrelease_utils.get_chromeos_release_version(
324 lsb_release_content=lsb_release_content)
325
326
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700327 def get_attached_duts(self, afe):
328 """Gather a list of duts that use this servo host.
329
330 @param afe: afe instance.
331
332 @returns list of duts.
Richard Barnette3a7697f2016-04-20 11:33:27 -0700333 """
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700334 return afe.get_hosts_by_attribute(
335 attribute=SERVO_HOST_ATTR, value=self.hostname)
336
337
338 def get_board(self):
339 """Determine the board for this servo host.
340
341 @returns a string representing this servo host's board.
342 """
343 return lsbrelease_utils.get_current_board(
344 lsb_release_content=self.run('cat /etc/lsb-release').stdout)
345
346
347 def _choose_dut_for_synchronized_reboot(self, dut_list, afe):
348 """Choose which dut to schedule servo host reboot job.
349
350 We'll want a semi-deterministic way of selecting which host should be
351 scheduled for the servo host reboot job. For now we'll sort the
352 list with the expectation the dut list will stay consistent.
353 From there we'll grab the first dut that is available so we
354 don't schedule a job on a dut that will never run.
355
356 @param dut_list: List of the dut hostnames to choose from.
357 @param afe: Instance of the AFE.
358
359 @return hostname of dut to schedule job on.
360 """
361 afe_hosts = afe.get_hosts(dut_list)
362 afe_hosts.sort()
363 for afe_host in afe_hosts:
364 if afe_host.status not in host_states.UNAVAILABLE_STATES:
365 return afe_host.hostname
366 # If they're all unavailable, just return the first sorted dut.
367 dut_list.sort()
368 return dut_list[0]
369
370
371 def _sync_job_scheduled_for_duts(self, dut_list, afe):
372 """Checks if a synchronized reboot has been scheduled for these duts.
373
374 Grab all the host queue entries that aren't completed for the duts and
375 see if any of them have the expected job name.
376
377 @param dut_list: List of duts to check on.
378 @param afe: Instance of the AFE.
379
380 @returns True if the job is scheduled, False otherwise.
381 """
382 afe_hosts = afe.get_hosts(dut_list)
383 for afe_host in afe_hosts:
384 hqes = afe.get_host_queue_entries(host=afe_host.id, complete=0)
385 for hqe in hqes:
386 job = afe.get_jobs(id=hqe.job.id)
Kevin Cheng55265902016-10-19 12:46:50 -0700387 if job and job[0].name in (_SERVO_HOST_REBOOT_TEST_NAME,
388 _SERVO_HOST_FORCE_REBOOT_TEST_NAME):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700389 return True
390 return False
391
392
Kevin Cheng55265902016-10-19 12:46:50 -0700393 def schedule_synchronized_reboot(self, dut_list, afe, force_reboot=False):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700394 """Schedule a job to reboot the servo host.
395
396 When we schedule a job, it will create a ServoHost object which will
397 go through this entire flow of checking if a reboot is needed and
398 trying to schedule it. There is probably a better approach to setting
399 up a synchronized reboot but I'm coming up short on better ideas so I
400 apologize for this circus show.
401
Kevin Cheng55265902016-10-19 12:46:50 -0700402 @param dut_list: List of duts that need to be locked.
403 @param afe: Instance of afe.
404 @param force_reboot: Boolean to indicate if a forced reboot should be
405 scheduled or not.
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700406 """
407 # If we've already scheduled job on a dut, we're done here.
408 if self._sync_job_scheduled_for_duts(dut_list, afe):
409 return
410
411 # Looks like we haven't scheduled a job yet.
Kevin Cheng55265902016-10-19 12:46:50 -0700412 test = (_SERVO_HOST_REBOOT_TEST_NAME if not force_reboot
413 else _SERVO_HOST_FORCE_REBOOT_TEST_NAME)
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700414 dut = self._choose_dut_for_synchronized_reboot(dut_list, afe)
415 getter = control_file_getter.FileSystemGetter([AUTOTEST_BASE])
Kevin Cheng55265902016-10-19 12:46:50 -0700416 control_file = getter.get_control_file_contents_by_name(test)
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700417 control_type = control_data.CONTROL_TYPE_NAMES.SERVER
Kevin Cheng79589982016-10-25 13:26:04 -0700418 try:
419 afe.create_job(control_file=control_file, name=test,
420 control_type=control_type, hosts=[dut])
421 except Exception as e:
422 # Sometimes creating the job will raise an exception. We'll log it
423 # but we don't want to fail because of it.
424 logging.exception('Scheduling reboot job failed: %s', e)
425 metadata = {'dut': dut,
426 'servo_host': self.hostname,
427 'error': str(e),
428 'details': traceback.format_exc()}
429 # We want to track how often we fail here so we can justify
430 # investing some effort into hardening up afe.create_job().
431 autotest_es.post(use_http=True,
432 type_str='servohost_Reboot_schedule_fail',
433 metadata=metadata)
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700434
435
436 def reboot(self, *args, **dargs):
437 """Reboot using special servo host reboot command."""
438 super(ServoHost, self).reboot(reboot_cmd=self.REBOOT_CMD,
439 *args, **dargs)
440
441
442 def _check_for_reboot(self, updater):
443 """Reboot this servo host if an upgrade is waiting.
Richard Barnette3a7697f2016-04-20 11:33:27 -0700444
445 If the host has successfully downloaded and finalized a new
446 build, reboot.
447
448 @param updater: a ChromiumOSUpdater instance for checking
449 whether reboot is needed.
450 @return Return a (status, build) tuple reflecting the
451 update_engine status and current build of the host
452 at the end of the call.
453 """
Richard Barnette9a26ad62016-06-10 12:03:08 -0700454 current_build_number = self._get_release_version()
Richard Barnette3a7697f2016-04-20 11:33:27 -0700455 status = updater.check_update_status()
456 if status == autoupdater.UPDATER_NEED_REBOOT:
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700457 # Check if we need to schedule an organized reboot.
Kevin Cheng79589982016-10-25 13:26:04 -0700458 afe = frontend_wrappers.RetryingAFE(
459 timeout_min=5, delay_sec=10,
460 server=server_site_utils.get_global_afe_hostname())
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700461 dut_list = self.get_attached_duts(afe)
462 logging.info('servo host has the following duts: %s', dut_list)
463 if len(dut_list) > 1:
464 logging.info('servo host has multiple duts, scheduling '
465 'synchronized reboot')
466 self.schedule_synchronized_reboot(dut_list, afe)
467 return status, current_build_number
468
469 logging.info('Rebooting servo host %s from build %s',
Richard Barnette3a7697f2016-04-20 11:33:27 -0700470 self.hostname, current_build_number)
471 # Tell the reboot() call not to wait for completion.
472 # Otherwise, the call will log reboot failure if servo does
473 # not come back. The logged reboot failure will lead to
474 # test job failure. If the test does not require servo, we
475 # don't want servo failure to fail the test with error:
476 # `Host did not return from reboot` in status.log.
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700477 self.reboot(fastsync=True, wait=False)
Richard Barnette3a7697f2016-04-20 11:33:27 -0700478
479 # We told the reboot() call not to wait, but we need to wait
480 # for the reboot before we continue. Alas. The code from
481 # here below is basically a copy of Host.wait_for_restart(),
482 # with the logging bits ripped out, so that they can't cause
483 # the failure logging problem described above.
484 #
485 # The black stain that this has left on my soul can never be
486 # erased.
487 old_boot_id = self.get_boot_id()
488 if not self.wait_down(timeout=self.WAIT_DOWN_REBOOT_TIMEOUT,
489 warning_timer=self.WAIT_DOWN_REBOOT_WARNING,
490 old_boot_id=old_boot_id):
491 raise error.AutoservHostError(
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700492 'servo host %s failed to shut down.' %
493 self.hostname)
Richard Barnette3a7697f2016-04-20 11:33:27 -0700494 if self.wait_up(timeout=120):
Richard Barnette9a26ad62016-06-10 12:03:08 -0700495 current_build_number = self._get_release_version()
Richard Barnette3a7697f2016-04-20 11:33:27 -0700496 status = updater.check_update_status()
497 logging.info('servo host %s back from reboot, with build %s',
498 self.hostname, current_build_number)
499 else:
500 raise error.AutoservHostError(
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700501 'servo host %s failed to come back from reboot.' %
502 self.hostname)
Richard Barnette3a7697f2016-04-20 11:33:27 -0700503 return status, current_build_number
504
505
beeps5e8c45a2013-12-17 22:05:11 -0800506 @_timer.decorate
Richard Barnette3a7697f2016-04-20 11:33:27 -0700507 def update_image(self, wait_for_update=False):
beeps5e8c45a2013-12-17 22:05:11 -0800508 """Update the image on the servo host, if needed.
509
J. Richard Barnette84895392015-04-30 12:31:01 -0700510 This method recognizes the following cases:
511 * If the Host is not running Chrome OS, do nothing.
512 * If a previously triggered update is now complete, reboot
513 to the new version.
514 * If the host is processing a previously triggered update,
515 do nothing.
516 * If the host is running a version of Chrome OS different
517 from the default for servo Hosts, trigger an update, but
518 don't wait for it to complete.
beeps5e8c45a2013-12-17 22:05:11 -0800519
Richard Barnette3a7697f2016-04-20 11:33:27 -0700520 @param wait_for_update If an update needs to be applied and
521 this is true, then don't return until the update is
522 downloaded and finalized, and the host rebooted.
beeps5e8c45a2013-12-17 22:05:11 -0800523 @raises dev_server.DevServerException: If all the devservers are down.
524 @raises site_utils.ParseBuildNameException: If the devserver returns
525 an invalid build name.
526 @raises autoupdater.ChromiumOSError: If something goes wrong in the
527 checking update engine client status or applying an update.
528 @raises AutoservRunError: If the update_engine_client isn't present on
529 the host, and the host is a cros_host.
J. Richard Barnette84895392015-04-30 12:31:01 -0700530
beeps5e8c45a2013-12-17 22:05:11 -0800531 """
Dan Shib795b5a2015-09-24 13:26:35 -0700532 # servod could be running in a Ubuntu workstation.
Richard Barnette9a26ad62016-06-10 12:03:08 -0700533 if not self.is_cros_host():
beeps5e8c45a2013-12-17 22:05:11 -0800534 logging.info('Not attempting an update, either %s is not running '
535 'chromeos or we cannot find enough information about '
536 'the host.', self.hostname)
537 return
538
Dan Shib795b5a2015-09-24 13:26:35 -0700539 if lsbrelease_utils.is_moblab():
540 logging.info('Not attempting an update, %s is running moblab.',
541 self.hostname)
542 return
543
Richard Barnette260cbd02016-10-06 12:23:28 -0700544 target_build = afe_utils.get_stable_cros_version(self.get_board())
J. Richard Barnette84895392015-04-30 12:31:01 -0700545 target_build_number = server_site_utils.ParseBuildName(
546 target_build)[3]
beeps5e8c45a2013-12-17 22:05:11 -0800547 ds = dev_server.ImageServer.resolve(self.hostname)
J. Richard Barnette84895392015-04-30 12:31:01 -0700548 url = ds.get_update_url(target_build)
beeps5e8c45a2013-12-17 22:05:11 -0800549
550 updater = autoupdater.ChromiumOSUpdater(update_url=url, host=self)
Richard Barnette3a7697f2016-04-20 11:33:27 -0700551 status, current_build_number = self._check_for_reboot(updater)
552 update_pending = True
beeps5e8c45a2013-12-17 22:05:11 -0800553 if status in autoupdater.UPDATER_PROCESSING_UPDATE:
554 logging.info('servo host %s already processing an update, update '
555 'engine client status=%s', self.hostname, status)
J. Richard Barnette84895392015-04-30 12:31:01 -0700556 elif current_build_number != target_build_number:
beeps5e8c45a2013-12-17 22:05:11 -0800557 logging.info('Using devserver url: %s to trigger update on '
558 'servo host %s, from %s to %s', url, self.hostname,
J. Richard Barnette84895392015-04-30 12:31:01 -0700559 current_build_number, target_build_number)
beeps5e8c45a2013-12-17 22:05:11 -0800560 try:
J. Richard Barnette84895392015-04-30 12:31:01 -0700561 ds.stage_artifacts(target_build,
562 artifacts=['full_payload'])
563 except Exception as e:
564 logging.error('Staging artifacts failed: %s', str(e))
565 logging.error('Abandoning update for this cycle.')
beeps5e8c45a2013-12-17 22:05:11 -0800566 else:
J. Richard Barnette84895392015-04-30 12:31:01 -0700567 try:
Richard Barnette7e53aa02016-05-20 10:49:40 -0700568 # TODO(jrbarnette): This 'touch' is a gross hack
569 # to get us past crbug.com/613603. Once that
570 # bug is resolved, we should remove this code.
571 self.run('touch /home/chronos/.oobe_completed')
J. Richard Barnette84895392015-04-30 12:31:01 -0700572 updater.trigger_update()
573 except autoupdater.RootFSUpdateError as e:
574 trigger_download_status = 'failed with %s' % str(e)
575 autotest_stats.Counter(
576 'servo_host.RootFSUpdateError').increment()
577 else:
578 trigger_download_status = 'passed'
579 logging.info('Triggered download and update %s for %s, '
580 'update engine currently in status %s',
581 trigger_download_status, self.hostname,
582 updater.check_update_status())
beeps5e8c45a2013-12-17 22:05:11 -0800583 else:
584 logging.info('servo host %s does not require an update.',
585 self.hostname)
Richard Barnette3a7697f2016-04-20 11:33:27 -0700586 update_pending = False
587
588 if update_pending and wait_for_update:
589 logging.info('Waiting for servo update to complete.')
590 self.run('update_engine_client --follow', ignore_status=True)
beeps5e8c45a2013-12-17 22:05:11 -0800591
592
Richard Barnette9a26ad62016-06-10 12:03:08 -0700593 def verify(self):
594 """Update the servo host and verify it's in a good state."""
Richard Barnette79d78c42016-05-25 09:31:21 -0700595 # TODO(jrbarnette) Old versions of beaglebone_servo include
Richard Barnette9a26ad62016-06-10 12:03:08 -0700596 # the powerd package. If you touch the .oobe_completed file
597 # (as we do to work around an update_engine problem), then
598 # powerd will eventually shut down the beaglebone for lack
599 # of (apparent) activity. Current versions of
Richard Barnette79d78c42016-05-25 09:31:21 -0700600 # beaglebone_servo don't have powerd, but until we can purge
601 # the lab of the old images, we need to make sure powerd
602 # isn't running.
603 self.run('stop powerd', ignore_status=True)
Richard Barnette9a26ad62016-06-10 12:03:08 -0700604 try:
605 self._repair_strategy.verify(self)
606 except:
607 self.disconnect_servo()
608 raise
Fang Deng5d518f42013-08-02 14:04:32 -0700609
610
Richard Barnette9a26ad62016-06-10 12:03:08 -0700611 def repair(self):
612 """Attempt to repair servo host."""
613 try:
614 self._repair_strategy.repair(self)
615 except:
616 self.disconnect_servo()
617 raise
Fang Deng5d518f42013-08-02 14:04:32 -0700618
619
Fang Dengd4fe7392013-09-20 12:18:21 -0700620 def has_power(self):
621 """Return whether or not the servo host is powered by PoE."""
622 # TODO(fdeng): See crbug.com/302791
623 # For now, assume all servo hosts in the lab have power.
624 return self.is_in_lab()
625
626
627 def power_cycle(self):
628 """Cycle power to this host via PoE if it is a lab device.
629
Richard Barnette9a26ad62016-06-10 12:03:08 -0700630 @raises AutoservRepairError if it fails to power cycle the
Fang Dengd4fe7392013-09-20 12:18:21 -0700631 servo host.
632
633 """
634 if self.has_power():
635 try:
636 rpm_client.set_power(self.hostname, 'CYCLE')
637 except (socket.error, xmlrpclib.Error,
638 httplib.BadStatusLine,
639 rpm_client.RemotePowerException) as e:
Richard Barnette9a26ad62016-06-10 12:03:08 -0700640 raise hosts.AutoservRepairError(
Fang Dengd4fe7392013-09-20 12:18:21 -0700641 'Power cycling %s failed: %s' % (self.hostname, e))
642 else:
643 logging.info('Skipping power cycling, not a lab device.')
644
645
Dan Shi4d478522014-02-14 13:46:32 -0800646 def get_servo(self):
647 """Get the cached servo.Servo object.
Fang Deng5d518f42013-08-02 14:04:32 -0700648
Dan Shi4d478522014-02-14 13:46:32 -0800649 @return: a servo.Servo object.
Fang Deng5d518f42013-08-02 14:04:32 -0700650 """
Dan Shi4d478522014-02-14 13:46:32 -0800651 return self._servo
652
653
Richard Barnetteea3e4602016-06-10 12:36:41 -0700654def make_servo_hostname(dut_hostname):
655 """Given a DUT's hostname, return the hostname of its servo.
656
657 @param dut_hostname: hostname of a DUT.
658
659 @return hostname of the DUT's servo.
660
661 """
662 host_parts = dut_hostname.split('.')
663 host_parts[0] = host_parts[0] + '-servo'
664 return '.'.join(host_parts)
665
666
667def servo_host_is_up(servo_hostname):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700668 """Given a servo host name, return if it's up or not.
Richard Barnetteea3e4602016-06-10 12:36:41 -0700669
670 @param servo_hostname: hostname of the servo host.
671
672 @return True if it's up, False otherwise
673 """
674 # Technically, this duplicates the SSH ping done early in the servo
675 # proxy initialization code. However, this ping ends in a couple
676 # seconds when if fails, rather than the 60 seconds it takes to decide
677 # that an SSH ping has timed out. Specifically, that timeout happens
678 # when our servo DNS name resolves, but there is no host at that IP.
679 logging.info('Pinging servo host at %s', servo_hostname)
680 ping_config = ping_runner.PingConfig(
681 servo_hostname, count=3,
682 ignore_result=True, ignore_status=True)
683 return ping_runner.PingRunner().ping(ping_config).received > 0
684
685
Richard Barnettee519dcd2016-08-15 17:37:17 -0700686def _map_afe_board_to_servo_board(afe_board):
687 """Map a board we get from the AFE to a servo appropriate value.
688
689 Many boards are identical to other boards for servo's purposes.
690 This function makes that mapping.
691
692 @param afe_board string board name received from AFE.
693 @return board we expect servo to have.
694
695 """
696 KNOWN_SUFFIXES = ['-freon', '_freon', '_moblab', '-cheets']
697 BOARD_MAP = {'gizmo': 'panther'}
698 mapped_board = afe_board
699 if afe_board in BOARD_MAP:
700 mapped_board = BOARD_MAP[afe_board]
701 else:
702 for suffix in KNOWN_SUFFIXES:
703 if afe_board.endswith(suffix):
704 mapped_board = afe_board[0:-len(suffix)]
705 break
706 if mapped_board != afe_board:
707 logging.info('Mapping AFE board=%s to %s', afe_board, mapped_board)
708 return mapped_board
709
710
Richard Barnetteea3e4602016-06-10 12:36:41 -0700711def _get_standard_servo_args(dut_host):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700712 """Return servo data associated with a given DUT.
Richard Barnetteea3e4602016-06-10 12:36:41 -0700713
714 This checks for the presence of servo host and port attached to the
715 given `dut_host`. This data should be stored in the
Kevin Cheng05ae2a42016-06-06 10:12:48 -0700716 `_afe_host.attributes` field in the provided `dut_host` parameter.
Richard Barnetteea3e4602016-06-10 12:36:41 -0700717
718 @param dut_host Instance of `Host` on which to find the servo
719 attributes.
720 @return A tuple of `servo_args` dict with host and an option port,
721 plus an `is_in_lab` flag indicating whether this in the CrOS
722 test lab, or some different environment.
723 """
724 servo_args = None
725 is_in_lab = False
726 is_ssp_moblab = False
727 if utils.is_in_container():
728 is_moblab = _CONFIG.get_config_value(
729 'SSP', 'is_moblab', type=bool, default=False)
730 is_ssp_moblab = is_moblab
731 else:
732 is_moblab = utils.is_moblab()
Kevin Cheng05ae2a42016-06-06 10:12:48 -0700733 attrs = dut_host._afe_host.attributes
Richard Barnetteea3e4602016-06-10 12:36:41 -0700734 if attrs and SERVO_HOST_ATTR in attrs:
735 servo_host = attrs[SERVO_HOST_ATTR]
736 if (is_ssp_moblab and servo_host in ['localhost', '127.0.0.1']):
737 servo_host = _CONFIG.get_config_value(
738 'SSP', 'host_container_ip', type=str, default=None)
739 servo_args = {SERVO_HOST_ATTR: servo_host}
740 if SERVO_PORT_ATTR in attrs:
Kevin Cheng692e5292016-08-14 00:23:24 -0700741 try:
742 servo_port = attrs[SERVO_PORT_ATTR]
743 servo_args[SERVO_PORT_ATTR] = int(servo_port)
744 except ValueError:
745 logging.error('servo port is not an int: %s', servo_port)
746 # Let's set the servo args to None since we're not creating
747 # the ServoHost object with the proper port now.
748 servo_args = None
Kevin Cheng643ce8a2016-09-15 15:42:12 -0700749 if SERVO_SERIAL_ATTR in attrs:
750 servo_args[SERVO_SERIAL_ATTR] = attrs[SERVO_SERIAL_ATTR]
Richard Barnetteea3e4602016-06-10 12:36:41 -0700751 is_in_lab = (not is_moblab
752 and utils.host_is_in_lab_zone(servo_host))
753
754 # TODO(jrbarnette): This test to use the default lab servo hostname
755 # is a legacy that we need only until every host in the DB has
756 # proper attributes.
757 elif (not is_moblab and
758 not dnsname_mangler.is_ip_address(dut_host.hostname)):
759 servo_host = make_servo_hostname(dut_host.hostname)
760 is_in_lab = utils.host_is_in_lab_zone(servo_host)
761 if is_in_lab:
762 servo_args = {SERVO_HOST_ATTR: servo_host}
Richard Barnette9a26ad62016-06-10 12:03:08 -0700763 if servo_args is not None:
764 servo_board = afe_utils.get_board(dut_host)
765 if servo_board is not None:
766 servo_board = _map_afe_board_to_servo_board(servo_board)
767 servo_args[SERVO_BOARD_ATTR] = servo_board
Richard Barnetteea3e4602016-06-10 12:36:41 -0700768 return servo_args, is_in_lab
769
770
Dan Shi023aae32016-05-25 11:13:01 -0700771def create_servo_host(dut, servo_args, try_lab_servo=False,
Richard Barnette9a26ad62016-06-10 12:03:08 -0700772 try_servo_repair=False):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700773 """Create a ServoHost object for a given DUT, if appropriate.
Dan Shi4d478522014-02-14 13:46:32 -0800774
Richard Barnette9a26ad62016-06-10 12:03:08 -0700775 This function attempts to create and verify or repair a `ServoHost`
776 object for a servo connected to the given `dut`, subject to various
777 constraints imposed by the parameters:
778 * When the `servo_args` parameter is not `None`, a servo
779 host must be created, and must be checked with `repair()`.
780 * Otherwise, if a servo exists in the lab and `try_lab_servo` is
781 true:
782 * If `try_servo_repair` is true, then create a servo host and
783 check it with `repair()`.
784 * Otherwise, if the servo responds to `ping` then create a
785 servo host and check it with `verify()`.
Fang Denge545abb2014-12-30 18:43:47 -0800786
Richard Barnette9a26ad62016-06-10 12:03:08 -0700787 In cases where `servo_args` was not `None`, repair failure
788 exceptions are passed back to the caller; otherwise, exceptions
Richard Barnette07c2e1d2016-10-26 14:24:28 -0700789 are logged and then discarded. Note that this only happens in cases
790 where we're called from a test (not special task) control file that
791 has an explicit dependency on servo. In that case, we require that
792 repair not write to `status.log`, so as to avoid polluting test
793 results.
794
795 TODO(jrbarnette): The special handling for servo in test control
796 files is a thorn in my flesh; I dearly hope to see it cut out before
797 my retirement.
Richard Barnette9a26ad62016-06-10 12:03:08 -0700798
799 Parameters for a servo host consist of a host name, port number, and
800 DUT board, and are determined from one of these sources, in order of
801 priority:
Richard Barnetteea3e4602016-06-10 12:36:41 -0700802 * Servo attributes from the `dut` parameter take precedence over
803 all other sources of information.
804 * If a DNS entry for the servo based on the DUT hostname exists in
805 the CrOS lab network, that hostname is used with the default
Richard Barnette9a26ad62016-06-10 12:03:08 -0700806 port and the DUT's board.
Richard Barnetteea3e4602016-06-10 12:36:41 -0700807 * If no other options are found, the parameters will be taken
Richard Barnette9a26ad62016-06-10 12:03:08 -0700808 from the `servo_args` dict passed in from the caller.
Richard Barnetteea3e4602016-06-10 12:36:41 -0700809
810 @param dut An instance of `Host` from which to take
811 servo parameters (if available).
812 @param servo_args A dictionary with servo parameters to use if
813 they can't be found from `dut`. If this
814 argument is supplied, unrepaired exceptions
815 from `verify()` will be passed back to the
816 caller.
817 @param try_lab_servo If not true, servo host creation will be
818 skipped unless otherwise required by the
819 caller.
Richard Barnette9a26ad62016-06-10 12:03:08 -0700820 @param try_servo_repair If true, check a servo host with
821 `repair()` instead of `verify()`.
Dan Shi4d478522014-02-14 13:46:32 -0800822
823 @returns: A ServoHost object or None. See comments above.
824
825 """
Richard Barnette07c2e1d2016-10-26 14:24:28 -0700826 servo_dependency = servo_args is not None
Richard Barnetteea3e4602016-06-10 12:36:41 -0700827 is_in_lab = False
Richard Barnette07c2e1d2016-10-26 14:24:28 -0700828 if dut is not None and (try_lab_servo or servo_dependency):
Richard Barnetteea3e4602016-06-10 12:36:41 -0700829 servo_args_override, is_in_lab = _get_standard_servo_args(dut)
830 if servo_args_override is not None:
831 servo_args = servo_args_override
832 if servo_args is None:
833 return None
Richard Barnette07c2e1d2016-10-26 14:24:28 -0700834 if (not servo_dependency and not try_servo_repair and
Richard Barnette9a26ad62016-06-10 12:03:08 -0700835 not servo_host_is_up(servo_args[SERVO_HOST_ATTR])):
Dan Shibbb0cb62014-03-24 17:50:57 -0700836 return None
Richard Barnette9a26ad62016-06-10 12:03:08 -0700837 newhost = ServoHost(is_in_lab=is_in_lab, **servo_args)
838 # Note that the logic of repair() includes everything done
839 # by verify(). It's sufficient to call one or the other;
840 # we don't need both.
Richard Barnette07c2e1d2016-10-26 14:24:28 -0700841 if servo_dependency:
842 newhost.repair(silent=True)
Richard Barnette9a26ad62016-06-10 12:03:08 -0700843 else:
844 try:
845 if try_servo_repair:
846 newhost.repair()
847 else:
848 newhost.verify()
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700849 except Exception:
Richard Barnette9a26ad62016-06-10 12:03:08 -0700850 operation = 'repair' if try_servo_repair else 'verification'
851 logging.exception('Servo %s failed for %s',
852 operation, newhost.hostname)
853 return newhost