blob: 4067f6a4c0eba9cd57bf005282eb629a4e0830e1 [file] [log] [blame]
Fang Deng5d518f42013-08-02 14:04:32 -07001# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4#
5# Expects to be run in an environment with sudo and no interactive password
6# prompt, such as within the Chromium OS development chroot.
7
8
9"""This file provides core logic for servo verify/repair process."""
10
11
12import httplib
13import logging
14import socket
Fang Deng5d518f42013-08-02 14:04:32 -070015import xmlrpclib
16
17from autotest_lib.client.bin import utils
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -070018from autotest_lib.client.common_lib import control_data
Fang Deng5d518f42013-08-02 14:04:32 -070019from autotest_lib.client.common_lib import error
beeps5e8c45a2013-12-17 22:05:11 -080020from autotest_lib.client.common_lib import global_config
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -070021from autotest_lib.client.common_lib import host_states
Richard Barnette9a26ad62016-06-10 12:03:08 -070022from autotest_lib.client.common_lib import hosts
Dan Shi0942b1d2015-03-31 11:07:00 -070023from autotest_lib.client.common_lib import lsbrelease_utils
beeps5e8c45a2013-12-17 22:05:11 -080024from autotest_lib.client.common_lib.cros import autoupdater
25from autotest_lib.client.common_lib.cros import dev_server
Fang Deng5d518f42013-08-02 14:04:32 -070026from autotest_lib.client.common_lib.cros import retry
Christopher Wileycef1f902014-06-19 11:11:23 -070027from autotest_lib.client.common_lib.cros.network import ping_runner
Hsinyu Chaoe0b08e62015-08-11 10:50:37 +000028from autotest_lib.client.cros import constants as client_constants
Richard Barnettee519dcd2016-08-15 17:37:17 -070029from autotest_lib.server import afe_utils
beeps5e8c45a2013-12-17 22:05:11 -080030from autotest_lib.server import site_utils as server_site_utils
Cheng-Yi Chiang22612862015-08-20 20:39:57 +080031from autotest_lib.server.cros import dnsname_mangler
Simran Basi0739d682015-02-25 16:22:56 -080032from autotest_lib.server.cros.dynamic_suite import frontend_wrappers
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -070033from autotest_lib.server.cros.dynamic_suite import control_file_getter
Richard Barnette9a26ad62016-06-10 12:03:08 -070034from autotest_lib.server.cros.servo import servo
35from autotest_lib.server.hosts import servo_repair
Fang Deng5d518f42013-08-02 14:04:32 -070036from autotest_lib.server.hosts import ssh_host
Fang Dengd4fe7392013-09-20 12:18:21 -070037from autotest_lib.site_utils.rpm_control_system import rpm_client
Fang Deng5d518f42013-08-02 14:04:32 -070038
Dan Shi5e2efb72017-02-07 11:40:23 -080039try:
40 from chromite.lib import metrics
41except ImportError:
42 metrics = utils.metrics_mock
43
Fang Deng5d518f42013-08-02 14:04:32 -070044
Simran Basi0739d682015-02-25 16:22:56 -080045# Names of the host attributes in the database that represent the values for
46# the servo_host and servo_port for a servo connected to the DUT.
47SERVO_HOST_ATTR = 'servo_host'
48SERVO_PORT_ATTR = 'servo_port'
Richard Barnettee519dcd2016-08-15 17:37:17 -070049SERVO_BOARD_ATTR = 'servo_board'
Kevin Cheng643ce8a2016-09-15 15:42:12 -070050SERVO_SERIAL_ATTR = 'servo_serial'
Simran Basi0739d682015-02-25 16:22:56 -080051
Dan Shi3b2adf62015-09-02 17:46:54 -070052_CONFIG = global_config.global_config
xixuan6cf6d2f2016-01-29 15:29:00 -080053ENABLE_SSH_TUNNEL_FOR_SERVO = _CONFIG.get_config_value(
54 'CROS', 'enable_ssh_tunnel_for_servo', type=bool, default=False)
Simran Basi0739d682015-02-25 16:22:56 -080055
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -070056AUTOTEST_BASE = _CONFIG.get_config_value(
57 'SCHEDULER', 'drone_installation_directory',
58 default='/usr/local/autotest')
59
60_SERVO_HOST_REBOOT_TEST_NAME = 'servohost_Reboot'
Kevin Cheng55265902016-10-19 12:46:50 -070061_SERVO_HOST_FORCE_REBOOT_TEST_NAME = 'servohost_Reboot.force_reboot'
Fang Deng5d518f42013-08-02 14:04:32 -070062
Fang Deng5d518f42013-08-02 14:04:32 -070063class ServoHost(ssh_host.SSHHost):
64 """Host class for a host that controls a servo, e.g. beaglebone."""
65
Richard Barnette9a26ad62016-06-10 12:03:08 -070066 DEFAULT_PORT = 9999
67
Dan Shie5b3c512014-08-21 12:12:09 -070068 # Timeout for initializing servo signals.
Wai-Hong Tam37b6ed32017-09-19 15:52:39 -070069 INITIALIZE_SERVO_TIMEOUT_SECS = 60
Richard Barnette9a26ad62016-06-10 12:03:08 -070070
xixuan6cf6d2f2016-01-29 15:29:00 -080071 # Ready test function
72 SERVO_READY_METHOD = 'get_version'
Fang Deng5d518f42013-08-02 14:04:32 -070073
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -070074 REBOOT_CMD = 'sleep 1; reboot & sleep 10; reboot -f'
75
Fang Deng5d518f42013-08-02 14:04:32 -070076
Richard Barnette17bfc6c2016-08-04 18:41:43 -070077 def _initialize(self, servo_host='localhost',
Richard Barnettee519dcd2016-08-15 17:37:17 -070078 servo_port=DEFAULT_PORT, servo_board=None,
Kevin Cheng643ce8a2016-09-15 15:42:12 -070079 servo_serial=None, is_in_lab=None, *args, **dargs):
Fang Deng5d518f42013-08-02 14:04:32 -070080 """Initialize a ServoHost instance.
81
82 A ServoHost instance represents a host that controls a servo.
83
84 @param servo_host: Name of the host where the servod process
85 is running.
86 @param servo_port: Port the servod process is listening on.
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -070087 @param servo_board: Board that the servo is connected to.
Dan Shi4d478522014-02-14 13:46:32 -080088 @param is_in_lab: True if the servo host is in Cros Lab. Default is set
89 to None, for which utils.host_is_in_lab_zone will be
90 called to check if the servo host is in Cros lab.
Fang Deng5d518f42013-08-02 14:04:32 -070091
92 """
93 super(ServoHost, self)._initialize(hostname=servo_host,
94 *args, **dargs)
Richard Barnettee519dcd2016-08-15 17:37:17 -070095 self.servo_port = servo_port
96 self.servo_board = servo_board
Kevin Cheng643ce8a2016-09-15 15:42:12 -070097 self.servo_serial = servo_serial
Richard Barnettee519dcd2016-08-15 17:37:17 -070098 self._servo = None
Richard Barnette9a26ad62016-06-10 12:03:08 -070099 self._repair_strategy = (
100 servo_repair.create_servo_repair_strategy())
Richard Barnettee519dcd2016-08-15 17:37:17 -0700101 self._is_localhost = (self.hostname == 'localhost')
102 if self._is_localhost:
103 self._is_in_lab = False
104 elif is_in_lab is None:
Dan Shi4d478522014-02-14 13:46:32 -0800105 self._is_in_lab = utils.host_is_in_lab_zone(self.hostname)
106 else:
107 self._is_in_lab = is_in_lab
xixuan6cf6d2f2016-01-29 15:29:00 -0800108
Richard Barnettee519dcd2016-08-15 17:37:17 -0700109 # Commands on the servo host must be run by the superuser.
110 # Our account on a remote host is root, but if our target is
111 # localhost then we might be running unprivileged. If so,
112 # `sudo` will have to be added to the commands.
Fang Deng5d518f42013-08-02 14:04:32 -0700113 if self._is_localhost:
114 self._sudo_required = utils.system_output('id -u') != '0'
115 else:
116 self._sudo_required = False
Richard Barnettee519dcd2016-08-15 17:37:17 -0700117
Richard Barnette9a26ad62016-06-10 12:03:08 -0700118
119 def connect_servo(self):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700120 """Establish a connection to the servod server on this host.
Richard Barnette9a26ad62016-06-10 12:03:08 -0700121
122 Initializes `self._servo` and then verifies that all network
123 connections are working. This will create an ssh tunnel if
124 it's required.
125
126 As a side effect of testing the connection, all signals on the
127 target servo are reset to default values, and the USB stick is
128 set to the neutral (off) position.
129 """
Kevin Cheng643ce8a2016-09-15 15:42:12 -0700130 servo_obj = servo.Servo(servo_host=self, servo_serial=self.servo_serial)
Richard Barnette9a26ad62016-06-10 12:03:08 -0700131 timeout, _ = retry.timeout(
132 servo_obj.initialize_dut,
133 timeout_sec=self.INITIALIZE_SERVO_TIMEOUT_SECS)
134 if timeout:
135 raise hosts.AutoservVerifyError(
136 'Servo initialize timed out.')
137 self._servo = servo_obj
138
139
140 def disconnect_servo(self):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700141 """Disconnect our servo if it exists.
Richard Barnette9a26ad62016-06-10 12:03:08 -0700142
143 If we've previously successfully connected to our servo,
144 disconnect any established ssh tunnel, and set `self._servo`
145 back to `None`.
146 """
147 if self._servo:
148 # N.B. This call is safe even without a tunnel:
149 # rpc_server_tracker.disconnect() silently ignores
150 # unknown ports.
151 self.rpc_server_tracker.disconnect(self.servo_port)
152 self._servo = None
Fang Deng5d518f42013-08-02 14:04:32 -0700153
154
155 def is_in_lab(self):
156 """Check whether the servo host is a lab device.
157
158 @returns: True if the servo host is in Cros Lab, otherwise False.
159
160 """
161 return self._is_in_lab
162
163
164 def is_localhost(self):
165 """Checks whether the servo host points to localhost.
166
167 @returns: True if it points to localhost, otherwise False.
168
169 """
170 return self._is_localhost
171
172
173 def get_servod_server_proxy(self):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700174 """Return a proxy that can be used to communicate with servod server.
Fang Deng5d518f42013-08-02 14:04:32 -0700175
176 @returns: An xmlrpclib.ServerProxy that is connected to the servod
177 server on the host.
Fang Deng5d518f42013-08-02 14:04:32 -0700178 """
Richard Barnette9a26ad62016-06-10 12:03:08 -0700179 if ENABLE_SSH_TUNNEL_FOR_SERVO and not self.is_localhost():
180 return self.rpc_server_tracker.xmlrpc_connect(
181 None, self.servo_port,
182 ready_test_name=self.SERVO_READY_METHOD,
183 timeout_seconds=60)
184 else:
185 remote = 'http://%s:%s' % (self.hostname, self.servo_port)
186 return xmlrpclib.ServerProxy(remote)
Fang Deng5d518f42013-08-02 14:04:32 -0700187
188
Richard Barnette9a26ad62016-06-10 12:03:08 -0700189 def is_cros_host(self):
beeps5e8c45a2013-12-17 22:05:11 -0800190 """Check if a servo host is running chromeos.
191
192 @return: True if the servo host is running chromeos.
193 False if it isn't, or we don't have enough information.
194 """
195 try:
196 result = self.run('grep -q CHROMEOS /etc/lsb-release',
197 ignore_status=True, timeout=10)
198 except (error.AutoservRunError, error.AutoservSSHTimeout):
199 return False
200 return result.exit_status == 0
201
202
Fang Deng5d518f42013-08-02 14:04:32 -0700203 def make_ssh_command(self, user='root', port=22, opts='', hosts_file=None,
204 connect_timeout=None, alive_interval=None):
205 """Override default make_ssh_command to use tuned options.
206
207 Tuning changes:
208 - ConnectTimeout=30; maximum of 30 seconds allowed for an SSH
209 connection failure. Consistency with remote_access.py.
210
211 - ServerAliveInterval=180; which causes SSH to ping connection every
212 180 seconds. In conjunction with ServerAliveCountMax ensures
213 that if the connection dies, Autotest will bail out quickly.
214
215 - ServerAliveCountMax=3; consistency with remote_access.py.
216
217 - ConnectAttempts=4; reduce flakiness in connection errors;
218 consistency with remote_access.py.
219
220 - UserKnownHostsFile=/dev/null; we don't care about the keys.
221
222 - SSH protocol forced to 2; needed for ServerAliveInterval.
223
224 @param user User name to use for the ssh connection.
225 @param port Port on the target host to use for ssh connection.
226 @param opts Additional options to the ssh command.
227 @param hosts_file Ignored.
228 @param connect_timeout Ignored.
229 @param alive_interval Ignored.
230
231 @returns: An ssh command with the requested settings.
232
233 """
234 base_command = ('/usr/bin/ssh -a -x %s -o StrictHostKeyChecking=no'
235 ' -o UserKnownHostsFile=/dev/null -o BatchMode=yes'
236 ' -o ConnectTimeout=30 -o ServerAliveInterval=180'
237 ' -o ServerAliveCountMax=3 -o ConnectionAttempts=4'
238 ' -o Protocol=2 -l %s -p %d')
239 return base_command % (opts, user, port)
240
241
242 def _make_scp_cmd(self, sources, dest):
243 """Format scp command.
244
245 Given a list of source paths and a destination path, produces the
246 appropriate scp command for encoding it. Remote paths must be
247 pre-encoded. Overrides _make_scp_cmd in AbstractSSHHost
248 to allow additional ssh options.
249
250 @param sources: A list of source paths to copy from.
251 @param dest: Destination path to copy to.
252
253 @returns: An scp command that copies |sources| on local machine to
254 |dest| on the remote servo host.
255
256 """
257 command = ('scp -rq %s -o BatchMode=yes -o StrictHostKeyChecking=no '
258 '-o UserKnownHostsFile=/dev/null -P %d %s "%s"')
259 return command % (self.master_ssh_option,
260 self.port, ' '.join(sources), dest)
261
262
263 def run(self, command, timeout=3600, ignore_status=False,
264 stdout_tee=utils.TEE_TO_LOGS, stderr_tee=utils.TEE_TO_LOGS,
Luigi Semenzatobfbd1f32017-01-06 10:41:18 -0800265 connect_timeout=30, ssh_failure_retry_ok=False,
266 options='', stdin=None, verbose=True, args=()):
Fang Deng5d518f42013-08-02 14:04:32 -0700267 """Run a command on the servo host.
268
269 Extends method `run` in SSHHost. If the servo host is a remote device,
270 it will call `run` in SSHost without changing anything.
271 If the servo host is 'localhost', it will call utils.system_output.
272
273 @param command: The command line string.
274 @param timeout: Time limit in seconds before attempting to
275 kill the running process. The run() function
276 will take a few seconds longer than 'timeout'
277 to complete if it has to kill the process.
278 @param ignore_status: Do not raise an exception, no matter
279 what the exit code of the command is.
280 @param stdout_tee/stderr_tee: Where to tee the stdout/stderr.
281 @param connect_timeout: SSH connection timeout (in seconds)
282 Ignored if host is 'localhost'.
283 @param options: String with additional ssh command options
284 Ignored if host is 'localhost'.
Luigi Semenzatobfbd1f32017-01-06 10:41:18 -0800285 @param ssh_failure_retry_ok: when True and ssh connection failure is
286 suspected, OK to retry command (but not
287 compulsory, and likely not needed here)
Fang Deng5d518f42013-08-02 14:04:32 -0700288 @param stdin: Stdin to pass (a string) to the executed command.
289 @param verbose: Log the commands.
290 @param args: Sequence of strings to pass as arguments to command by
291 quoting them in " and escaping their contents if necessary.
292
293 @returns: A utils.CmdResult object.
294
295 @raises AutoservRunError if the command failed.
296 @raises AutoservSSHTimeout SSH connection has timed out. Only applies
297 when servo host is not 'localhost'.
298
299 """
300 run_args = {'command': command, 'timeout': timeout,
301 'ignore_status': ignore_status, 'stdout_tee': stdout_tee,
302 'stderr_tee': stderr_tee, 'stdin': stdin,
303 'verbose': verbose, 'args': args}
304 if self.is_localhost():
305 if self._sudo_required:
Michael Tangf9b3ada2016-11-18 16:01:05 -0800306 run_args['command'] = 'sudo -n sh -c "%s"' % utils.sh_escape(
307 command)
Fang Deng5d518f42013-08-02 14:04:32 -0700308 try:
309 return utils.run(**run_args)
310 except error.CmdError as e:
311 logging.error(e)
312 raise error.AutoservRunError('command execution error',
313 e.result_obj)
314 else:
315 run_args['connect_timeout'] = connect_timeout
316 run_args['options'] = options
317 return super(ServoHost, self).run(**run_args)
318
319
Richard Barnette9a26ad62016-06-10 12:03:08 -0700320 def _get_release_version(self):
Dan Shi0942b1d2015-03-31 11:07:00 -0700321 """Get the value of attribute CHROMEOS_RELEASE_VERSION from lsb-release.
322
323 @returns The version string in lsb-release, under attribute
324 CHROMEOS_RELEASE_VERSION.
325 """
326 lsb_release_content = self.run(
327 'cat "%s"' % client_constants.LSB_RELEASE).stdout.strip()
328 return lsbrelease_utils.get_chromeos_release_version(
329 lsb_release_content=lsb_release_content)
330
331
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700332 def get_attached_duts(self, afe):
333 """Gather a list of duts that use this servo host.
334
335 @param afe: afe instance.
336
337 @returns list of duts.
Richard Barnette3a7697f2016-04-20 11:33:27 -0700338 """
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700339 return afe.get_hosts_by_attribute(
340 attribute=SERVO_HOST_ATTR, value=self.hostname)
341
342
343 def get_board(self):
344 """Determine the board for this servo host.
345
346 @returns a string representing this servo host's board.
347 """
348 return lsbrelease_utils.get_current_board(
349 lsb_release_content=self.run('cat /etc/lsb-release').stdout)
350
351
352 def _choose_dut_for_synchronized_reboot(self, dut_list, afe):
353 """Choose which dut to schedule servo host reboot job.
354
355 We'll want a semi-deterministic way of selecting which host should be
356 scheduled for the servo host reboot job. For now we'll sort the
357 list with the expectation the dut list will stay consistent.
358 From there we'll grab the first dut that is available so we
359 don't schedule a job on a dut that will never run.
360
361 @param dut_list: List of the dut hostnames to choose from.
362 @param afe: Instance of the AFE.
363
364 @return hostname of dut to schedule job on.
365 """
366 afe_hosts = afe.get_hosts(dut_list)
367 afe_hosts.sort()
368 for afe_host in afe_hosts:
369 if afe_host.status not in host_states.UNAVAILABLE_STATES:
370 return afe_host.hostname
371 # If they're all unavailable, just return the first sorted dut.
372 dut_list.sort()
373 return dut_list[0]
374
375
376 def _sync_job_scheduled_for_duts(self, dut_list, afe):
377 """Checks if a synchronized reboot has been scheduled for these duts.
378
379 Grab all the host queue entries that aren't completed for the duts and
380 see if any of them have the expected job name.
381
382 @param dut_list: List of duts to check on.
383 @param afe: Instance of the AFE.
384
385 @returns True if the job is scheduled, False otherwise.
386 """
387 afe_hosts = afe.get_hosts(dut_list)
388 for afe_host in afe_hosts:
389 hqes = afe.get_host_queue_entries(host=afe_host.id, complete=0)
390 for hqe in hqes:
391 job = afe.get_jobs(id=hqe.job.id)
Kevin Cheng55265902016-10-19 12:46:50 -0700392 if job and job[0].name in (_SERVO_HOST_REBOOT_TEST_NAME,
393 _SERVO_HOST_FORCE_REBOOT_TEST_NAME):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700394 return True
395 return False
396
397
Kevin Cheng55265902016-10-19 12:46:50 -0700398 def schedule_synchronized_reboot(self, dut_list, afe, force_reboot=False):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700399 """Schedule a job to reboot the servo host.
400
401 When we schedule a job, it will create a ServoHost object which will
402 go through this entire flow of checking if a reboot is needed and
403 trying to schedule it. There is probably a better approach to setting
404 up a synchronized reboot but I'm coming up short on better ideas so I
405 apologize for this circus show.
406
Kevin Cheng55265902016-10-19 12:46:50 -0700407 @param dut_list: List of duts that need to be locked.
408 @param afe: Instance of afe.
409 @param force_reboot: Boolean to indicate if a forced reboot should be
410 scheduled or not.
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700411 """
412 # If we've already scheduled job on a dut, we're done here.
413 if self._sync_job_scheduled_for_duts(dut_list, afe):
414 return
415
416 # Looks like we haven't scheduled a job yet.
Kevin Cheng55265902016-10-19 12:46:50 -0700417 test = (_SERVO_HOST_REBOOT_TEST_NAME if not force_reboot
418 else _SERVO_HOST_FORCE_REBOOT_TEST_NAME)
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700419 dut = self._choose_dut_for_synchronized_reboot(dut_list, afe)
420 getter = control_file_getter.FileSystemGetter([AUTOTEST_BASE])
Kevin Cheng55265902016-10-19 12:46:50 -0700421 control_file = getter.get_control_file_contents_by_name(test)
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700422 control_type = control_data.CONTROL_TYPE_NAMES.SERVER
Kevin Cheng79589982016-10-25 13:26:04 -0700423 try:
424 afe.create_job(control_file=control_file, name=test,
425 control_type=control_type, hosts=[dut])
426 except Exception as e:
427 # Sometimes creating the job will raise an exception. We'll log it
428 # but we don't want to fail because of it.
Aviv Keshet5ae0a002017-05-05 10:23:33 -0700429 logging.exception('Scheduling reboot job failed due to Exception.')
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700430
431
432 def reboot(self, *args, **dargs):
433 """Reboot using special servo host reboot command."""
434 super(ServoHost, self).reboot(reboot_cmd=self.REBOOT_CMD,
435 *args, **dargs)
436
437
438 def _check_for_reboot(self, updater):
439 """Reboot this servo host if an upgrade is waiting.
Richard Barnette3a7697f2016-04-20 11:33:27 -0700440
441 If the host has successfully downloaded and finalized a new
442 build, reboot.
443
444 @param updater: a ChromiumOSUpdater instance for checking
445 whether reboot is needed.
446 @return Return a (status, build) tuple reflecting the
447 update_engine status and current build of the host
448 at the end of the call.
449 """
Richard Barnette9a26ad62016-06-10 12:03:08 -0700450 current_build_number = self._get_release_version()
Richard Barnette3a7697f2016-04-20 11:33:27 -0700451 status = updater.check_update_status()
452 if status == autoupdater.UPDATER_NEED_REBOOT:
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700453 # Check if we need to schedule an organized reboot.
Kevin Cheng79589982016-10-25 13:26:04 -0700454 afe = frontend_wrappers.RetryingAFE(
455 timeout_min=5, delay_sec=10,
456 server=server_site_utils.get_global_afe_hostname())
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700457 dut_list = self.get_attached_duts(afe)
458 logging.info('servo host has the following duts: %s', dut_list)
459 if len(dut_list) > 1:
460 logging.info('servo host has multiple duts, scheduling '
461 'synchronized reboot')
462 self.schedule_synchronized_reboot(dut_list, afe)
463 return status, current_build_number
464
465 logging.info('Rebooting servo host %s from build %s',
Richard Barnette3a7697f2016-04-20 11:33:27 -0700466 self.hostname, current_build_number)
467 # Tell the reboot() call not to wait for completion.
468 # Otherwise, the call will log reboot failure if servo does
469 # not come back. The logged reboot failure will lead to
470 # test job failure. If the test does not require servo, we
471 # don't want servo failure to fail the test with error:
472 # `Host did not return from reboot` in status.log.
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700473 self.reboot(fastsync=True, wait=False)
Richard Barnette3a7697f2016-04-20 11:33:27 -0700474
475 # We told the reboot() call not to wait, but we need to wait
476 # for the reboot before we continue. Alas. The code from
477 # here below is basically a copy of Host.wait_for_restart(),
478 # with the logging bits ripped out, so that they can't cause
479 # the failure logging problem described above.
480 #
481 # The black stain that this has left on my soul can never be
482 # erased.
483 old_boot_id = self.get_boot_id()
484 if not self.wait_down(timeout=self.WAIT_DOWN_REBOOT_TIMEOUT,
485 warning_timer=self.WAIT_DOWN_REBOOT_WARNING,
486 old_boot_id=old_boot_id):
487 raise error.AutoservHostError(
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700488 'servo host %s failed to shut down.' %
489 self.hostname)
Richard Barnette3a7697f2016-04-20 11:33:27 -0700490 if self.wait_up(timeout=120):
Richard Barnette9a26ad62016-06-10 12:03:08 -0700491 current_build_number = self._get_release_version()
Richard Barnette3a7697f2016-04-20 11:33:27 -0700492 status = updater.check_update_status()
493 logging.info('servo host %s back from reboot, with build %s',
494 self.hostname, current_build_number)
495 else:
496 raise error.AutoservHostError(
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700497 'servo host %s failed to come back from reboot.' %
498 self.hostname)
Richard Barnette3a7697f2016-04-20 11:33:27 -0700499 return status, current_build_number
500
501
Richard Barnette3a7697f2016-04-20 11:33:27 -0700502 def update_image(self, wait_for_update=False):
beeps5e8c45a2013-12-17 22:05:11 -0800503 """Update the image on the servo host, if needed.
504
J. Richard Barnette84895392015-04-30 12:31:01 -0700505 This method recognizes the following cases:
506 * If the Host is not running Chrome OS, do nothing.
507 * If a previously triggered update is now complete, reboot
508 to the new version.
509 * If the host is processing a previously triggered update,
510 do nothing.
511 * If the host is running a version of Chrome OS different
512 from the default for servo Hosts, trigger an update, but
513 don't wait for it to complete.
beeps5e8c45a2013-12-17 22:05:11 -0800514
Richard Barnette3a7697f2016-04-20 11:33:27 -0700515 @param wait_for_update If an update needs to be applied and
516 this is true, then don't return until the update is
517 downloaded and finalized, and the host rebooted.
beeps5e8c45a2013-12-17 22:05:11 -0800518 @raises dev_server.DevServerException: If all the devservers are down.
519 @raises site_utils.ParseBuildNameException: If the devserver returns
520 an invalid build name.
521 @raises autoupdater.ChromiumOSError: If something goes wrong in the
522 checking update engine client status or applying an update.
523 @raises AutoservRunError: If the update_engine_client isn't present on
524 the host, and the host is a cros_host.
J. Richard Barnette84895392015-04-30 12:31:01 -0700525
beeps5e8c45a2013-12-17 22:05:11 -0800526 """
Dan Shib795b5a2015-09-24 13:26:35 -0700527 # servod could be running in a Ubuntu workstation.
Richard Barnette9a26ad62016-06-10 12:03:08 -0700528 if not self.is_cros_host():
beeps5e8c45a2013-12-17 22:05:11 -0800529 logging.info('Not attempting an update, either %s is not running '
530 'chromeos or we cannot find enough information about '
531 'the host.', self.hostname)
532 return
533
Dan Shib795b5a2015-09-24 13:26:35 -0700534 if lsbrelease_utils.is_moblab():
535 logging.info('Not attempting an update, %s is running moblab.',
536 self.hostname)
537 return
538
Richard Barnette383ef9c2016-12-13 11:56:49 -0800539 target_build = afe_utils.get_stable_cros_image_name(self.get_board())
J. Richard Barnette84895392015-04-30 12:31:01 -0700540 target_build_number = server_site_utils.ParseBuildName(
541 target_build)[3]
xixuanfa2d92a2016-12-09 09:45:27 -0800542 # For servo image staging, we want it as more widely distributed as
543 # possible, so that devservers' load can be evenly distributed. So use
544 # hostname instead of target_build as hash.
545 ds = dev_server.ImageServer.resolve(self.hostname,
546 hostname=self.hostname)
J. Richard Barnette84895392015-04-30 12:31:01 -0700547 url = ds.get_update_url(target_build)
beeps5e8c45a2013-12-17 22:05:11 -0800548
549 updater = autoupdater.ChromiumOSUpdater(update_url=url, host=self)
Richard Barnette3a7697f2016-04-20 11:33:27 -0700550 status, current_build_number = self._check_for_reboot(updater)
551 update_pending = True
beeps5e8c45a2013-12-17 22:05:11 -0800552 if status in autoupdater.UPDATER_PROCESSING_UPDATE:
553 logging.info('servo host %s already processing an update, update '
554 'engine client status=%s', self.hostname, status)
Allen Li66aa2542017-06-26 15:26:27 -0700555 elif status == autoupdater.UPDATER_NEED_REBOOT:
556 return
J. Richard Barnette84895392015-04-30 12:31:01 -0700557 elif current_build_number != target_build_number:
beeps5e8c45a2013-12-17 22:05:11 -0800558 logging.info('Using devserver url: %s to trigger update on '
559 'servo host %s, from %s to %s', url, self.hostname,
J. Richard Barnette84895392015-04-30 12:31:01 -0700560 current_build_number, target_build_number)
beeps5e8c45a2013-12-17 22:05:11 -0800561 try:
J. Richard Barnette84895392015-04-30 12:31:01 -0700562 ds.stage_artifacts(target_build,
563 artifacts=['full_payload'])
564 except Exception as e:
565 logging.error('Staging artifacts failed: %s', str(e))
566 logging.error('Abandoning update for this cycle.')
beeps5e8c45a2013-12-17 22:05:11 -0800567 else:
J. Richard Barnette84895392015-04-30 12:31:01 -0700568 try:
Richard Barnette7e53aa02016-05-20 10:49:40 -0700569 # TODO(jrbarnette): This 'touch' is a gross hack
570 # to get us past crbug.com/613603. Once that
571 # bug is resolved, we should remove this code.
572 self.run('touch /home/chronos/.oobe_completed')
J. Richard Barnette84895392015-04-30 12:31:01 -0700573 updater.trigger_update()
574 except autoupdater.RootFSUpdateError as e:
575 trigger_download_status = 'failed with %s' % str(e)
Aviv Keshet11836322016-11-22 11:32:01 -0800576 metrics.Counter('chromeos/autotest/servo/'
577 'rootfs_update_failed').increment()
J. Richard Barnette84895392015-04-30 12:31:01 -0700578 else:
579 trigger_download_status = 'passed'
580 logging.info('Triggered download and update %s for %s, '
581 'update engine currently in status %s',
582 trigger_download_status, self.hostname,
583 updater.check_update_status())
beeps5e8c45a2013-12-17 22:05:11 -0800584 else:
585 logging.info('servo host %s does not require an update.',
586 self.hostname)
Richard Barnette3a7697f2016-04-20 11:33:27 -0700587 update_pending = False
588
589 if update_pending and wait_for_update:
590 logging.info('Waiting for servo update to complete.')
591 self.run('update_engine_client --follow', ignore_status=True)
beeps5e8c45a2013-12-17 22:05:11 -0800592
593
Richard Barnette1edbb162016-11-01 11:47:50 -0700594 def verify(self, silent=False):
595 """Update the servo host and verify it's in a good state.
596
597 @param silent If true, suppress logging in `status.log`.
598 """
Richard Barnette79d78c42016-05-25 09:31:21 -0700599 # TODO(jrbarnette) Old versions of beaglebone_servo include
Richard Barnette9a26ad62016-06-10 12:03:08 -0700600 # the powerd package. If you touch the .oobe_completed file
601 # (as we do to work around an update_engine problem), then
602 # powerd will eventually shut down the beaglebone for lack
603 # of (apparent) activity. Current versions of
Richard Barnette79d78c42016-05-25 09:31:21 -0700604 # beaglebone_servo don't have powerd, but until we can purge
605 # the lab of the old images, we need to make sure powerd
606 # isn't running.
607 self.run('stop powerd', ignore_status=True)
Richard Barnette9a26ad62016-06-10 12:03:08 -0700608 try:
Richard Barnette1edbb162016-11-01 11:47:50 -0700609 self._repair_strategy.verify(self, silent)
Richard Barnette9a26ad62016-06-10 12:03:08 -0700610 except:
611 self.disconnect_servo()
612 raise
Fang Deng5d518f42013-08-02 14:04:32 -0700613
614
Richard Barnette1edbb162016-11-01 11:47:50 -0700615 def repair(self, silent=False):
616 """Attempt to repair servo host.
617
618 @param silent If true, suppress logging in `status.log`.
619 """
Richard Barnette9a26ad62016-06-10 12:03:08 -0700620 try:
Richard Barnette1edbb162016-11-01 11:47:50 -0700621 self._repair_strategy.repair(self, silent)
Richard Barnette9a26ad62016-06-10 12:03:08 -0700622 except:
623 self.disconnect_servo()
624 raise
Fang Deng5d518f42013-08-02 14:04:32 -0700625
626
Fang Dengd4fe7392013-09-20 12:18:21 -0700627 def has_power(self):
628 """Return whether or not the servo host is powered by PoE."""
629 # TODO(fdeng): See crbug.com/302791
630 # For now, assume all servo hosts in the lab have power.
631 return self.is_in_lab()
632
633
634 def power_cycle(self):
635 """Cycle power to this host via PoE if it is a lab device.
636
Richard Barnette9a26ad62016-06-10 12:03:08 -0700637 @raises AutoservRepairError if it fails to power cycle the
Fang Dengd4fe7392013-09-20 12:18:21 -0700638 servo host.
639
640 """
641 if self.has_power():
642 try:
643 rpm_client.set_power(self.hostname, 'CYCLE')
644 except (socket.error, xmlrpclib.Error,
645 httplib.BadStatusLine,
646 rpm_client.RemotePowerException) as e:
Richard Barnette9a26ad62016-06-10 12:03:08 -0700647 raise hosts.AutoservRepairError(
Fang Dengd4fe7392013-09-20 12:18:21 -0700648 'Power cycling %s failed: %s' % (self.hostname, e))
649 else:
650 logging.info('Skipping power cycling, not a lab device.')
651
652
Dan Shi4d478522014-02-14 13:46:32 -0800653 def get_servo(self):
654 """Get the cached servo.Servo object.
Fang Deng5d518f42013-08-02 14:04:32 -0700655
Dan Shi4d478522014-02-14 13:46:32 -0800656 @return: a servo.Servo object.
Fang Deng5d518f42013-08-02 14:04:32 -0700657 """
Dan Shi4d478522014-02-14 13:46:32 -0800658 return self._servo
659
660
Richard Barnetteea3e4602016-06-10 12:36:41 -0700661def make_servo_hostname(dut_hostname):
662 """Given a DUT's hostname, return the hostname of its servo.
663
664 @param dut_hostname: hostname of a DUT.
665
666 @return hostname of the DUT's servo.
667
668 """
669 host_parts = dut_hostname.split('.')
670 host_parts[0] = host_parts[0] + '-servo'
671 return '.'.join(host_parts)
672
673
674def servo_host_is_up(servo_hostname):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700675 """Given a servo host name, return if it's up or not.
Richard Barnetteea3e4602016-06-10 12:36:41 -0700676
677 @param servo_hostname: hostname of the servo host.
678
679 @return True if it's up, False otherwise
680 """
681 # Technically, this duplicates the SSH ping done early in the servo
682 # proxy initialization code. However, this ping ends in a couple
683 # seconds when if fails, rather than the 60 seconds it takes to decide
684 # that an SSH ping has timed out. Specifically, that timeout happens
685 # when our servo DNS name resolves, but there is no host at that IP.
686 logging.info('Pinging servo host at %s', servo_hostname)
687 ping_config = ping_runner.PingConfig(
688 servo_hostname, count=3,
689 ignore_result=True, ignore_status=True)
690 return ping_runner.PingRunner().ping(ping_config).received > 0
691
692
Richard Barnettee519dcd2016-08-15 17:37:17 -0700693def _map_afe_board_to_servo_board(afe_board):
694 """Map a board we get from the AFE to a servo appropriate value.
695
696 Many boards are identical to other boards for servo's purposes.
697 This function makes that mapping.
698
699 @param afe_board string board name received from AFE.
700 @return board we expect servo to have.
701
702 """
703 KNOWN_SUFFIXES = ['-freon', '_freon', '_moblab', '-cheets']
704 BOARD_MAP = {'gizmo': 'panther'}
705 mapped_board = afe_board
706 if afe_board in BOARD_MAP:
707 mapped_board = BOARD_MAP[afe_board]
708 else:
709 for suffix in KNOWN_SUFFIXES:
710 if afe_board.endswith(suffix):
711 mapped_board = afe_board[0:-len(suffix)]
712 break
713 if mapped_board != afe_board:
714 logging.info('Mapping AFE board=%s to %s', afe_board, mapped_board)
715 return mapped_board
716
717
Richard Barnetteea3e4602016-06-10 12:36:41 -0700718def _get_standard_servo_args(dut_host):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700719 """Return servo data associated with a given DUT.
Richard Barnetteea3e4602016-06-10 12:36:41 -0700720
721 This checks for the presence of servo host and port attached to the
722 given `dut_host`. This data should be stored in the
Kevin Cheng05ae2a42016-06-06 10:12:48 -0700723 `_afe_host.attributes` field in the provided `dut_host` parameter.
Richard Barnetteea3e4602016-06-10 12:36:41 -0700724
725 @param dut_host Instance of `Host` on which to find the servo
726 attributes.
727 @return A tuple of `servo_args` dict with host and an option port,
728 plus an `is_in_lab` flag indicating whether this in the CrOS
729 test lab, or some different environment.
730 """
731 servo_args = None
732 is_in_lab = False
733 is_ssp_moblab = False
734 if utils.is_in_container():
735 is_moblab = _CONFIG.get_config_value(
736 'SSP', 'is_moblab', type=bool, default=False)
737 is_ssp_moblab = is_moblab
738 else:
739 is_moblab = utils.is_moblab()
Kevin Cheng05ae2a42016-06-06 10:12:48 -0700740 attrs = dut_host._afe_host.attributes
Richard Barnetteea3e4602016-06-10 12:36:41 -0700741 if attrs and SERVO_HOST_ATTR in attrs:
742 servo_host = attrs[SERVO_HOST_ATTR]
743 if (is_ssp_moblab and servo_host in ['localhost', '127.0.0.1']):
744 servo_host = _CONFIG.get_config_value(
745 'SSP', 'host_container_ip', type=str, default=None)
746 servo_args = {SERVO_HOST_ATTR: servo_host}
747 if SERVO_PORT_ATTR in attrs:
Kevin Cheng692e5292016-08-14 00:23:24 -0700748 try:
749 servo_port = attrs[SERVO_PORT_ATTR]
750 servo_args[SERVO_PORT_ATTR] = int(servo_port)
751 except ValueError:
752 logging.error('servo port is not an int: %s', servo_port)
753 # Let's set the servo args to None since we're not creating
754 # the ServoHost object with the proper port now.
755 servo_args = None
Kevin Cheng643ce8a2016-09-15 15:42:12 -0700756 if SERVO_SERIAL_ATTR in attrs:
757 servo_args[SERVO_SERIAL_ATTR] = attrs[SERVO_SERIAL_ATTR]
Richard Barnetteea3e4602016-06-10 12:36:41 -0700758 is_in_lab = (not is_moblab
759 and utils.host_is_in_lab_zone(servo_host))
760
761 # TODO(jrbarnette): This test to use the default lab servo hostname
762 # is a legacy that we need only until every host in the DB has
763 # proper attributes.
764 elif (not is_moblab and
765 not dnsname_mangler.is_ip_address(dut_host.hostname)):
766 servo_host = make_servo_hostname(dut_host.hostname)
767 is_in_lab = utils.host_is_in_lab_zone(servo_host)
768 if is_in_lab:
769 servo_args = {SERVO_HOST_ATTR: servo_host}
Richard Barnette9a26ad62016-06-10 12:03:08 -0700770 if servo_args is not None:
Prathmesh Prabhua3bb7652017-02-09 11:42:13 -0800771 info = dut_host.host_info_store.get()
772 if info.board:
773 servo_args[SERVO_BOARD_ATTR] = _map_afe_board_to_servo_board(
774 info.board)
Richard Barnetteea3e4602016-06-10 12:36:41 -0700775 return servo_args, is_in_lab
776
777
Dan Shi023aae32016-05-25 11:13:01 -0700778def create_servo_host(dut, servo_args, try_lab_servo=False,
Richard Barnette9a26ad62016-06-10 12:03:08 -0700779 try_servo_repair=False):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700780 """Create a ServoHost object for a given DUT, if appropriate.
Dan Shi4d478522014-02-14 13:46:32 -0800781
Richard Barnette9a26ad62016-06-10 12:03:08 -0700782 This function attempts to create and verify or repair a `ServoHost`
783 object for a servo connected to the given `dut`, subject to various
784 constraints imposed by the parameters:
785 * When the `servo_args` parameter is not `None`, a servo
786 host must be created, and must be checked with `repair()`.
787 * Otherwise, if a servo exists in the lab and `try_lab_servo` is
788 true:
789 * If `try_servo_repair` is true, then create a servo host and
790 check it with `repair()`.
791 * Otherwise, if the servo responds to `ping` then create a
792 servo host and check it with `verify()`.
Fang Denge545abb2014-12-30 18:43:47 -0800793
Richard Barnette9a26ad62016-06-10 12:03:08 -0700794 In cases where `servo_args` was not `None`, repair failure
795 exceptions are passed back to the caller; otherwise, exceptions
Richard Barnette07c2e1d2016-10-26 14:24:28 -0700796 are logged and then discarded. Note that this only happens in cases
797 where we're called from a test (not special task) control file that
798 has an explicit dependency on servo. In that case, we require that
799 repair not write to `status.log`, so as to avoid polluting test
800 results.
801
802 TODO(jrbarnette): The special handling for servo in test control
803 files is a thorn in my flesh; I dearly hope to see it cut out before
804 my retirement.
Richard Barnette9a26ad62016-06-10 12:03:08 -0700805
806 Parameters for a servo host consist of a host name, port number, and
807 DUT board, and are determined from one of these sources, in order of
808 priority:
Richard Barnetteea3e4602016-06-10 12:36:41 -0700809 * Servo attributes from the `dut` parameter take precedence over
810 all other sources of information.
811 * If a DNS entry for the servo based on the DUT hostname exists in
812 the CrOS lab network, that hostname is used with the default
Richard Barnette9a26ad62016-06-10 12:03:08 -0700813 port and the DUT's board.
Richard Barnetteea3e4602016-06-10 12:36:41 -0700814 * If no other options are found, the parameters will be taken
Richard Barnette9a26ad62016-06-10 12:03:08 -0700815 from the `servo_args` dict passed in from the caller.
Richard Barnetteea3e4602016-06-10 12:36:41 -0700816
817 @param dut An instance of `Host` from which to take
818 servo parameters (if available).
819 @param servo_args A dictionary with servo parameters to use if
820 they can't be found from `dut`. If this
821 argument is supplied, unrepaired exceptions
822 from `verify()` will be passed back to the
823 caller.
824 @param try_lab_servo If not true, servo host creation will be
825 skipped unless otherwise required by the
826 caller.
Richard Barnette9a26ad62016-06-10 12:03:08 -0700827 @param try_servo_repair If true, check a servo host with
828 `repair()` instead of `verify()`.
Dan Shi4d478522014-02-14 13:46:32 -0800829
830 @returns: A ServoHost object or None. See comments above.
831
832 """
Richard Barnette07c2e1d2016-10-26 14:24:28 -0700833 servo_dependency = servo_args is not None
Richard Barnetteea3e4602016-06-10 12:36:41 -0700834 is_in_lab = False
Richard Barnette07c2e1d2016-10-26 14:24:28 -0700835 if dut is not None and (try_lab_servo or servo_dependency):
Richard Barnetteea3e4602016-06-10 12:36:41 -0700836 servo_args_override, is_in_lab = _get_standard_servo_args(dut)
837 if servo_args_override is not None:
838 servo_args = servo_args_override
839 if servo_args is None:
840 return None
Richard Barnette07c2e1d2016-10-26 14:24:28 -0700841 if (not servo_dependency and not try_servo_repair and
Richard Barnette9a26ad62016-06-10 12:03:08 -0700842 not servo_host_is_up(servo_args[SERVO_HOST_ATTR])):
Dan Shibbb0cb62014-03-24 17:50:57 -0700843 return None
Richard Barnette9a26ad62016-06-10 12:03:08 -0700844 newhost = ServoHost(is_in_lab=is_in_lab, **servo_args)
845 # Note that the logic of repair() includes everything done
846 # by verify(). It's sufficient to call one or the other;
847 # we don't need both.
Richard Barnette07c2e1d2016-10-26 14:24:28 -0700848 if servo_dependency:
849 newhost.repair(silent=True)
Richard Barnette9a26ad62016-06-10 12:03:08 -0700850 else:
851 try:
852 if try_servo_repair:
853 newhost.repair()
854 else:
855 newhost.verify()
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700856 except Exception:
Richard Barnette9a26ad62016-06-10 12:03:08 -0700857 operation = 'repair' if try_servo_repair else 'verification'
858 logging.exception('Servo %s failed for %s',
859 operation, newhost.hostname)
860 return newhost