blob: d33a51e2476887f5582e1e65182de5829f77491b [file] [log] [blame]
Fang Deng5d518f42013-08-02 14:04:32 -07001# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4#
5# Expects to be run in an environment with sudo and no interactive password
6# prompt, such as within the Chromium OS development chroot.
7
8
9"""This file provides core logic for servo verify/repair process."""
10
11
12import httplib
13import logging
14import socket
Fang Deng5d518f42013-08-02 14:04:32 -070015import xmlrpclib
Raul E Rangel52ca2e82018-07-03 14:10:14 -060016import os
Fang Deng5d518f42013-08-02 14:04:32 -070017
18from autotest_lib.client.bin import utils
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -070019from autotest_lib.client.common_lib import control_data
Fang Deng5d518f42013-08-02 14:04:32 -070020from autotest_lib.client.common_lib import error
beeps5e8c45a2013-12-17 22:05:11 -080021from autotest_lib.client.common_lib import global_config
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -070022from autotest_lib.client.common_lib import host_states
Richard Barnette9a26ad62016-06-10 12:03:08 -070023from autotest_lib.client.common_lib import hosts
Dan Shi0942b1d2015-03-31 11:07:00 -070024from autotest_lib.client.common_lib import lsbrelease_utils
beeps5e8c45a2013-12-17 22:05:11 -080025from autotest_lib.client.common_lib.cros import dev_server
Fang Deng5d518f42013-08-02 14:04:32 -070026from autotest_lib.client.common_lib.cros import retry
Christopher Wileycef1f902014-06-19 11:11:23 -070027from autotest_lib.client.common_lib.cros.network import ping_runner
Hsinyu Chaoe0b08e62015-08-11 10:50:37 +000028from autotest_lib.client.cros import constants as client_constants
Richard Barnettee519dcd2016-08-15 17:37:17 -070029from autotest_lib.server import afe_utils
Prathmesh Prabhucbd5ebb2018-08-28 17:04:50 -070030from autotest_lib.server import site_utils as server_utils
Richard Barnetted31580e2018-05-14 19:58:00 +000031from autotest_lib.server.cros import autoupdater
Cheng-Yi Chiang22612862015-08-20 20:39:57 +080032from autotest_lib.server.cros import dnsname_mangler
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -070033from autotest_lib.server.cros.dynamic_suite import control_file_getter
Richard Barnetted31580e2018-05-14 19:58:00 +000034from autotest_lib.server.cros.dynamic_suite import frontend_wrappers
Richard Barnette9a26ad62016-06-10 12:03:08 -070035from autotest_lib.server.cros.servo import servo
Prathmesh Prabhuc2c6d542018-04-20 14:28:45 -070036from autotest_lib.server.hosts import base_classes
Richard Barnetted31580e2018-05-14 19:58:00 +000037from autotest_lib.server.hosts import servo_repair
Fang Deng5d518f42013-08-02 14:04:32 -070038from autotest_lib.server.hosts import ssh_host
Fang Dengd4fe7392013-09-20 12:18:21 -070039from autotest_lib.site_utils.rpm_control_system import rpm_client
Fang Deng5d518f42013-08-02 14:04:32 -070040
Dan Shi5e2efb72017-02-07 11:40:23 -080041try:
42 from chromite.lib import metrics
43except ImportError:
44 metrics = utils.metrics_mock
45
Fang Deng5d518f42013-08-02 14:04:32 -070046
Simran Basi0739d682015-02-25 16:22:56 -080047# Names of the host attributes in the database that represent the values for
48# the servo_host and servo_port for a servo connected to the DUT.
49SERVO_HOST_ATTR = 'servo_host'
50SERVO_PORT_ATTR = 'servo_port'
Richard Barnettee519dcd2016-08-15 17:37:17 -070051SERVO_BOARD_ATTR = 'servo_board'
Kevin Cheng643ce8a2016-09-15 15:42:12 -070052SERVO_SERIAL_ATTR = 'servo_serial'
Prathmesh Prabhucba44292018-08-28 17:44:45 -070053SERVO_ATTR_KEYS = (
54 SERVO_BOARD_ATTR,
55 SERVO_HOST_ATTR,
56 SERVO_PORT_ATTR,
57 SERVO_SERIAL_ATTR,
58)
Simran Basi0739d682015-02-25 16:22:56 -080059
Dan Shi3b2adf62015-09-02 17:46:54 -070060_CONFIG = global_config.global_config
xixuan6cf6d2f2016-01-29 15:29:00 -080061ENABLE_SSH_TUNNEL_FOR_SERVO = _CONFIG.get_config_value(
62 'CROS', 'enable_ssh_tunnel_for_servo', type=bool, default=False)
Simran Basi0739d682015-02-25 16:22:56 -080063
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -070064AUTOTEST_BASE = _CONFIG.get_config_value(
65 'SCHEDULER', 'drone_installation_directory',
66 default='/usr/local/autotest')
67
68_SERVO_HOST_REBOOT_TEST_NAME = 'servohost_Reboot'
Kevin Cheng55265902016-10-19 12:46:50 -070069_SERVO_HOST_FORCE_REBOOT_TEST_NAME = 'servohost_Reboot.force_reboot'
Fang Deng5d518f42013-08-02 14:04:32 -070070
Fang Deng5d518f42013-08-02 14:04:32 -070071class ServoHost(ssh_host.SSHHost):
72 """Host class for a host that controls a servo, e.g. beaglebone."""
73
Raul E Rangel52ca2e82018-07-03 14:10:14 -060074 DEFAULT_PORT = int(os.getenv('SERVOD_PORT', '9999'))
Richard Barnette9a26ad62016-06-10 12:03:08 -070075
Dan Shie5b3c512014-08-21 12:12:09 -070076 # Timeout for initializing servo signals.
Wai-Hong Tam37b6ed32017-09-19 15:52:39 -070077 INITIALIZE_SERVO_TIMEOUT_SECS = 60
Richard Barnette9a26ad62016-06-10 12:03:08 -070078
xixuan6cf6d2f2016-01-29 15:29:00 -080079 # Ready test function
80 SERVO_READY_METHOD = 'get_version'
Fang Deng5d518f42013-08-02 14:04:32 -070081
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -070082 REBOOT_CMD = 'sleep 1; reboot & sleep 10; reboot -f'
83
Fang Deng5d518f42013-08-02 14:04:32 -070084
Richard Barnette17bfc6c2016-08-04 18:41:43 -070085 def _initialize(self, servo_host='localhost',
Richard Barnettee519dcd2016-08-15 17:37:17 -070086 servo_port=DEFAULT_PORT, servo_board=None,
Kevin Cheng643ce8a2016-09-15 15:42:12 -070087 servo_serial=None, is_in_lab=None, *args, **dargs):
Fang Deng5d518f42013-08-02 14:04:32 -070088 """Initialize a ServoHost instance.
89
90 A ServoHost instance represents a host that controls a servo.
91
92 @param servo_host: Name of the host where the servod process
93 is running.
Raul E Rangel52ca2e82018-07-03 14:10:14 -060094 @param servo_port: Port the servod process is listening on. Defaults
95 to the SERVOD_PORT environment variable if set,
96 otherwise 9999.
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -070097 @param servo_board: Board that the servo is connected to.
Dan Shi4d478522014-02-14 13:46:32 -080098 @param is_in_lab: True if the servo host is in Cros Lab. Default is set
99 to None, for which utils.host_is_in_lab_zone will be
100 called to check if the servo host is in Cros lab.
Fang Deng5d518f42013-08-02 14:04:32 -0700101
102 """
103 super(ServoHost, self)._initialize(hostname=servo_host,
104 *args, **dargs)
Richard Barnette42f4db92018-08-23 15:05:15 -0700105 self.servo_port = int(servo_port)
Richard Barnettee519dcd2016-08-15 17:37:17 -0700106 self.servo_board = servo_board
Kevin Cheng643ce8a2016-09-15 15:42:12 -0700107 self.servo_serial = servo_serial
Richard Barnettee519dcd2016-08-15 17:37:17 -0700108 self._servo = None
Richard Barnette9a26ad62016-06-10 12:03:08 -0700109 self._repair_strategy = (
110 servo_repair.create_servo_repair_strategy())
Richard Barnettee519dcd2016-08-15 17:37:17 -0700111 self._is_localhost = (self.hostname == 'localhost')
112 if self._is_localhost:
113 self._is_in_lab = False
114 elif is_in_lab is None:
Dan Shi4d478522014-02-14 13:46:32 -0800115 self._is_in_lab = utils.host_is_in_lab_zone(self.hostname)
116 else:
117 self._is_in_lab = is_in_lab
xixuan6cf6d2f2016-01-29 15:29:00 -0800118
Richard Barnettee519dcd2016-08-15 17:37:17 -0700119 # Commands on the servo host must be run by the superuser.
120 # Our account on a remote host is root, but if our target is
121 # localhost then we might be running unprivileged. If so,
122 # `sudo` will have to be added to the commands.
Fang Deng5d518f42013-08-02 14:04:32 -0700123 if self._is_localhost:
124 self._sudo_required = utils.system_output('id -u') != '0'
125 else:
126 self._sudo_required = False
Richard Barnettee519dcd2016-08-15 17:37:17 -0700127
Richard Barnette9a26ad62016-06-10 12:03:08 -0700128
129 def connect_servo(self):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700130 """Establish a connection to the servod server on this host.
Richard Barnette9a26ad62016-06-10 12:03:08 -0700131
132 Initializes `self._servo` and then verifies that all network
133 connections are working. This will create an ssh tunnel if
134 it's required.
135
136 As a side effect of testing the connection, all signals on the
137 target servo are reset to default values, and the USB stick is
138 set to the neutral (off) position.
139 """
Kevin Cheng643ce8a2016-09-15 15:42:12 -0700140 servo_obj = servo.Servo(servo_host=self, servo_serial=self.servo_serial)
Richard Barnette9a26ad62016-06-10 12:03:08 -0700141 timeout, _ = retry.timeout(
142 servo_obj.initialize_dut,
143 timeout_sec=self.INITIALIZE_SERVO_TIMEOUT_SECS)
144 if timeout:
145 raise hosts.AutoservVerifyError(
146 'Servo initialize timed out.')
147 self._servo = servo_obj
148
149
150 def disconnect_servo(self):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700151 """Disconnect our servo if it exists.
Richard Barnette9a26ad62016-06-10 12:03:08 -0700152
153 If we've previously successfully connected to our servo,
154 disconnect any established ssh tunnel, and set `self._servo`
155 back to `None`.
156 """
157 if self._servo:
158 # N.B. This call is safe even without a tunnel:
159 # rpc_server_tracker.disconnect() silently ignores
160 # unknown ports.
161 self.rpc_server_tracker.disconnect(self.servo_port)
162 self._servo = None
Fang Deng5d518f42013-08-02 14:04:32 -0700163
164
165 def is_in_lab(self):
166 """Check whether the servo host is a lab device.
167
168 @returns: True if the servo host is in Cros Lab, otherwise False.
169
170 """
171 return self._is_in_lab
172
173
174 def is_localhost(self):
175 """Checks whether the servo host points to localhost.
176
177 @returns: True if it points to localhost, otherwise False.
178
179 """
180 return self._is_localhost
181
182
183 def get_servod_server_proxy(self):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700184 """Return a proxy that can be used to communicate with servod server.
Fang Deng5d518f42013-08-02 14:04:32 -0700185
186 @returns: An xmlrpclib.ServerProxy that is connected to the servod
187 server on the host.
Fang Deng5d518f42013-08-02 14:04:32 -0700188 """
Richard Barnette9a26ad62016-06-10 12:03:08 -0700189 if ENABLE_SSH_TUNNEL_FOR_SERVO and not self.is_localhost():
190 return self.rpc_server_tracker.xmlrpc_connect(
191 None, self.servo_port,
192 ready_test_name=self.SERVO_READY_METHOD,
193 timeout_seconds=60)
194 else:
195 remote = 'http://%s:%s' % (self.hostname, self.servo_port)
196 return xmlrpclib.ServerProxy(remote)
Fang Deng5d518f42013-08-02 14:04:32 -0700197
198
Richard Barnette9a26ad62016-06-10 12:03:08 -0700199 def is_cros_host(self):
beeps5e8c45a2013-12-17 22:05:11 -0800200 """Check if a servo host is running chromeos.
201
202 @return: True if the servo host is running chromeos.
203 False if it isn't, or we don't have enough information.
204 """
205 try:
206 result = self.run('grep -q CHROMEOS /etc/lsb-release',
207 ignore_status=True, timeout=10)
208 except (error.AutoservRunError, error.AutoservSSHTimeout):
209 return False
210 return result.exit_status == 0
211
212
Fang Deng5d518f42013-08-02 14:04:32 -0700213 def make_ssh_command(self, user='root', port=22, opts='', hosts_file=None,
Dean Liaoe3e75f62017-11-14 10:36:43 +0800214 connect_timeout=None, alive_interval=None,
215 alive_count_max=None, connection_attempts=None):
Fang Deng5d518f42013-08-02 14:04:32 -0700216 """Override default make_ssh_command to use tuned options.
217
218 Tuning changes:
219 - ConnectTimeout=30; maximum of 30 seconds allowed for an SSH
220 connection failure. Consistency with remote_access.py.
221
222 - ServerAliveInterval=180; which causes SSH to ping connection every
223 180 seconds. In conjunction with ServerAliveCountMax ensures
224 that if the connection dies, Autotest will bail out quickly.
225
226 - ServerAliveCountMax=3; consistency with remote_access.py.
227
228 - ConnectAttempts=4; reduce flakiness in connection errors;
229 consistency with remote_access.py.
230
231 - UserKnownHostsFile=/dev/null; we don't care about the keys.
232
233 - SSH protocol forced to 2; needed for ServerAliveInterval.
234
235 @param user User name to use for the ssh connection.
236 @param port Port on the target host to use for ssh connection.
237 @param opts Additional options to the ssh command.
238 @param hosts_file Ignored.
239 @param connect_timeout Ignored.
240 @param alive_interval Ignored.
Dean Liaoe3e75f62017-11-14 10:36:43 +0800241 @param alive_count_max Ignored.
242 @param connection_attempts Ignored.
Fang Deng5d518f42013-08-02 14:04:32 -0700243
244 @returns: An ssh command with the requested settings.
245
246 """
Dean Liaoe3e75f62017-11-14 10:36:43 +0800247 options = ' '.join([opts, '-o Protocol=2'])
248 return super(ServoHost, self).make_ssh_command(
249 user=user, port=port, opts=options, hosts_file='/dev/null',
250 connect_timeout=30, alive_interval=180, alive_count_max=3,
251 connection_attempts=4)
Fang Deng5d518f42013-08-02 14:04:32 -0700252
253
254 def _make_scp_cmd(self, sources, dest):
255 """Format scp command.
256
257 Given a list of source paths and a destination path, produces the
258 appropriate scp command for encoding it. Remote paths must be
259 pre-encoded. Overrides _make_scp_cmd in AbstractSSHHost
260 to allow additional ssh options.
261
262 @param sources: A list of source paths to copy from.
263 @param dest: Destination path to copy to.
264
265 @returns: An scp command that copies |sources| on local machine to
266 |dest| on the remote servo host.
267
268 """
269 command = ('scp -rq %s -o BatchMode=yes -o StrictHostKeyChecking=no '
270 '-o UserKnownHostsFile=/dev/null -P %d %s "%s"')
271 return command % (self.master_ssh_option,
272 self.port, ' '.join(sources), dest)
273
274
275 def run(self, command, timeout=3600, ignore_status=False,
276 stdout_tee=utils.TEE_TO_LOGS, stderr_tee=utils.TEE_TO_LOGS,
Luigi Semenzatobfbd1f32017-01-06 10:41:18 -0800277 connect_timeout=30, ssh_failure_retry_ok=False,
278 options='', stdin=None, verbose=True, args=()):
Fang Deng5d518f42013-08-02 14:04:32 -0700279 """Run a command on the servo host.
280
281 Extends method `run` in SSHHost. If the servo host is a remote device,
282 it will call `run` in SSHost without changing anything.
283 If the servo host is 'localhost', it will call utils.system_output.
284
285 @param command: The command line string.
286 @param timeout: Time limit in seconds before attempting to
287 kill the running process. The run() function
288 will take a few seconds longer than 'timeout'
289 to complete if it has to kill the process.
290 @param ignore_status: Do not raise an exception, no matter
291 what the exit code of the command is.
292 @param stdout_tee/stderr_tee: Where to tee the stdout/stderr.
293 @param connect_timeout: SSH connection timeout (in seconds)
294 Ignored if host is 'localhost'.
295 @param options: String with additional ssh command options
296 Ignored if host is 'localhost'.
Luigi Semenzatobfbd1f32017-01-06 10:41:18 -0800297 @param ssh_failure_retry_ok: when True and ssh connection failure is
298 suspected, OK to retry command (but not
299 compulsory, and likely not needed here)
Fang Deng5d518f42013-08-02 14:04:32 -0700300 @param stdin: Stdin to pass (a string) to the executed command.
301 @param verbose: Log the commands.
302 @param args: Sequence of strings to pass as arguments to command by
303 quoting them in " and escaping their contents if necessary.
304
305 @returns: A utils.CmdResult object.
306
307 @raises AutoservRunError if the command failed.
308 @raises AutoservSSHTimeout SSH connection has timed out. Only applies
309 when servo host is not 'localhost'.
310
311 """
312 run_args = {'command': command, 'timeout': timeout,
313 'ignore_status': ignore_status, 'stdout_tee': stdout_tee,
314 'stderr_tee': stderr_tee, 'stdin': stdin,
315 'verbose': verbose, 'args': args}
316 if self.is_localhost():
317 if self._sudo_required:
Michael Tangf9b3ada2016-11-18 16:01:05 -0800318 run_args['command'] = 'sudo -n sh -c "%s"' % utils.sh_escape(
319 command)
Fang Deng5d518f42013-08-02 14:04:32 -0700320 try:
321 return utils.run(**run_args)
322 except error.CmdError as e:
323 logging.error(e)
324 raise error.AutoservRunError('command execution error',
325 e.result_obj)
326 else:
327 run_args['connect_timeout'] = connect_timeout
328 run_args['options'] = options
329 return super(ServoHost, self).run(**run_args)
330
331
Richard Barnette9a26ad62016-06-10 12:03:08 -0700332 def _get_release_version(self):
Dan Shi0942b1d2015-03-31 11:07:00 -0700333 """Get the value of attribute CHROMEOS_RELEASE_VERSION from lsb-release.
334
335 @returns The version string in lsb-release, under attribute
336 CHROMEOS_RELEASE_VERSION.
337 """
338 lsb_release_content = self.run(
339 'cat "%s"' % client_constants.LSB_RELEASE).stdout.strip()
340 return lsbrelease_utils.get_chromeos_release_version(
341 lsb_release_content=lsb_release_content)
342
343
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700344 def get_attached_duts(self, afe):
345 """Gather a list of duts that use this servo host.
346
347 @param afe: afe instance.
348
349 @returns list of duts.
Richard Barnette3a7697f2016-04-20 11:33:27 -0700350 """
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700351 return afe.get_hosts_by_attribute(
352 attribute=SERVO_HOST_ATTR, value=self.hostname)
353
354
355 def get_board(self):
356 """Determine the board for this servo host.
357
358 @returns a string representing this servo host's board.
359 """
360 return lsbrelease_utils.get_current_board(
361 lsb_release_content=self.run('cat /etc/lsb-release').stdout)
362
363
364 def _choose_dut_for_synchronized_reboot(self, dut_list, afe):
365 """Choose which dut to schedule servo host reboot job.
366
367 We'll want a semi-deterministic way of selecting which host should be
368 scheduled for the servo host reboot job. For now we'll sort the
369 list with the expectation the dut list will stay consistent.
370 From there we'll grab the first dut that is available so we
371 don't schedule a job on a dut that will never run.
372
373 @param dut_list: List of the dut hostnames to choose from.
374 @param afe: Instance of the AFE.
375
376 @return hostname of dut to schedule job on.
377 """
378 afe_hosts = afe.get_hosts(dut_list)
379 afe_hosts.sort()
380 for afe_host in afe_hosts:
381 if afe_host.status not in host_states.UNAVAILABLE_STATES:
382 return afe_host.hostname
383 # If they're all unavailable, just return the first sorted dut.
384 dut_list.sort()
385 return dut_list[0]
386
387
388 def _sync_job_scheduled_for_duts(self, dut_list, afe):
389 """Checks if a synchronized reboot has been scheduled for these duts.
390
391 Grab all the host queue entries that aren't completed for the duts and
392 see if any of them have the expected job name.
393
394 @param dut_list: List of duts to check on.
395 @param afe: Instance of the AFE.
396
397 @returns True if the job is scheduled, False otherwise.
398 """
399 afe_hosts = afe.get_hosts(dut_list)
400 for afe_host in afe_hosts:
401 hqes = afe.get_host_queue_entries(host=afe_host.id, complete=0)
402 for hqe in hqes:
403 job = afe.get_jobs(id=hqe.job.id)
Kevin Cheng55265902016-10-19 12:46:50 -0700404 if job and job[0].name in (_SERVO_HOST_REBOOT_TEST_NAME,
405 _SERVO_HOST_FORCE_REBOOT_TEST_NAME):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700406 return True
407 return False
408
409
Kevin Cheng55265902016-10-19 12:46:50 -0700410 def schedule_synchronized_reboot(self, dut_list, afe, force_reboot=False):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700411 """Schedule a job to reboot the servo host.
412
413 When we schedule a job, it will create a ServoHost object which will
414 go through this entire flow of checking if a reboot is needed and
415 trying to schedule it. There is probably a better approach to setting
416 up a synchronized reboot but I'm coming up short on better ideas so I
417 apologize for this circus show.
418
Kevin Cheng55265902016-10-19 12:46:50 -0700419 @param dut_list: List of duts that need to be locked.
420 @param afe: Instance of afe.
421 @param force_reboot: Boolean to indicate if a forced reboot should be
422 scheduled or not.
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700423 """
424 # If we've already scheduled job on a dut, we're done here.
425 if self._sync_job_scheduled_for_duts(dut_list, afe):
426 return
427
428 # Looks like we haven't scheduled a job yet.
Kevin Cheng55265902016-10-19 12:46:50 -0700429 test = (_SERVO_HOST_REBOOT_TEST_NAME if not force_reboot
430 else _SERVO_HOST_FORCE_REBOOT_TEST_NAME)
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700431 dut = self._choose_dut_for_synchronized_reboot(dut_list, afe)
432 getter = control_file_getter.FileSystemGetter([AUTOTEST_BASE])
Kevin Cheng55265902016-10-19 12:46:50 -0700433 control_file = getter.get_control_file_contents_by_name(test)
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700434 control_type = control_data.CONTROL_TYPE_NAMES.SERVER
Kevin Cheng79589982016-10-25 13:26:04 -0700435 try:
436 afe.create_job(control_file=control_file, name=test,
437 control_type=control_type, hosts=[dut])
438 except Exception as e:
439 # Sometimes creating the job will raise an exception. We'll log it
440 # but we don't want to fail because of it.
Aviv Keshet5ae0a002017-05-05 10:23:33 -0700441 logging.exception('Scheduling reboot job failed due to Exception.')
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700442
443
444 def reboot(self, *args, **dargs):
445 """Reboot using special servo host reboot command."""
446 super(ServoHost, self).reboot(reboot_cmd=self.REBOOT_CMD,
447 *args, **dargs)
448
449
450 def _check_for_reboot(self, updater):
451 """Reboot this servo host if an upgrade is waiting.
Richard Barnette3a7697f2016-04-20 11:33:27 -0700452
453 If the host has successfully downloaded and finalized a new
454 build, reboot.
455
456 @param updater: a ChromiumOSUpdater instance for checking
457 whether reboot is needed.
458 @return Return a (status, build) tuple reflecting the
459 update_engine status and current build of the host
460 at the end of the call.
461 """
Richard Barnette9a26ad62016-06-10 12:03:08 -0700462 current_build_number = self._get_release_version()
Richard Barnette3a7697f2016-04-20 11:33:27 -0700463 status = updater.check_update_status()
464 if status == autoupdater.UPDATER_NEED_REBOOT:
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700465 # Check if we need to schedule an organized reboot.
Kevin Cheng79589982016-10-25 13:26:04 -0700466 afe = frontend_wrappers.RetryingAFE(
467 timeout_min=5, delay_sec=10,
Prathmesh Prabhucbd5ebb2018-08-28 17:04:50 -0700468 server=server_utils.get_global_afe_hostname())
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700469 dut_list = self.get_attached_duts(afe)
470 logging.info('servo host has the following duts: %s', dut_list)
471 if len(dut_list) > 1:
472 logging.info('servo host has multiple duts, scheduling '
473 'synchronized reboot')
474 self.schedule_synchronized_reboot(dut_list, afe)
475 return status, current_build_number
476
477 logging.info('Rebooting servo host %s from build %s',
Richard Barnette3a7697f2016-04-20 11:33:27 -0700478 self.hostname, current_build_number)
479 # Tell the reboot() call not to wait for completion.
480 # Otherwise, the call will log reboot failure if servo does
481 # not come back. The logged reboot failure will lead to
482 # test job failure. If the test does not require servo, we
483 # don't want servo failure to fail the test with error:
484 # `Host did not return from reboot` in status.log.
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700485 self.reboot(fastsync=True, wait=False)
Richard Barnette3a7697f2016-04-20 11:33:27 -0700486
487 # We told the reboot() call not to wait, but we need to wait
488 # for the reboot before we continue. Alas. The code from
489 # here below is basically a copy of Host.wait_for_restart(),
490 # with the logging bits ripped out, so that they can't cause
491 # the failure logging problem described above.
492 #
493 # The black stain that this has left on my soul can never be
494 # erased.
495 old_boot_id = self.get_boot_id()
496 if not self.wait_down(timeout=self.WAIT_DOWN_REBOOT_TIMEOUT,
497 warning_timer=self.WAIT_DOWN_REBOOT_WARNING,
498 old_boot_id=old_boot_id):
499 raise error.AutoservHostError(
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700500 'servo host %s failed to shut down.' %
501 self.hostname)
Richard Barnette3a7697f2016-04-20 11:33:27 -0700502 if self.wait_up(timeout=120):
Richard Barnette9a26ad62016-06-10 12:03:08 -0700503 current_build_number = self._get_release_version()
Richard Barnette3a7697f2016-04-20 11:33:27 -0700504 status = updater.check_update_status()
505 logging.info('servo host %s back from reboot, with build %s',
506 self.hostname, current_build_number)
507 else:
508 raise error.AutoservHostError(
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700509 'servo host %s failed to come back from reboot.' %
510 self.hostname)
Richard Barnette3a7697f2016-04-20 11:33:27 -0700511 return status, current_build_number
512
513
Richard Barnette3a7697f2016-04-20 11:33:27 -0700514 def update_image(self, wait_for_update=False):
beeps5e8c45a2013-12-17 22:05:11 -0800515 """Update the image on the servo host, if needed.
516
J. Richard Barnette84895392015-04-30 12:31:01 -0700517 This method recognizes the following cases:
518 * If the Host is not running Chrome OS, do nothing.
519 * If a previously triggered update is now complete, reboot
520 to the new version.
521 * If the host is processing a previously triggered update,
522 do nothing.
523 * If the host is running a version of Chrome OS different
524 from the default for servo Hosts, trigger an update, but
525 don't wait for it to complete.
beeps5e8c45a2013-12-17 22:05:11 -0800526
Richard Barnette3a7697f2016-04-20 11:33:27 -0700527 @param wait_for_update If an update needs to be applied and
528 this is true, then don't return until the update is
529 downloaded and finalized, and the host rebooted.
beeps5e8c45a2013-12-17 22:05:11 -0800530 @raises dev_server.DevServerException: If all the devservers are down.
531 @raises site_utils.ParseBuildNameException: If the devserver returns
532 an invalid build name.
beeps5e8c45a2013-12-17 22:05:11 -0800533 @raises AutoservRunError: If the update_engine_client isn't present on
534 the host, and the host is a cros_host.
J. Richard Barnette84895392015-04-30 12:31:01 -0700535
beeps5e8c45a2013-12-17 22:05:11 -0800536 """
Dan Shib795b5a2015-09-24 13:26:35 -0700537 # servod could be running in a Ubuntu workstation.
Richard Barnette9a26ad62016-06-10 12:03:08 -0700538 if not self.is_cros_host():
beeps5e8c45a2013-12-17 22:05:11 -0800539 logging.info('Not attempting an update, either %s is not running '
540 'chromeos or we cannot find enough information about '
541 'the host.', self.hostname)
542 return
543
Dan Shib795b5a2015-09-24 13:26:35 -0700544 if lsbrelease_utils.is_moblab():
545 logging.info('Not attempting an update, %s is running moblab.',
546 self.hostname)
547 return
548
Richard Barnette383ef9c2016-12-13 11:56:49 -0800549 target_build = afe_utils.get_stable_cros_image_name(self.get_board())
Prathmesh Prabhucbd5ebb2018-08-28 17:04:50 -0700550 target_build_number = server_utils.ParseBuildName(
J. Richard Barnette84895392015-04-30 12:31:01 -0700551 target_build)[3]
xixuanfa2d92a2016-12-09 09:45:27 -0800552 # For servo image staging, we want it as more widely distributed as
553 # possible, so that devservers' load can be evenly distributed. So use
554 # hostname instead of target_build as hash.
555 ds = dev_server.ImageServer.resolve(self.hostname,
556 hostname=self.hostname)
J. Richard Barnette84895392015-04-30 12:31:01 -0700557 url = ds.get_update_url(target_build)
beeps5e8c45a2013-12-17 22:05:11 -0800558
559 updater = autoupdater.ChromiumOSUpdater(update_url=url, host=self)
Richard Barnette3a7697f2016-04-20 11:33:27 -0700560 status, current_build_number = self._check_for_reboot(updater)
561 update_pending = True
beeps5e8c45a2013-12-17 22:05:11 -0800562 if status in autoupdater.UPDATER_PROCESSING_UPDATE:
563 logging.info('servo host %s already processing an update, update '
564 'engine client status=%s', self.hostname, status)
Allen Li66aa2542017-06-26 15:26:27 -0700565 elif status == autoupdater.UPDATER_NEED_REBOOT:
566 return
J. Richard Barnette84895392015-04-30 12:31:01 -0700567 elif current_build_number != target_build_number:
beeps5e8c45a2013-12-17 22:05:11 -0800568 logging.info('Using devserver url: %s to trigger update on '
569 'servo host %s, from %s to %s', url, self.hostname,
J. Richard Barnette84895392015-04-30 12:31:01 -0700570 current_build_number, target_build_number)
beeps5e8c45a2013-12-17 22:05:11 -0800571 try:
J. Richard Barnette84895392015-04-30 12:31:01 -0700572 ds.stage_artifacts(target_build,
573 artifacts=['full_payload'])
574 except Exception as e:
575 logging.error('Staging artifacts failed: %s', str(e))
576 logging.error('Abandoning update for this cycle.')
beeps5e8c45a2013-12-17 22:05:11 -0800577 else:
J. Richard Barnette84895392015-04-30 12:31:01 -0700578 try:
579 updater.trigger_update()
580 except autoupdater.RootFSUpdateError as e:
581 trigger_download_status = 'failed with %s' % str(e)
Aviv Keshet11836322016-11-22 11:32:01 -0800582 metrics.Counter('chromeos/autotest/servo/'
583 'rootfs_update_failed').increment()
J. Richard Barnette84895392015-04-30 12:31:01 -0700584 else:
585 trigger_download_status = 'passed'
586 logging.info('Triggered download and update %s for %s, '
587 'update engine currently in status %s',
588 trigger_download_status, self.hostname,
589 updater.check_update_status())
beeps5e8c45a2013-12-17 22:05:11 -0800590 else:
591 logging.info('servo host %s does not require an update.',
592 self.hostname)
Richard Barnette3a7697f2016-04-20 11:33:27 -0700593 update_pending = False
594
595 if update_pending and wait_for_update:
596 logging.info('Waiting for servo update to complete.')
597 self.run('update_engine_client --follow', ignore_status=True)
beeps5e8c45a2013-12-17 22:05:11 -0800598
599
Richard Barnette1edbb162016-11-01 11:47:50 -0700600 def verify(self, silent=False):
601 """Update the servo host and verify it's in a good state.
602
603 @param silent If true, suppress logging in `status.log`.
604 """
Richard Barnetteabbdc252018-07-26 16:57:42 -0700605 message = 'Beginning verify for servo host %s port %s serial %s'
606 message %= (self.hostname, self.servo_port, self.servo_serial)
607 self.record('INFO', None, None, message)
Richard Barnette9a26ad62016-06-10 12:03:08 -0700608 try:
Richard Barnette1edbb162016-11-01 11:47:50 -0700609 self._repair_strategy.verify(self, silent)
Richard Barnette9a26ad62016-06-10 12:03:08 -0700610 except:
611 self.disconnect_servo()
612 raise
Fang Deng5d518f42013-08-02 14:04:32 -0700613
614
Richard Barnette1edbb162016-11-01 11:47:50 -0700615 def repair(self, silent=False):
616 """Attempt to repair servo host.
617
618 @param silent If true, suppress logging in `status.log`.
619 """
Richard Barnetteabbdc252018-07-26 16:57:42 -0700620 message = 'Beginning repair for servo host %s port %s serial %s'
621 message %= (self.hostname, self.servo_port, self.servo_serial)
622 self.record('INFO', None, None, message)
Richard Barnette9a26ad62016-06-10 12:03:08 -0700623 try:
Richard Barnette1edbb162016-11-01 11:47:50 -0700624 self._repair_strategy.repair(self, silent)
Richard Barnette9a26ad62016-06-10 12:03:08 -0700625 except:
626 self.disconnect_servo()
627 raise
Fang Deng5d518f42013-08-02 14:04:32 -0700628
629
Fang Dengd4fe7392013-09-20 12:18:21 -0700630 def has_power(self):
631 """Return whether or not the servo host is powered by PoE."""
632 # TODO(fdeng): See crbug.com/302791
633 # For now, assume all servo hosts in the lab have power.
634 return self.is_in_lab()
635
636
637 def power_cycle(self):
638 """Cycle power to this host via PoE if it is a lab device.
639
Richard Barnette9a26ad62016-06-10 12:03:08 -0700640 @raises AutoservRepairError if it fails to power cycle the
Fang Dengd4fe7392013-09-20 12:18:21 -0700641 servo host.
642
643 """
644 if self.has_power():
645 try:
646 rpm_client.set_power(self.hostname, 'CYCLE')
647 except (socket.error, xmlrpclib.Error,
648 httplib.BadStatusLine,
649 rpm_client.RemotePowerException) as e:
Richard Barnette9a26ad62016-06-10 12:03:08 -0700650 raise hosts.AutoservRepairError(
Fang Dengd4fe7392013-09-20 12:18:21 -0700651 'Power cycling %s failed: %s' % (self.hostname, e))
652 else:
653 logging.info('Skipping power cycling, not a lab device.')
654
655
Dan Shi4d478522014-02-14 13:46:32 -0800656 def get_servo(self):
657 """Get the cached servo.Servo object.
Fang Deng5d518f42013-08-02 14:04:32 -0700658
Dan Shi4d478522014-02-14 13:46:32 -0800659 @return: a servo.Servo object.
Fang Deng5d518f42013-08-02 14:04:32 -0700660 """
Dan Shi4d478522014-02-14 13:46:32 -0800661 return self._servo
662
663
Congbin Guoa1f9cba2018-07-03 11:36:59 -0700664 def close(self):
665 """Stop UART logging and close the host object."""
666 if self._servo:
Congbin Guo2e5e2a22018-07-27 10:32:48 -0700667 # In some cases when we run as lab-tools, the job object is None.
668 if self.job:
669 self._servo.dump_uart_streams(self.job.resultdir)
Congbin Guoa1f9cba2018-07-03 11:36:59 -0700670 self._servo.close()
671
672 super(ServoHost, self).close()
673
674
Richard Barnetteea3e4602016-06-10 12:36:41 -0700675def make_servo_hostname(dut_hostname):
676 """Given a DUT's hostname, return the hostname of its servo.
677
678 @param dut_hostname: hostname of a DUT.
679
680 @return hostname of the DUT's servo.
681
682 """
683 host_parts = dut_hostname.split('.')
684 host_parts[0] = host_parts[0] + '-servo'
685 return '.'.join(host_parts)
686
687
688def servo_host_is_up(servo_hostname):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700689 """Given a servo host name, return if it's up or not.
Richard Barnetteea3e4602016-06-10 12:36:41 -0700690
691 @param servo_hostname: hostname of the servo host.
692
693 @return True if it's up, False otherwise
694 """
695 # Technically, this duplicates the SSH ping done early in the servo
696 # proxy initialization code. However, this ping ends in a couple
697 # seconds when if fails, rather than the 60 seconds it takes to decide
698 # that an SSH ping has timed out. Specifically, that timeout happens
699 # when our servo DNS name resolves, but there is no host at that IP.
700 logging.info('Pinging servo host at %s', servo_hostname)
701 ping_config = ping_runner.PingConfig(
702 servo_hostname, count=3,
703 ignore_result=True, ignore_status=True)
704 return ping_runner.PingRunner().ping(ping_config).received > 0
705
706
Richard Barnettee519dcd2016-08-15 17:37:17 -0700707def _map_afe_board_to_servo_board(afe_board):
708 """Map a board we get from the AFE to a servo appropriate value.
709
710 Many boards are identical to other boards for servo's purposes.
711 This function makes that mapping.
712
713 @param afe_board string board name received from AFE.
714 @return board we expect servo to have.
715
716 """
717 KNOWN_SUFFIXES = ['-freon', '_freon', '_moblab', '-cheets']
718 BOARD_MAP = {'gizmo': 'panther'}
719 mapped_board = afe_board
720 if afe_board in BOARD_MAP:
721 mapped_board = BOARD_MAP[afe_board]
722 else:
723 for suffix in KNOWN_SUFFIXES:
724 if afe_board.endswith(suffix):
725 mapped_board = afe_board[0:-len(suffix)]
726 break
727 if mapped_board != afe_board:
728 logging.info('Mapping AFE board=%s to %s', afe_board, mapped_board)
729 return mapped_board
730
731
Prathmesh Prabhub4810232018-09-07 13:24:08 -0700732def get_servo_args_for_host(dut_host):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700733 """Return servo data associated with a given DUT.
Richard Barnetteea3e4602016-06-10 12:36:41 -0700734
Richard Barnetteea3e4602016-06-10 12:36:41 -0700735 @param dut_host Instance of `Host` on which to find the servo
736 attributes.
Prathmesh Prabhuf605dd32018-08-28 17:09:04 -0700737 @return `servo_args` dict with host and an optional port.
Richard Barnetteea3e4602016-06-10 12:36:41 -0700738 """
Prathmesh Prabhucba44292018-08-28 17:44:45 -0700739 info = dut_host.host_info_store.get()
740 servo_args = {k: v for k, v in info.attributes.iteritems()
741 if k in SERVO_ATTR_KEYS}
Richard Barnetteea3e4602016-06-10 12:36:41 -0700742
743 # TODO(jrbarnette): This test to use the default lab servo hostname
744 # is a legacy that we need only until every host in the DB has
745 # proper attributes.
Prathmesh Prabhucba44292018-08-28 17:44:45 -0700746 if (SERVO_HOST_ATTR not in servo_args
747 and not (utils.in_moblab_ssp() or lsbrelease_utils.is_moblab())):
Richard Barnetteea3e4602016-06-10 12:36:41 -0700748 servo_host = make_servo_hostname(dut_host.hostname)
Prathmesh Prabhu7ddf6002018-09-04 15:04:29 -0700749 if utils.host_is_in_lab_zone(servo_host):
Prathmesh Prabhucba44292018-08-28 17:44:45 -0700750 servo_args[SERVO_HOST_ATTR] = servo_host
Prathmesh Prabhucbd5ebb2018-08-28 17:04:50 -0700751
Prathmesh Prabhucba44292018-08-28 17:44:45 -0700752 if SERVO_PORT_ATTR in servo_args:
753 try:
754 servo_args[SERVO_PORT_ATTR] = int(servo_args[SERVO_PORT_ATTR])
755 except ValueError:
756 logging.error('servo port is not an int: %s',
757 servo_args[SERVO_PORT_ATTR])
758 # Reset servo_args because we don't want to use an invalid port.
759 servo_args.pop(SERVO_HOST_ATTR, None)
760
761 if info.board:
762 servo_args[SERVO_BOARD_ATTR] = _map_afe_board_to_servo_board(info.board)
Prathmesh Prabhu6f5f6362018-09-05 17:20:31 -0700763 return servo_args if SERVO_HOST_ATTR in servo_args else None
Richard Barnetteea3e4602016-06-10 12:36:41 -0700764
765
Prathmesh Prabhuefb1b482018-08-28 17:15:05 -0700766def _tweak_args_for_ssp_moblab(servo_args):
767 if servo_args[SERVO_HOST_ATTR] in ['localhost', '127.0.0.1']:
768 servo_args[SERVO_HOST_ATTR] = _CONFIG.get_config_value(
769 'SSP', 'host_container_ip', type=str, default=None)
770
771
Dan Shi023aae32016-05-25 11:13:01 -0700772def create_servo_host(dut, servo_args, try_lab_servo=False,
Richard Barnette9a26ad62016-06-10 12:03:08 -0700773 try_servo_repair=False):
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700774 """Create a ServoHost object for a given DUT, if appropriate.
Dan Shi4d478522014-02-14 13:46:32 -0800775
Richard Barnette9a26ad62016-06-10 12:03:08 -0700776 This function attempts to create and verify or repair a `ServoHost`
777 object for a servo connected to the given `dut`, subject to various
778 constraints imposed by the parameters:
779 * When the `servo_args` parameter is not `None`, a servo
780 host must be created, and must be checked with `repair()`.
781 * Otherwise, if a servo exists in the lab and `try_lab_servo` is
782 true:
783 * If `try_servo_repair` is true, then create a servo host and
784 check it with `repair()`.
785 * Otherwise, if the servo responds to `ping` then create a
786 servo host and check it with `verify()`.
Fang Denge545abb2014-12-30 18:43:47 -0800787
Richard Barnette9a26ad62016-06-10 12:03:08 -0700788 In cases where `servo_args` was not `None`, repair failure
789 exceptions are passed back to the caller; otherwise, exceptions
Richard Barnette07c2e1d2016-10-26 14:24:28 -0700790 are logged and then discarded. Note that this only happens in cases
791 where we're called from a test (not special task) control file that
792 has an explicit dependency on servo. In that case, we require that
793 repair not write to `status.log`, so as to avoid polluting test
794 results.
795
796 TODO(jrbarnette): The special handling for servo in test control
797 files is a thorn in my flesh; I dearly hope to see it cut out before
798 my retirement.
Richard Barnette9a26ad62016-06-10 12:03:08 -0700799
800 Parameters for a servo host consist of a host name, port number, and
801 DUT board, and are determined from one of these sources, in order of
802 priority:
Richard Barnetteea3e4602016-06-10 12:36:41 -0700803 * Servo attributes from the `dut` parameter take precedence over
804 all other sources of information.
805 * If a DNS entry for the servo based on the DUT hostname exists in
806 the CrOS lab network, that hostname is used with the default
Richard Barnette9a26ad62016-06-10 12:03:08 -0700807 port and the DUT's board.
Richard Barnetteea3e4602016-06-10 12:36:41 -0700808 * If no other options are found, the parameters will be taken
Richard Barnette9a26ad62016-06-10 12:03:08 -0700809 from the `servo_args` dict passed in from the caller.
Richard Barnetteea3e4602016-06-10 12:36:41 -0700810
811 @param dut An instance of `Host` from which to take
812 servo parameters (if available).
813 @param servo_args A dictionary with servo parameters to use if
814 they can't be found from `dut`. If this
815 argument is supplied, unrepaired exceptions
816 from `verify()` will be passed back to the
817 caller.
818 @param try_lab_servo If not true, servo host creation will be
819 skipped unless otherwise required by the
820 caller.
Richard Barnette9a26ad62016-06-10 12:03:08 -0700821 @param try_servo_repair If true, check a servo host with
822 `repair()` instead of `verify()`.
Dan Shi4d478522014-02-14 13:46:32 -0800823
824 @returns: A ServoHost object or None. See comments above.
825
826 """
Richard Barnette07c2e1d2016-10-26 14:24:28 -0700827 servo_dependency = servo_args is not None
Richard Barnette07c2e1d2016-10-26 14:24:28 -0700828 if dut is not None and (try_lab_servo or servo_dependency):
Prathmesh Prabhub4810232018-09-07 13:24:08 -0700829 servo_args_override = get_servo_args_for_host(dut)
Richard Barnetteea3e4602016-06-10 12:36:41 -0700830 if servo_args_override is not None:
Prathmesh Prabhuefb1b482018-08-28 17:15:05 -0700831 if utils.in_moblab_ssp():
832 _tweak_args_for_ssp_moblab(servo_args_override)
Prathmesh Prabhu88bf6052018-08-28 16:21:26 -0700833 logging.debug(
834 'Overriding provided servo_args (%s) with arguments'
835 ' determined from the host (%s)',
836 servo_args,
837 servo_args_override,
838 )
Richard Barnetteea3e4602016-06-10 12:36:41 -0700839 servo_args = servo_args_override
Prathmesh Prabhucba44292018-08-28 17:44:45 -0700840
Richard Barnetteea3e4602016-06-10 12:36:41 -0700841 if servo_args is None:
Prathmesh Prabhu88bf6052018-08-28 16:21:26 -0700842 logging.debug('No servo_args provided, and failed to find overrides.')
Richard Barnetteea3e4602016-06-10 12:36:41 -0700843 return None
Prathmesh Prabhucba44292018-08-28 17:44:45 -0700844 if SERVO_HOST_ATTR not in servo_args:
845 logging.debug('%s attribute missing from servo_args: %s',
846 SERVO_HOST_ATTR, servo_args)
847 return None
Richard Barnette07c2e1d2016-10-26 14:24:28 -0700848 if (not servo_dependency and not try_servo_repair and
Richard Barnette9a26ad62016-06-10 12:03:08 -0700849 not servo_host_is_up(servo_args[SERVO_HOST_ATTR])):
Prathmesh Prabhu88bf6052018-08-28 16:21:26 -0700850 logging.debug('ServoHost is not up.')
Dan Shibbb0cb62014-03-24 17:50:57 -0700851 return None
Prathmesh Prabhu88bf6052018-08-28 16:21:26 -0700852
Prathmesh Prabhuf605dd32018-08-28 17:09:04 -0700853 newhost = ServoHost(
854 is_in_lab=(servo_args
855 and server_utils.host_in_lab(
856 servo_args[SERVO_HOST_ATTR])),
857 **servo_args
858 )
Prathmesh Prabhuc2c6d542018-04-20 14:28:45 -0700859 base_classes.send_creation_metric(newhost)
Prathmesh Prabhu88bf6052018-08-28 16:21:26 -0700860
Richard Barnette9a26ad62016-06-10 12:03:08 -0700861 # Note that the logic of repair() includes everything done
862 # by verify(). It's sufficient to call one or the other;
863 # we don't need both.
Richard Barnette07c2e1d2016-10-26 14:24:28 -0700864 if servo_dependency:
865 newhost.repair(silent=True)
Prathmesh Prabhu88bf6052018-08-28 16:21:26 -0700866 return newhost
867
868 if try_servo_repair:
869 try:
870 newhost.repair()
871 except Exception:
872 logging.exception('servo repair failed for %s', newhost.hostname)
Richard Barnette9a26ad62016-06-10 12:03:08 -0700873 else:
874 try:
Prathmesh Prabhu88bf6052018-08-28 16:21:26 -0700875 newhost.verify()
Kevin Cheng5f2ba6c2016-09-28 10:20:05 -0700876 except Exception:
Prathmesh Prabhu88bf6052018-08-28 16:21:26 -0700877 logging.exception('servo verify failed for %s', newhost.hostname)
Richard Barnette9a26ad62016-06-10 12:03:08 -0700878 return newhost