commit cdd27b29f1eebbb834a78140924cb0c2c0f73bca
author Garry Wang <xianuowang@chromium.org> Mon Jan 13 14:59:11 2020 -0800
committer Commit Bot <commit-bot@chromium.org> Wed Jan 15 04:58:28 2020 +0000
tree ce0f74b4e77aa0f8951da1757d6cd7f3595a17ab
parent 4192ff41eb4922997c732d362695c2536dd397ec

autotest: catch and forgive exception if restart servod failed outside of repair

BUG=chromium:1041167
TEST=run servo repair locally.

Change-Id: I1fa778111e84f071590ccd143560964c4fba0941
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/autotest/+/1999465
Commit-Queue: Garry Wang <xianuowang@chromium.org>
Tested-by: Garry Wang <xianuowang@chromium.org>
Reviewed-by: Allen Li <ayatane@chromium.org>

server/hosts/servo_host.py

diff --git a/server/hosts/servo_host.py b/server/hosts/servo_host.py
index 0efbee3..3ba5f94 100644
--- a/server/hosts/servo_host.py
+++ b/server/hosts/servo_host.py
@@ -267,7 +267,7 @@
         cmd += ' PORT=%d' % self.servo_port
         if self.servo_serial:
             cmd += ' SERIAL=%s' % self.servo_serial
-        self.run(cmd)
+        self.run(cmd, timeout=60)
 
         # There's a lag between when `start servod` completes and when
         # the _ServodConnectionVerifier trigger can actually succeed.
@@ -296,7 +296,8 @@
             return
 
         logging.debug('Stopping servod on port %s', self.servo_port)
-        self.run('stop servod PORT=%d' % self.servo_port, ignore_status=True)
+        self.run('stop servod PORT=%d' % self.servo_port,
+                 timeout=60, ignore_status=True)
         logging.debug('Wait %s seconds for servod process fully teardown.',
                       SERVOD_TEARDOWN_TIMEOUT)
         time.sleep(SERVOD_TEARDOWN_TIMEOUT)
@@ -523,7 +524,16 @@
         return None
 
     newhost = ServoHost(**servo_args)
-    newhost.restart_servod(quick_startup=True)
+    try:
+        newhost.restart_servod(quick_startup=True)
+    except error.AutoservSSHTimeout:
+        logging.warning("Restart servod failed due ssh connection "
+                        "to servohost timed out. This error is forgiven"
+                        " here, we will retry in servo repair process.")
+    except error.AutoservRunError as e:
+        logging.warning("Restart servod failed due to:\n%s\n"
+                        "This error is forgiven here, we will retry"
+                        " in servo repair process.", str(e))
 
     # TODO(gregorynisbet): Clean all of this up.
     logging.debug('create_servo_host: attempt to set info store on '