commit f0146256f74e582aa794b8f211084c40c43038a5
author Mike Frysinger <vapier@chromium.org> Mon Sep 02 13:31:05 2019 -0400
committer Mike Frysinger <vapier@chromium.org> Mon Sep 02 23:29:56 2019 +0000
tree 313c369f82d36cc1d40e1b785930550699e380d7
parent d3e9d0596b1e20b30f1ee45133d4853d94346abc

cbuildbot_launch: add debugging logic to cleanup timeout

The chroot cleanup is timing out on bots.  Add some debugging
logic for when this happens to try & figure out what's going on.

BUG=chromium:1000034
TEST=precq passes

Change-Id: I999c166da36f1f1075bab226661bb2f22bb9990f
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/chromite/+/1780892
Tested-by: Mike Frysinger <vapier@chromium.org>
Reviewed-by: Mike Frysinger <vapier@chromium.org>

scripts/cbuildbot_launch.py

diff --git a/scripts/cbuildbot_launch.py b/scripts/cbuildbot_launch.py
index 16c4544..06b14ec 100644
--- a/scripts/cbuildbot_launch.py
+++ b/scripts/cbuildbot_launch.py
@@ -32,6 +32,7 @@
 from chromite.lib import cros_sdk_lib
 from chromite.lib import metrics
 from chromite.lib import osutils
+from chromite.lib import timeout_util
 from chromite.lib import ts_mon_config
 from chromite.scripts import cbuildbot
 
@@ -424,7 +425,17 @@
   chroot_dir = os.path.join(buildroot, constants.DEFAULT_CHROOT_DIR)
   logging.info('Cleaning up chroot at %s', chroot_dir)
   if os.path.exists(chroot_dir) or os.path.exists(chroot_dir + '.img'):
-    cros_sdk_lib.CleanupChrootMount(chroot_dir, delete=False)
+    try:
+      cros_sdk_lib.CleanupChrootMount(chroot_dir, delete=False)
+    except timeout_util.TimeoutError:
+      logging.exception('Cleaning up chroot timed out')
+      # Dump debug info to help https://crbug.com/1000034.
+      cros_build_lib.RunCommand(['mount'], error_code_ok=False)
+      cros_build_lib.SudoRunCommand(['losetup', '-a'], error_code_ok=False)
+      cros_build_lib.RunCommand(['dmesg'], error_code_ok=False)
+      return False
+
+  return True
 
 
 def ConfigureGlobalEnvironment():
@@ -509,7 +520,8 @@
       SetLastBuildState(root, build_state)
 
       with metrics.SecondsTimer(METRIC_CHROOT_CLEANUP):
-        CleanupChroot(buildroot)
+        if not CleanupChroot(buildroot):
+          result = 1
 
       return result