Exponentially backoff and sleep between retries to Google Storage.
Occassionally google storage can be flaky. When it is, it is often
for a small amount of time. Often retrying during these occurences
without a sleep will not result in success. This changes adds
exponential backoff to our gsutil attempts to the downloader.
BUG=chromium-os:31229
TEST=unittests
Change-Id: Ie27ec4ec171b2956a24cfb1281afced697afce17
Reviewed-on: https://gerrit.chromium.org/gerrit/25143
Reviewed-by: Chris Sosa <sosa@chromium.org>
Tested-by: Chris Sosa <sosa@chromium.org>
Commit-Ready: Chris Sosa <sosa@chromium.org>
diff --git a/gsutil_util.py b/gsutil_util.py
index 5567a60..4342bf2 100644
--- a/gsutil_util.py
+++ b/gsutil_util.py
@@ -5,6 +5,7 @@
"""Module containing gsutil helper methods."""
import subprocess
+import time
GSUTIL_ATTEMPTS = 5
@@ -17,12 +18,15 @@
def GSUtilRun(cmd, err_msg):
"""Runs a GSUTIL command up to GSUTIL_ATTEMPTS number of times.
+ Attempts are tried with exponential backoff.
+
Returns:
stdout of the called gsutil command.
Raises:
subprocess.CalledProcessError if all attempt to run gsutil cmd fails.
"""
proc = None
+ sleep_timeout = 1
for _attempt in range(GSUTIL_ATTEMPTS):
# Note processes can hang when capturing from stderr. This command
# specifically doesn't pipe stderr.
@@ -30,6 +34,10 @@
stdout, _stderr = proc.communicate()
if proc.returncode == 0:
return stdout
+
+ time.sleep(sleep_timeout)
+ sleep_timeout *= 2
+
else:
raise GSUtilError('%s GSUTIL cmd %s failed with return code %d' % (
err_msg, cmd, proc.returncode))