Make gsutil_util more friendly to interactive scripts.
gsutil_util was written to be extremely protected against Google
Storage hiccups that can occur when it is continuously running.
However, for interactive clients, its over-retrying will make it hard
to use interactively when a file really doesn't exist.
This change makes it so you can have both by checking the stderr
of gsutil to check for error msgs. If matched no objects or
non-existent object are returned, it's not gs flake.
BUG=chromium-os:37575
TEST=Ran it with autotest change.
Change-Id: I4db15f4ebbb426b2ec1f09e473de914c495a11f5
Reviewed-on: https://gerrit.chromium.org/gerrit/44247
Tested-by: Chris Sosa <sosa@chromium.org>
Reviewed-by: Gilad Arnold <garnold@chromium.org>
Commit-Queue: Chris Sosa <sosa@chromium.org>
diff --git a/gsutil_util.py b/gsutil_util.py
index d827f20..a264cf7 100644
--- a/gsutil_util.py
+++ b/gsutil_util.py
@@ -43,20 +43,24 @@
"""
proc = None
sleep_timeout = 1
+ stderr = None
for _attempt in range(GSUTIL_ATTEMPTS):
- # Note processes can hang when capturing from stderr. This command
- # specifically doesn't pipe stderr.
- proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
- stdout, _stderr = proc.communicate()
+ proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ stdout, stderr = proc.communicate()
if proc.returncode == 0:
return stdout
+ elif 'matched no objects' in stderr or 'non-existent object' in stderr:
+ # TODO(sosa): Note this is a heuristic that makes us not re-attempt
+ # unnecessarily. However, if it fails, the worst that can happen is just
+ # waiting longer than necessary.
+ break
time.sleep(sleep_timeout)
sleep_timeout *= 2
- else:
- raise GSUtilError('%s GSUTIL cmd %s failed with return code %d' % (
- err_msg, cmd, proc.returncode))
+ raise GSUtilError('%s GSUTIL cmd %s failed with return code %d:\n\n%s' % (
+ err_msg, cmd, proc.returncode, stderr))
def DownloadFromGS(src, dst):