devserver: stat explicit GS artifact files via getacl
With this change, when an artifact file name does not contain glob
wildcards, we will first attempt to stat its existence via gsutil
getacl. Uses:
* It allows us to stage files that do not appear in the manifest
(UPLOADED file) on the GS directory, such as autotest tarballs that
are being uploaded after the fact by paygen (see associated bug).
Note that getacl is successful iff the file name maps to a single file
that is present in the archive directory.
* For all other cases, it will just fall back to the existing methods,
namely reading the manifest file, or resorting to gsutil ls.
BUG=chromium:275750
TEST=Unit tests
Change-Id: Id4d5236b4a781db23d3d78ebf492387ebb74bfb8
Reviewed-on: https://chromium-review.googlesource.com/167435
Tested-by: Gilad Arnold <garnold@chromium.org>
Reviewed-by: Chris Sosa <sosa@chromium.org>
Commit-Queue: Gilad Arnold <garnold@chromium.org>
diff --git a/gsutil_util.py b/gsutil_util.py
index 3cc884a..78806aa 100644
--- a/gsutil_util.py
+++ b/gsutil_util.py
@@ -89,7 +89,7 @@
def _GlobHasWildcards(pattern):
"""Returns True if a glob pattern contains any wildcards."""
- return len(pattern) > len(pattern.translate(None, '*.[]'))
+ return len(pattern) > len(pattern.translate(None, '*?[]'))
def GetGSNamesWithWait(pattern, archive_url, err_str, timeout=600, delay=10,
@@ -117,25 +117,41 @@
"""
# Define the different methods used for obtaining the list of files on the
- # archive directory, in the order in which they are attempted.
+ # archive directory, in the order in which they are attempted. Each method is
+ # defined by a tuple consisting of (i) the gsutil command-line to be
+ # executed; (ii) the error message to use in case of a failure (returned in
+ # the corresponding exception); (iii) the desired return value to use in case
+ # of success, or None if the actual command output should be used.
get_methods = []
+ # If the pattern is a glob and contains no wildcards, we'll first attempt to
+ # stat the file via getacl.
+ if not (is_regex_pattern or _GlobHasWildcards(pattern)):
+ get_methods.append(('gsutil getacl %s/%s' % (archive_url, pattern),
+ 'Failed to getacl on the artifact file.',
+ pattern))
+
# The default method is to check the manifest file in the archive directory.
get_methods.append(('gsutil cat %s/%s' % (archive_url, UPLOADED_LIST),
- 'Failed to get a list of uploaded files.'))
+ 'Failed to get a list of uploaded files.',
+ None))
# For backward compatibility, we fall back to using "gsutil ls" when the
# manifest file is not present.
get_methods.append(('gsutil ls %s/*' % archive_url,
- 'Failed to list archive directory contents.'))
+ 'Failed to list archive directory contents.',
+ None))
deadline = time.time() + timeout
while True:
uploaded_list = []
- for cmd, msg in get_methods:
+ for cmd, msg, override_result in get_methods:
try:
result = GSUtilRun(cmd, msg)
except GSUtilError:
continue # It didn't work, try the next method.
+ if override_result:
+ result = override_result
+
# Make sure we're dealing with artifact base names only.
uploaded_list = [os.path.basename(p) for p in result.splitlines()]
break