blob: 353c2f7a06c97437101e2ac89afc9a88706b9fc3 [file] [log] [blame]
Chris Sosa47a7d4e2012-03-28 11:26:55 -07001# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Module containing gsutil helper methods."""
6
joychenf8f07e22013-07-12 17:45:51 -07007import distutils.version
Gilad Arnold950569b2013-08-27 14:38:01 -07008import fnmatch
9import os
Chris Sosa76e44b92013-01-31 12:11:38 -080010import random
11import re
Chris Sosa47a7d4e2012-03-28 11:26:55 -070012import subprocess
Chris Sosa101fd862012-06-12 17:44:53 -070013import time
Chris Sosa47a7d4e2012-03-28 11:26:55 -070014
joychenf8f07e22013-07-12 17:45:51 -070015import devserver_constants
Chris Sosa76e44b92013-01-31 12:11:38 -080016import log_util
17
Gilad Arnoldabb352e2012-09-23 01:24:27 -070018
Chris Sosa7cd23202013-10-15 17:22:57 -070019GSUTIL_ATTEMPTS = 1
Chris Sosa76e44b92013-01-31 12:11:38 -080020UPLOADED_LIST = 'UPLOADED'
21
22
23# Module-local log function.
24def _Log(message, *args):
25 return log_util.LogWithTag('GSUTIL_UTIL', message, *args)
Chris Sosa47a7d4e2012-03-28 11:26:55 -070026
27
28class GSUtilError(Exception):
Chris Sosa76e44b92013-01-31 12:11:38 -080029 """Exception raised when we run into an error running gsutil."""
30 pass
31
32
33class PatternNotSpecific(Exception):
34 """Raised when unexpectedly more than one item is returned for a pattern."""
Chris Sosa47a7d4e2012-03-28 11:26:55 -070035 pass
36
37
38def GSUtilRun(cmd, err_msg):
39 """Runs a GSUTIL command up to GSUTIL_ATTEMPTS number of times.
40
Chris Sosa101fd862012-06-12 17:44:53 -070041 Attempts are tried with exponential backoff.
42
Gilad Arnold950569b2013-08-27 14:38:01 -070043 Args:
44 cmd: a string containing the gsutil command to run.
45 err_msg: string prepended to the exception thrown in case of a failure.
Chris Sosa47a7d4e2012-03-28 11:26:55 -070046 Returns:
47 stdout of the called gsutil command.
48 Raises:
Gilad Arnold950569b2013-08-27 14:38:01 -070049 GSUtilError: if all attempts to run gsutil have failed.
Chris Sosa47a7d4e2012-03-28 11:26:55 -070050 """
51 proc = None
Chris Sosa101fd862012-06-12 17:44:53 -070052 sleep_timeout = 1
Chris Sosa407e8c52013-02-27 15:33:35 -080053 stderr = None
Gilad Arnold950569b2013-08-27 14:38:01 -070054 for _ in range(GSUTIL_ATTEMPTS):
Chris Sosa407e8c52013-02-27 15:33:35 -080055 proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
56 stderr=subprocess.PIPE)
57 stdout, stderr = proc.communicate()
Ben Zhang1723fe42016-08-23 16:32:51 -070058 _Log('crbug.com/639314 GSUtilRun({0}, {1})={2}, stdout=<<<{3}>>>, stderr=<<<{4}>>>'.format(
59 cmd, err_msg, proc.returncode, stdout, stderr))
Chris Sosa47a7d4e2012-03-28 11:26:55 -070060 if proc.returncode == 0:
61 return stdout
Yu-Ju Hongc8974c02014-11-18 11:40:30 -080062
63 not_exist_messages = ('matched no objects', 'non-existent object',
64 'no urls matched')
65 if (stderr and any(x in stderr.lower() for x in not_exist_messages) or
66 stdout and any(x in stdout.lower() for x in not_exist_messages)):
67 # If the object does not exist, exit now instead of wasting time
68 # on retrying. Note that `gsutil stat` prints error message to
69 # stdout instead (b/16020252), so we check both stdout and
70 # stderr.
Chris Sosa407e8c52013-02-27 15:33:35 -080071 break
Yu-Ju Hongc8974c02014-11-18 11:40:30 -080072
73 if proc.returncode == 127:
Chris Sosa6de29302013-03-14 15:27:36 -070074 raise GSUtilError('gsutil tool not found in your path.')
Chris Sosa101fd862012-06-12 17:44:53 -070075
76 time.sleep(sleep_timeout)
77 sleep_timeout *= 2
78
Chris Sosa407e8c52013-02-27 15:33:35 -080079 raise GSUtilError('%s GSUTIL cmd %s failed with return code %d:\n\n%s' % (
80 err_msg, cmd, proc.returncode, stderr))
Chris Sosa47a7d4e2012-03-28 11:26:55 -070081
82
83def DownloadFromGS(src, dst):
84 """Downloads object from gs_url |src| to |dst|.
85
Gilad Arnold950569b2013-08-27 14:38:01 -070086 Args:
87 src: source file on GS that needs to be downloaded.
88 dst: file to copy the source file to.
Chris Sosa47a7d4e2012-03-28 11:26:55 -070089 Raises:
90 GSUtilError: if an error occurs during the download.
91 """
92 cmd = 'gsutil cp %s %s' % (src, dst)
93 msg = 'Failed to download "%s".' % src
94 GSUtilRun(cmd, msg)
Chris Sosa76e44b92013-01-31 12:11:38 -080095
96
Gilad Arnold950569b2013-08-27 14:38:01 -070097def _GlobHasWildcards(pattern):
98 """Returns True if a glob pattern contains any wildcards."""
Gilad Arnoldc703c8c2013-08-29 10:41:27 -070099 return len(pattern) > len(pattern.translate(None, '*?[]'))
Chris Sosa76e44b92013-01-31 12:11:38 -0800100
101
Gilad Arnold950569b2013-08-27 14:38:01 -0700102def GetGSNamesWithWait(pattern, archive_url, err_str, timeout=600, delay=10,
103 is_regex_pattern=False):
Chris Sosa76e44b92013-01-31 12:11:38 -0800104 """Returns the google storage names specified by the given pattern.
105
106 This method polls Google Storage until the target artifacts specified by the
107 pattern is available or until the timeout occurs. Because we may not know the
108 exact name of the target artifacts, the method accepts a filename pattern,
109 to identify whether an artifact whose name matches the pattern exists (e.g.
Gilad Arnold950569b2013-08-27 14:38:01 -0700110 use pattern '*_full_*' to search for the full payload
Chris Sosa76e44b92013-01-31 12:11:38 -0800111 'chromeos_R17-1413.0.0-a1_x86-mario_full_dev.bin'). Returns the name only if
112 found before the timeout.
113
114 Args:
Gilad Arnold950569b2013-08-27 14:38:01 -0700115 pattern: a path pattern (glob or regex) identifying the files we need.
Chris Sosa76e44b92013-01-31 12:11:38 -0800116 archive_url: URL of the Google Storage bucket.
117 err_str: String to display in the error message on error.
Gilad Arnold950569b2013-08-27 14:38:01 -0700118 timeout: how long are we allowed to keep trying.
119 delay: how long to wait between attempts.
Chris Sosa6317e9a2013-09-06 16:43:44 -0700120 is_regex_pattern: Whether the pattern is a regex (otherwise a glob).
Chris Sosa76e44b92013-01-31 12:11:38 -0800121 Returns:
122 The list of artifacts matching the pattern in Google Storage bucket or None
Gilad Arnold950569b2013-08-27 14:38:01 -0700123 if not found.
Chris Sosa76e44b92013-01-31 12:11:38 -0800124
Chris Sosa76e44b92013-01-31 12:11:38 -0800125 """
Gilad Arnold950569b2013-08-27 14:38:01 -0700126 # Define the different methods used for obtaining the list of files on the
Gilad Arnoldc703c8c2013-08-29 10:41:27 -0700127 # archive directory, in the order in which they are attempted. Each method is
128 # defined by a tuple consisting of (i) the gsutil command-line to be
129 # executed; (ii) the error message to use in case of a failure (returned in
130 # the corresponding exception); (iii) the desired return value to use in case
131 # of success, or None if the actual command output should be used.
Gilad Arnold950569b2013-08-27 14:38:01 -0700132 get_methods = []
Gilad Arnoldc703c8c2013-08-29 10:41:27 -0700133 # If the pattern is a glob and contains no wildcards, we'll first attempt to
Chris Sosa6317e9a2013-09-06 16:43:44 -0700134 # stat the file via du.
Gilad Arnoldc703c8c2013-08-29 10:41:27 -0700135 if not (is_regex_pattern or _GlobHasWildcards(pattern)):
Yu-Ju Hongc853b6e2014-11-17 15:04:55 -0800136 get_methods.append(('gsutil stat %s/%s' % (archive_url, pattern),
137 'Failed to stat on the artifact file.', pattern))
Gilad Arnoldc703c8c2013-08-29 10:41:27 -0700138
Gilad Arnold950569b2013-08-27 14:38:01 -0700139 # The default method is to check the manifest file in the archive directory.
140 get_methods.append(('gsutil cat %s/%s' % (archive_url, UPLOADED_LIST),
Gilad Arnoldc703c8c2013-08-29 10:41:27 -0700141 'Failed to get a list of uploaded files.',
142 None))
Gilad Arnold950569b2013-08-27 14:38:01 -0700143 # For backward compatibility, we fall back to using "gsutil ls" when the
144 # manifest file is not present.
145 get_methods.append(('gsutil ls %s/*' % archive_url,
Gilad Arnoldc703c8c2013-08-29 10:41:27 -0700146 'Failed to list archive directory contents.',
147 None))
Gilad Arnold950569b2013-08-27 14:38:01 -0700148
Chris Sosa76e44b92013-01-31 12:11:38 -0800149 deadline = time.time() + timeout
Ben Zhang1723fe42016-08-23 16:32:51 -0700150 _Log('crbug.com/639314 GetGSNamesWithWait():')
Chris Sosa6de29302013-03-14 15:27:36 -0700151 while True:
Chris Sosa76e44b92013-01-31 12:11:38 -0800152 uploaded_list = []
Gilad Arnoldc703c8c2013-08-29 10:41:27 -0700153 for cmd, msg, override_result in get_methods:
Gilad Arnold950569b2013-08-27 14:38:01 -0700154 try:
155 result = GSUtilRun(cmd, msg)
156 except GSUtilError:
Ben Zhang1723fe42016-08-23 16:32:51 -0700157 _Log('crbug.com/639314 GSUtilError, try next method')
Gilad Arnold950569b2013-08-27 14:38:01 -0700158 continue # It didn't work, try the next method.
Chris Sosa76e44b92013-01-31 12:11:38 -0800159
Gilad Arnoldc703c8c2013-08-29 10:41:27 -0700160 if override_result:
Ben Zhang1723fe42016-08-23 16:32:51 -0700161 _Log('crbug.com/639314 override_result={0}'.format(override_result))
Gilad Arnoldc703c8c2013-08-29 10:41:27 -0700162 result = override_result
163
Gilad Arnold950569b2013-08-27 14:38:01 -0700164 # Make sure we're dealing with artifact base names only.
165 uploaded_list = [os.path.basename(p) for p in result.splitlines()]
Ben Zhang1723fe42016-08-23 16:32:51 -0700166 _Log('crbug.com/639314 uploaded_list={0}'.format(uploaded_list))
Gilad Arnold950569b2013-08-27 14:38:01 -0700167 break
Chris Sosa76e44b92013-01-31 12:11:38 -0800168
Gilad Arnold950569b2013-08-27 14:38:01 -0700169 # Only keep files matching the target artifact name/pattern.
170 if is_regex_pattern:
171 filter_re = re.compile(pattern)
172 matching_names = [f for f in uploaded_list
173 if filter_re.search(f) is not None]
174 else:
175 matching_names = fnmatch.filter(uploaded_list, pattern)
176
Ben Zhang1723fe42016-08-23 16:32:51 -0700177 _Log('crbug.com/639314 matching_names={0}, is_regex_pattern={1}'.format(
178 matching_names, is_regex_pattern))
Gilad Arnold950569b2013-08-27 14:38:01 -0700179 if matching_names:
180 return matching_names
Chris Sosa76e44b92013-01-31 12:11:38 -0800181
Chris Sosa6de29302013-03-14 15:27:36 -0700182 # Don't delay past deadline.
183 to_delay = random.uniform(1.5 * delay, 2.5 * delay)
184 if to_delay < (deadline - time.time()):
185 _Log('Retrying in %f seconds...%s', to_delay, err_str)
186 time.sleep(to_delay)
187 else:
188 return None
joychenf8f07e22013-07-12 17:45:51 -0700189
190
joychen562699a2013-08-13 15:22:14 -0700191def GetLatestVersionFromGSDir(gsutil_dir, with_release=True):
joychenf8f07e22013-07-12 17:45:51 -0700192 """Returns most recent version number found in a GS directory.
193
194 This lists out the contents of the given GS bucket or regex to GS buckets,
195 and tries to grab the newest version found in the directory names.
Gilad Arnold950569b2013-08-27 14:38:01 -0700196
197 Args:
198 gsutil_dir: directory location on GS to check.
199 with_release: whether versions include a release milestone (e.g. R12).
200 Returns:
201 The most recent version number found.
202
joychenf8f07e22013-07-12 17:45:51 -0700203 """
Kuang-che Wuff75f3a2016-08-30 12:17:25 +0800204 cmd = 'gsutil ls -d %s' % gsutil_dir
joychenf8f07e22013-07-12 17:45:51 -0700205 msg = 'Failed to find most recent builds at %s' % gsutil_dir
206 dir_names = [p.split('/')[-2] for p in GSUtilRun(cmd, msg).splitlines()]
207 try:
Gilad Arnold950569b2013-08-27 14:38:01 -0700208 filter_re = re.compile(devserver_constants.VERSION_RE if with_release
209 else devserver_constants.VERSION)
210 versions = filter(filter_re.match, dir_names)
joychenf8f07e22013-07-12 17:45:51 -0700211 latest_version = max(versions, key=distutils.version.LooseVersion)
212 except ValueError:
213 raise GSUtilError(msg)
214
215 return latest_version