Chris Sosa | 47a7d4e | 2012-03-28 11:26:55 -0700 | [diff] [blame] | 1 | # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. |
| 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | |
| 5 | """Module containing gsutil helper methods.""" |
| 6 | |
joychen | f8f07e2 | 2013-07-12 17:45:51 -0700 | [diff] [blame] | 7 | import distutils.version |
Gilad Arnold | 950569b | 2013-08-27 14:38:01 -0700 | [diff] [blame] | 8 | import fnmatch |
| 9 | import os |
Chris Sosa | 76e44b9 | 2013-01-31 12:11:38 -0800 | [diff] [blame] | 10 | import random |
| 11 | import re |
Chris Sosa | 47a7d4e | 2012-03-28 11:26:55 -0700 | [diff] [blame] | 12 | import subprocess |
Chris Sosa | 101fd86 | 2012-06-12 17:44:53 -0700 | [diff] [blame] | 13 | import time |
Chris Sosa | 47a7d4e | 2012-03-28 11:26:55 -0700 | [diff] [blame] | 14 | |
joychen | f8f07e2 | 2013-07-12 17:45:51 -0700 | [diff] [blame] | 15 | import devserver_constants |
Chris Sosa | 76e44b9 | 2013-01-31 12:11:38 -0800 | [diff] [blame] | 16 | import log_util |
| 17 | |
Gilad Arnold | abb352e | 2012-09-23 01:24:27 -0700 | [diff] [blame] | 18 | |
Chris Sosa | 7cd2320 | 2013-10-15 17:22:57 -0700 | [diff] [blame] | 19 | GSUTIL_ATTEMPTS = 1 |
Chris Sosa | 76e44b9 | 2013-01-31 12:11:38 -0800 | [diff] [blame] | 20 | UPLOADED_LIST = 'UPLOADED' |
| 21 | |
| 22 | |
| 23 | # Module-local log function. |
| 24 | def _Log(message, *args): |
| 25 | return log_util.LogWithTag('GSUTIL_UTIL', message, *args) |
Chris Sosa | 47a7d4e | 2012-03-28 11:26:55 -0700 | [diff] [blame] | 26 | |
| 27 | |
| 28 | class GSUtilError(Exception): |
Chris Sosa | 76e44b9 | 2013-01-31 12:11:38 -0800 | [diff] [blame] | 29 | """Exception raised when we run into an error running gsutil.""" |
| 30 | pass |
| 31 | |
| 32 | |
| 33 | class PatternNotSpecific(Exception): |
| 34 | """Raised when unexpectedly more than one item is returned for a pattern.""" |
Chris Sosa | 47a7d4e | 2012-03-28 11:26:55 -0700 | [diff] [blame] | 35 | pass |
| 36 | |
| 37 | |
| 38 | def GSUtilRun(cmd, err_msg): |
| 39 | """Runs a GSUTIL command up to GSUTIL_ATTEMPTS number of times. |
| 40 | |
Chris Sosa | 101fd86 | 2012-06-12 17:44:53 -0700 | [diff] [blame] | 41 | Attempts are tried with exponential backoff. |
| 42 | |
Gilad Arnold | 950569b | 2013-08-27 14:38:01 -0700 | [diff] [blame] | 43 | Args: |
| 44 | cmd: a string containing the gsutil command to run. |
| 45 | err_msg: string prepended to the exception thrown in case of a failure. |
Chris Sosa | 47a7d4e | 2012-03-28 11:26:55 -0700 | [diff] [blame] | 46 | Returns: |
| 47 | stdout of the called gsutil command. |
| 48 | Raises: |
Gilad Arnold | 950569b | 2013-08-27 14:38:01 -0700 | [diff] [blame] | 49 | GSUtilError: if all attempts to run gsutil have failed. |
Chris Sosa | 47a7d4e | 2012-03-28 11:26:55 -0700 | [diff] [blame] | 50 | """ |
| 51 | proc = None |
Chris Sosa | 101fd86 | 2012-06-12 17:44:53 -0700 | [diff] [blame] | 52 | sleep_timeout = 1 |
Chris Sosa | 407e8c5 | 2013-02-27 15:33:35 -0800 | [diff] [blame] | 53 | stderr = None |
Gilad Arnold | 950569b | 2013-08-27 14:38:01 -0700 | [diff] [blame] | 54 | for _ in range(GSUTIL_ATTEMPTS): |
Chris Sosa | 407e8c5 | 2013-02-27 15:33:35 -0800 | [diff] [blame] | 55 | proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, |
| 56 | stderr=subprocess.PIPE) |
| 57 | stdout, stderr = proc.communicate() |
Ben Zhang | 1723fe4 | 2016-08-23 16:32:51 -0700 | [diff] [blame] | 58 | _Log('crbug.com/639314 GSUtilRun({0}, {1})={2}, stdout=<<<{3}>>>, stderr=<<<{4}>>>'.format( |
| 59 | cmd, err_msg, proc.returncode, stdout, stderr)) |
Chris Sosa | 47a7d4e | 2012-03-28 11:26:55 -0700 | [diff] [blame] | 60 | if proc.returncode == 0: |
| 61 | return stdout |
Yu-Ju Hong | c8974c0 | 2014-11-18 11:40:30 -0800 | [diff] [blame] | 62 | |
| 63 | not_exist_messages = ('matched no objects', 'non-existent object', |
| 64 | 'no urls matched') |
| 65 | if (stderr and any(x in stderr.lower() for x in not_exist_messages) or |
| 66 | stdout and any(x in stdout.lower() for x in not_exist_messages)): |
| 67 | # If the object does not exist, exit now instead of wasting time |
| 68 | # on retrying. Note that `gsutil stat` prints error message to |
| 69 | # stdout instead (b/16020252), so we check both stdout and |
| 70 | # stderr. |
Chris Sosa | 407e8c5 | 2013-02-27 15:33:35 -0800 | [diff] [blame] | 71 | break |
Yu-Ju Hong | c8974c0 | 2014-11-18 11:40:30 -0800 | [diff] [blame] | 72 | |
| 73 | if proc.returncode == 127: |
Chris Sosa | 6de2930 | 2013-03-14 15:27:36 -0700 | [diff] [blame] | 74 | raise GSUtilError('gsutil tool not found in your path.') |
Chris Sosa | 101fd86 | 2012-06-12 17:44:53 -0700 | [diff] [blame] | 75 | |
| 76 | time.sleep(sleep_timeout) |
| 77 | sleep_timeout *= 2 |
| 78 | |
Chris Sosa | 407e8c5 | 2013-02-27 15:33:35 -0800 | [diff] [blame] | 79 | raise GSUtilError('%s GSUTIL cmd %s failed with return code %d:\n\n%s' % ( |
| 80 | err_msg, cmd, proc.returncode, stderr)) |
Chris Sosa | 47a7d4e | 2012-03-28 11:26:55 -0700 | [diff] [blame] | 81 | |
| 82 | |
| 83 | def DownloadFromGS(src, dst): |
| 84 | """Downloads object from gs_url |src| to |dst|. |
| 85 | |
Gilad Arnold | 950569b | 2013-08-27 14:38:01 -0700 | [diff] [blame] | 86 | Args: |
| 87 | src: source file on GS that needs to be downloaded. |
| 88 | dst: file to copy the source file to. |
Chris Sosa | 47a7d4e | 2012-03-28 11:26:55 -0700 | [diff] [blame] | 89 | Raises: |
| 90 | GSUtilError: if an error occurs during the download. |
| 91 | """ |
| 92 | cmd = 'gsutil cp %s %s' % (src, dst) |
| 93 | msg = 'Failed to download "%s".' % src |
| 94 | GSUtilRun(cmd, msg) |
Chris Sosa | 76e44b9 | 2013-01-31 12:11:38 -0800 | [diff] [blame] | 95 | |
| 96 | |
Gilad Arnold | 950569b | 2013-08-27 14:38:01 -0700 | [diff] [blame] | 97 | def _GlobHasWildcards(pattern): |
| 98 | """Returns True if a glob pattern contains any wildcards.""" |
Gilad Arnold | c703c8c | 2013-08-29 10:41:27 -0700 | [diff] [blame] | 99 | return len(pattern) > len(pattern.translate(None, '*?[]')) |
Chris Sosa | 76e44b9 | 2013-01-31 12:11:38 -0800 | [diff] [blame] | 100 | |
| 101 | |
Gilad Arnold | 950569b | 2013-08-27 14:38:01 -0700 | [diff] [blame] | 102 | def GetGSNamesWithWait(pattern, archive_url, err_str, timeout=600, delay=10, |
| 103 | is_regex_pattern=False): |
Chris Sosa | 76e44b9 | 2013-01-31 12:11:38 -0800 | [diff] [blame] | 104 | """Returns the google storage names specified by the given pattern. |
| 105 | |
| 106 | This method polls Google Storage until the target artifacts specified by the |
| 107 | pattern is available or until the timeout occurs. Because we may not know the |
| 108 | exact name of the target artifacts, the method accepts a filename pattern, |
| 109 | to identify whether an artifact whose name matches the pattern exists (e.g. |
Gilad Arnold | 950569b | 2013-08-27 14:38:01 -0700 | [diff] [blame] | 110 | use pattern '*_full_*' to search for the full payload |
Chris Sosa | 76e44b9 | 2013-01-31 12:11:38 -0800 | [diff] [blame] | 111 | 'chromeos_R17-1413.0.0-a1_x86-mario_full_dev.bin'). Returns the name only if |
| 112 | found before the timeout. |
| 113 | |
| 114 | Args: |
Gilad Arnold | 950569b | 2013-08-27 14:38:01 -0700 | [diff] [blame] | 115 | pattern: a path pattern (glob or regex) identifying the files we need. |
Chris Sosa | 76e44b9 | 2013-01-31 12:11:38 -0800 | [diff] [blame] | 116 | archive_url: URL of the Google Storage bucket. |
| 117 | err_str: String to display in the error message on error. |
Gilad Arnold | 950569b | 2013-08-27 14:38:01 -0700 | [diff] [blame] | 118 | timeout: how long are we allowed to keep trying. |
| 119 | delay: how long to wait between attempts. |
Chris Sosa | 6317e9a | 2013-09-06 16:43:44 -0700 | [diff] [blame] | 120 | is_regex_pattern: Whether the pattern is a regex (otherwise a glob). |
Chris Sosa | 76e44b9 | 2013-01-31 12:11:38 -0800 | [diff] [blame] | 121 | Returns: |
| 122 | The list of artifacts matching the pattern in Google Storage bucket or None |
Gilad Arnold | 950569b | 2013-08-27 14:38:01 -0700 | [diff] [blame] | 123 | if not found. |
Chris Sosa | 76e44b9 | 2013-01-31 12:11:38 -0800 | [diff] [blame] | 124 | |
Chris Sosa | 76e44b9 | 2013-01-31 12:11:38 -0800 | [diff] [blame] | 125 | """ |
Gilad Arnold | 950569b | 2013-08-27 14:38:01 -0700 | [diff] [blame] | 126 | # Define the different methods used for obtaining the list of files on the |
Gilad Arnold | c703c8c | 2013-08-29 10:41:27 -0700 | [diff] [blame] | 127 | # archive directory, in the order in which they are attempted. Each method is |
| 128 | # defined by a tuple consisting of (i) the gsutil command-line to be |
| 129 | # executed; (ii) the error message to use in case of a failure (returned in |
| 130 | # the corresponding exception); (iii) the desired return value to use in case |
| 131 | # of success, or None if the actual command output should be used. |
Gilad Arnold | 950569b | 2013-08-27 14:38:01 -0700 | [diff] [blame] | 132 | get_methods = [] |
Gilad Arnold | c703c8c | 2013-08-29 10:41:27 -0700 | [diff] [blame] | 133 | # If the pattern is a glob and contains no wildcards, we'll first attempt to |
Chris Sosa | 6317e9a | 2013-09-06 16:43:44 -0700 | [diff] [blame] | 134 | # stat the file via du. |
Gilad Arnold | c703c8c | 2013-08-29 10:41:27 -0700 | [diff] [blame] | 135 | if not (is_regex_pattern or _GlobHasWildcards(pattern)): |
Yu-Ju Hong | c853b6e | 2014-11-17 15:04:55 -0800 | [diff] [blame] | 136 | get_methods.append(('gsutil stat %s/%s' % (archive_url, pattern), |
| 137 | 'Failed to stat on the artifact file.', pattern)) |
Gilad Arnold | c703c8c | 2013-08-29 10:41:27 -0700 | [diff] [blame] | 138 | |
Gilad Arnold | 950569b | 2013-08-27 14:38:01 -0700 | [diff] [blame] | 139 | # The default method is to check the manifest file in the archive directory. |
| 140 | get_methods.append(('gsutil cat %s/%s' % (archive_url, UPLOADED_LIST), |
Gilad Arnold | c703c8c | 2013-08-29 10:41:27 -0700 | [diff] [blame] | 141 | 'Failed to get a list of uploaded files.', |
| 142 | None)) |
Gilad Arnold | 950569b | 2013-08-27 14:38:01 -0700 | [diff] [blame] | 143 | # For backward compatibility, we fall back to using "gsutil ls" when the |
| 144 | # manifest file is not present. |
| 145 | get_methods.append(('gsutil ls %s/*' % archive_url, |
Gilad Arnold | c703c8c | 2013-08-29 10:41:27 -0700 | [diff] [blame] | 146 | 'Failed to list archive directory contents.', |
| 147 | None)) |
Gilad Arnold | 950569b | 2013-08-27 14:38:01 -0700 | [diff] [blame] | 148 | |
Chris Sosa | 76e44b9 | 2013-01-31 12:11:38 -0800 | [diff] [blame] | 149 | deadline = time.time() + timeout |
Ben Zhang | 1723fe4 | 2016-08-23 16:32:51 -0700 | [diff] [blame] | 150 | _Log('crbug.com/639314 GetGSNamesWithWait():') |
Chris Sosa | 6de2930 | 2013-03-14 15:27:36 -0700 | [diff] [blame] | 151 | while True: |
Chris Sosa | 76e44b9 | 2013-01-31 12:11:38 -0800 | [diff] [blame] | 152 | uploaded_list = [] |
Gilad Arnold | c703c8c | 2013-08-29 10:41:27 -0700 | [diff] [blame] | 153 | for cmd, msg, override_result in get_methods: |
Gilad Arnold | 950569b | 2013-08-27 14:38:01 -0700 | [diff] [blame] | 154 | try: |
| 155 | result = GSUtilRun(cmd, msg) |
| 156 | except GSUtilError: |
Ben Zhang | 1723fe4 | 2016-08-23 16:32:51 -0700 | [diff] [blame] | 157 | _Log('crbug.com/639314 GSUtilError, try next method') |
Gilad Arnold | 950569b | 2013-08-27 14:38:01 -0700 | [diff] [blame] | 158 | continue # It didn't work, try the next method. |
Chris Sosa | 76e44b9 | 2013-01-31 12:11:38 -0800 | [diff] [blame] | 159 | |
Gilad Arnold | c703c8c | 2013-08-29 10:41:27 -0700 | [diff] [blame] | 160 | if override_result: |
Ben Zhang | 1723fe4 | 2016-08-23 16:32:51 -0700 | [diff] [blame] | 161 | _Log('crbug.com/639314 override_result={0}'.format(override_result)) |
Gilad Arnold | c703c8c | 2013-08-29 10:41:27 -0700 | [diff] [blame] | 162 | result = override_result |
| 163 | |
Gilad Arnold | 950569b | 2013-08-27 14:38:01 -0700 | [diff] [blame] | 164 | # Make sure we're dealing with artifact base names only. |
| 165 | uploaded_list = [os.path.basename(p) for p in result.splitlines()] |
Ben Zhang | 1723fe4 | 2016-08-23 16:32:51 -0700 | [diff] [blame] | 166 | _Log('crbug.com/639314 uploaded_list={0}'.format(uploaded_list)) |
Gilad Arnold | 950569b | 2013-08-27 14:38:01 -0700 | [diff] [blame] | 167 | break |
Chris Sosa | 76e44b9 | 2013-01-31 12:11:38 -0800 | [diff] [blame] | 168 | |
Gilad Arnold | 950569b | 2013-08-27 14:38:01 -0700 | [diff] [blame] | 169 | # Only keep files matching the target artifact name/pattern. |
| 170 | if is_regex_pattern: |
| 171 | filter_re = re.compile(pattern) |
| 172 | matching_names = [f for f in uploaded_list |
| 173 | if filter_re.search(f) is not None] |
| 174 | else: |
| 175 | matching_names = fnmatch.filter(uploaded_list, pattern) |
| 176 | |
Ben Zhang | 1723fe4 | 2016-08-23 16:32:51 -0700 | [diff] [blame] | 177 | _Log('crbug.com/639314 matching_names={0}, is_regex_pattern={1}'.format( |
| 178 | matching_names, is_regex_pattern)) |
Gilad Arnold | 950569b | 2013-08-27 14:38:01 -0700 | [diff] [blame] | 179 | if matching_names: |
| 180 | return matching_names |
Chris Sosa | 76e44b9 | 2013-01-31 12:11:38 -0800 | [diff] [blame] | 181 | |
Chris Sosa | 6de2930 | 2013-03-14 15:27:36 -0700 | [diff] [blame] | 182 | # Don't delay past deadline. |
| 183 | to_delay = random.uniform(1.5 * delay, 2.5 * delay) |
| 184 | if to_delay < (deadline - time.time()): |
| 185 | _Log('Retrying in %f seconds...%s', to_delay, err_str) |
| 186 | time.sleep(to_delay) |
| 187 | else: |
| 188 | return None |
joychen | f8f07e2 | 2013-07-12 17:45:51 -0700 | [diff] [blame] | 189 | |
| 190 | |
joychen | 562699a | 2013-08-13 15:22:14 -0700 | [diff] [blame] | 191 | def GetLatestVersionFromGSDir(gsutil_dir, with_release=True): |
joychen | f8f07e2 | 2013-07-12 17:45:51 -0700 | [diff] [blame] | 192 | """Returns most recent version number found in a GS directory. |
| 193 | |
| 194 | This lists out the contents of the given GS bucket or regex to GS buckets, |
| 195 | and tries to grab the newest version found in the directory names. |
Gilad Arnold | 950569b | 2013-08-27 14:38:01 -0700 | [diff] [blame] | 196 | |
| 197 | Args: |
| 198 | gsutil_dir: directory location on GS to check. |
| 199 | with_release: whether versions include a release milestone (e.g. R12). |
| 200 | Returns: |
| 201 | The most recent version number found. |
| 202 | |
joychen | f8f07e2 | 2013-07-12 17:45:51 -0700 | [diff] [blame] | 203 | """ |
Kuang-che Wu | ff75f3a | 2016-08-30 12:17:25 +0800 | [diff] [blame] | 204 | cmd = 'gsutil ls -d %s' % gsutil_dir |
joychen | f8f07e2 | 2013-07-12 17:45:51 -0700 | [diff] [blame] | 205 | msg = 'Failed to find most recent builds at %s' % gsutil_dir |
| 206 | dir_names = [p.split('/')[-2] for p in GSUtilRun(cmd, msg).splitlines()] |
| 207 | try: |
Gilad Arnold | 950569b | 2013-08-27 14:38:01 -0700 | [diff] [blame] | 208 | filter_re = re.compile(devserver_constants.VERSION_RE if with_release |
| 209 | else devserver_constants.VERSION) |
| 210 | versions = filter(filter_re.match, dir_names) |
joychen | f8f07e2 | 2013-07-12 17:45:51 -0700 | [diff] [blame] | 211 | latest_version = max(versions, key=distutils.version.LooseVersion) |
| 212 | except ValueError: |
| 213 | raise GSUtilError(msg) |
| 214 | |
| 215 | return latest_version |