blob: 55e4cb035d13aa7bee0fa28a7616a66cb9d8f97c [file] [log] [blame]
hinoka@chromium.org7a790542014-12-10 02:04:39 +00001#!/usr/bin/env python
2# Copyright 2014 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Run a pinned gsutil."""
7
8
9import argparse
hinoka@chromium.org7a790542014-12-10 02:04:39 +000010import base64
dnj@chromium.org605d81d2015-09-18 22:33:53 +000011import contextlib
primiano@chromium.orgdf351762014-12-18 11:12:34 +000012import hashlib
hinoka@chromium.org7a790542014-12-10 02:04:39 +000013import json
primiano@chromium.orgdf351762014-12-18 11:12:34 +000014import os
15import shutil
hinoka@chromium.org7a790542014-12-10 02:04:39 +000016import subprocess
primiano@chromium.orgdf351762014-12-18 11:12:34 +000017import sys
dnj@chromium.org605d81d2015-09-18 22:33:53 +000018import tempfile
19import time
Raul Tambreb946b232019-03-26 14:48:46 +000020
21try:
22 import urllib2 as urllib
23except ImportError: # For Py3 compatibility
24 import urllib.request as urllib
25
primiano@chromium.orgdf351762014-12-18 11:12:34 +000026import zipfile
hinoka@chromium.org7a790542014-12-10 02:04:39 +000027
28
29GSUTIL_URL = 'https://storage.googleapis.com/pub/'
30API_URL = 'https://www.googleapis.com/storage/v1/b/pub/o/'
31
32THIS_DIR = os.path.dirname(os.path.abspath(__file__))
33DEFAULT_BIN_DIR = os.path.join(THIS_DIR, 'external_bin', 'gsutil')
34DEFAULT_FALLBACK_GSUTIL = os.path.join(
35 THIS_DIR, 'third_party', 'gsutil', 'gsutil')
36
Dan Jacques509776e2017-09-07 18:01:08 -070037IS_WINDOWS = os.name == 'nt'
38
39
hinoka@chromium.org7a790542014-12-10 02:04:39 +000040class InvalidGsutilError(Exception):
41 pass
42
43
hinoka@chromium.org7a790542014-12-10 02:04:39 +000044def download_gsutil(version, target_dir):
45 """Downloads gsutil into the target_dir."""
46 filename = 'gsutil_%s.zip' % version
47 target_filename = os.path.join(target_dir, filename)
48
49 # Check if the target exists already.
50 if os.path.exists(target_filename):
51 md5_calc = hashlib.md5()
52 with open(target_filename, 'rb') as f:
53 while True:
54 buf = f.read(4096)
55 if not buf:
56 break
57 md5_calc.update(buf)
58 local_md5 = md5_calc.hexdigest()
59
60 metadata_url = '%s%s' % (API_URL, filename)
Raul Tambreb946b232019-03-26 14:48:46 +000061 metadata = json.load(urllib.urlopen(metadata_url))
hinoka@chromium.org7a790542014-12-10 02:04:39 +000062 remote_md5 = base64.b64decode(metadata['md5Hash'])
63
64 if local_md5 == remote_md5:
65 return target_filename
66 os.remove(target_filename)
67
68 # Do the download.
69 url = '%s%s' % (GSUTIL_URL, filename)
Raul Tambreb946b232019-03-26 14:48:46 +000070 u = urllib.urlopen(url)
hinoka@chromium.org7a790542014-12-10 02:04:39 +000071 with open(target_filename, 'wb') as f:
72 while True:
73 buf = u.read(4096)
74 if not buf:
75 break
76 f.write(buf)
77 return target_filename
78
79
dnj@chromium.org605d81d2015-09-18 22:33:53 +000080@contextlib.contextmanager
81def temporary_directory(base):
82 tmpdir = tempfile.mkdtemp(prefix='gsutil_py', dir=base)
83 try:
84 yield tmpdir
85 finally:
86 if os.path.isdir(tmpdir):
87 shutil.rmtree(tmpdir)
88
89def ensure_gsutil(version, target, clean):
hinoka@chromium.org7a790542014-12-10 02:04:39 +000090 bin_dir = os.path.join(target, 'gsutil_%s' % version)
91 gsutil_bin = os.path.join(bin_dir, 'gsutil', 'gsutil')
Ryan Tseng83fd81f2017-10-23 11:13:48 -070092 gsutil_flag = os.path.join(bin_dir, 'gsutil', 'install.flag')
93 # We assume that if gsutil_flag exists, then we have a good version
94 # of the gsutil package.
95 if not clean and os.path.isfile(gsutil_flag):
hinoka@chromium.org7a790542014-12-10 02:04:39 +000096 # Everything is awesome! we're all done here.
97 return gsutil_bin
98
dnj@chromium.org605d81d2015-09-18 22:33:53 +000099 if not os.path.exists(target):
100 os.makedirs(target)
101 with temporary_directory(target) as instance_dir:
hinoka@chromium.org7a790542014-12-10 02:04:39 +0000102 # Clean up if we're redownloading a corrupted gsutil.
dnj@chromium.org605d81d2015-09-18 22:33:53 +0000103 cleanup_path = os.path.join(instance_dir, 'clean')
104 try:
105 os.rename(bin_dir, cleanup_path)
106 except (OSError, IOError):
107 cleanup_path = None
108 if cleanup_path:
109 shutil.rmtree(cleanup_path)
110
111 download_dir = os.path.join(instance_dir, 'download')
112 target_zip_filename = download_gsutil(version, instance_dir)
113 with zipfile.ZipFile(target_zip_filename, 'r') as target_zip:
114 target_zip.extractall(download_dir)
115
116 try:
117 os.rename(download_dir, bin_dir)
118 except (OSError, IOError):
119 # Something else did this in parallel.
120 pass
Ryan Tseng83fd81f2017-10-23 11:13:48 -0700121 # Final check that the gsutil bin exists. This should never fail.
122 if not os.path.isfile(gsutil_bin):
123 raise InvalidGsutilError()
124 # Drop a flag file.
125 with open(gsutil_flag, 'w') as f:
126 f.write('This flag file is dropped by gsutil.py')
hinoka@chromium.org7a790542014-12-10 02:04:39 +0000127
hinoka@chromium.org7a790542014-12-10 02:04:39 +0000128 return gsutil_bin
129
130
dnj@chromium.org605d81d2015-09-18 22:33:53 +0000131def run_gsutil(force_version, fallback, target, args, clean=False):
hinoka@chromium.org7a790542014-12-10 02:04:39 +0000132 if force_version:
dnj@chromium.org605d81d2015-09-18 22:33:53 +0000133 gsutil_bin = ensure_gsutil(force_version, target, clean)
hinoka@chromium.org7a790542014-12-10 02:04:39 +0000134 else:
135 gsutil_bin = fallback
hinoka@chromium.orgfdb9ce32016-04-05 23:57:12 +0000136 disable_update = ['-o', 'GSUtil:software_update_check_period=0']
Dan Jacques509776e2017-09-07 18:01:08 -0700137
Chris Nardiab816ce2017-10-31 15:45:05 -0400138 if sys.platform == 'cygwin':
139 # This script requires Windows Python, so invoke with depot_tools'
140 # Python.
141 def winpath(path):
142 return subprocess.check_output(['cygpath', '-w', path]).strip()
143 cmd = ['python.bat', winpath(__file__)]
144 cmd.extend(args)
145 sys.exit(subprocess.call(cmd))
146 assert sys.platform != 'cygwin'
147
Dan Jacques509776e2017-09-07 18:01:08 -0700148 # Run "gsutil" through "vpython". We need to do this because on GCE instances,
149 # expectations are made about Python having access to "google-compute-engine"
150 # and "boto" packages that are not met with non-system Python (e.g., bundles).
151 cmd = [
152 'vpython',
153 '-vpython-spec', os.path.join(THIS_DIR, 'gsutil.vpython'),
154 '--',
155 gsutil_bin
156 ] + disable_update + args
157 return subprocess.call(cmd, shell=IS_WINDOWS)
hinoka@chromium.org7a790542014-12-10 02:04:39 +0000158
159
160def parse_args():
dnj@chromium.org605d81d2015-09-18 22:33:53 +0000161 bin_dir = os.environ.get('DEPOT_TOOLS_GSUTIL_BIN_DIR', DEFAULT_BIN_DIR)
162
hinoka@chromium.org7a790542014-12-10 02:04:39 +0000163 parser = argparse.ArgumentParser()
Caleb Rouleau63e216c2018-03-29 14:20:37 -0700164 parser.add_argument('--force-version', default='4.30')
dnj@chromium.org605d81d2015-09-18 22:33:53 +0000165 parser.add_argument('--clean', action='store_true',
166 help='Clear any existing gsutil package, forcing a new download.')
hinoka@chromium.org7a790542014-12-10 02:04:39 +0000167 parser.add_argument('--fallback', default=DEFAULT_FALLBACK_GSUTIL)
dnj@chromium.org605d81d2015-09-18 22:33:53 +0000168 parser.add_argument('--target', default=bin_dir,
169 help='The target directory to download/store a gsutil version in. '
170 '(default is %(default)s).')
hinoka@chromium.org7a790542014-12-10 02:04:39 +0000171 parser.add_argument('args', nargs=argparse.REMAINDER)
172
hinoka@chromium.orgc13b0542014-12-18 01:06:20 +0000173 args, extras = parser.parse_known_args()
174 if args.args and args.args[0] == '--':
175 args.args.pop(0)
176 if extras:
177 args.args = extras + args.args
dnj@chromium.org605d81d2015-09-18 22:33:53 +0000178 return args
hinoka@chromium.org7a790542014-12-10 02:04:39 +0000179
180
181def main():
dnj@chromium.org605d81d2015-09-18 22:33:53 +0000182 args = parse_args()
183 return run_gsutil(args.force_version, args.fallback, args.target, args.args,
184 clean=args.clean)
hinoka@chromium.org7a790542014-12-10 02:04:39 +0000185
186if __name__ == '__main__':
187 sys.exit(main())