Revert "Reland "Reland "Update gsutil to use gsutil version 4.61, python3."""
This reverts commit 457736028de6b719c0f1907a54321385cf9a9eb9.
Reason for revert: crbug/1227849 - android builders aren't happy,
also a report from user about android not working properly.
Original change's description:
> Reland "Reland "Update gsutil to use gsutil version 4.61, python3.""
>
> This is a reland of e53a593956446795a68b5314459e86de38caa761
>
> Additional bug fixes:
> * handle race condition in gsutil when creating its directory
> * limit to one gsutil process on darwin due to bug in python3
>
> Original change's description:
> > Reland "Update gsutil to use gsutil version 4.61, python3."
> >
> > This reverts commit af121aeec9a2e4436c7609a39fcb61131a815b83.
> >
> > Reason for revert: re-landing with a switch back to vpython to get the compiled C extension version of crcmod for performance.
> >
> > Original change's description:
> > > Revert "Update gsutil to use gsutil version 4.61, python3."
> > >
> > > This reverts commit f059ec936899adfd83f63fff46fdb61ae3a39537.
> > >
> > > Reason for revert: Reverting because we probably need to be using vpython and a compiled crcmod instead. See, e.g.,. b/188591640.
> > >
> > > Original change's description:
> > > > Update gsutil to use gsutil version 4.61, python3.
> > > >
> > > > This CL updates the gsutil.py wrapper to download and use
> > > > v4.61 of GCP's gsutil, which is Python3-compatible.
> > > >
> > > > v4.61 appears to be fully self-contained and have all of the
> > > > packages it needs vendored into it. So, there's no reason to
> > > > use vpython anymore, and this CL removes that.
> > > >
> > > > Also, this CL removes the 'fallback' option to gsutil and
> > > > the ability to force a version switch, as this should no
> > > > longer be necessary (it was added for a migration back in 2014
> > > > but apparently this code was never removed afterwards).
> > > >
> > > > This CL also updates download_from_google_storage.py and
> > > > upload_to_google_storage.py to similarly not have the version flags
> > > > and to just use regular python3, not vpython3.
> > > >
> > > > Bug: 1184108
> > > > Change-Id: I0d1a8351dba2d3ad1f927afa333fb10959f19443
> > > > Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2898439
> > > > Reviewed-by: Mike Frysinger <vapier@chromium.org>
> > > > Reviewed-by: Josip Sokcevic <sokcevic@google.com>
> > > > Reviewed-by: Robbie Iannucci <iannucci@chromium.org>
> > > > Commit-Queue: Dirk Pranke <dpranke@google.com>
> > >
> > > Bug: 1184108
> > > Change-Id: I8e21a9a40d81e4e185642f866855b6838f80f1c2
> > > Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2905904
> > > Bot-Commit: Rubber Stamper <rubber-stamper@appspot.gserviceaccount.com>
> > > Commit-Queue: Dirk Pranke <dpranke@google.com>
> >
> > Bug: 1184108
> > Change-Id: I5d6d6d06842e08517488471c144972818fcf02ff
> > Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2907155
> > Reviewed-by: Mike Frysinger <vapier@chromium.org>
> > Reviewed-by: Josip Sokcevic <sokcevic@google.com>
> > Commit-Queue: Dirk Pranke <dpranke@google.com>
>
> Bug: 1184108
> Change-Id: Ibb5d886fd22e3553521ff8ad6e2b4435844ef972
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2988716
> Reviewed-by: Dirk Pranke <dpranke@google.com>
> Reviewed-by: Mike Frysinger <vapier@chromium.org>
> Reviewed-by: Robbie Iannucci <iannucci@chromium.org>
> Commit-Queue: Josip Sokcevic <sokcevic@google.com>
Bug: 1184108
Bug: 1227849
Change-Id: Ib4176b94861e9f85caf6ed74050111a07b1e308d
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/3021641
Reviewed-by: Dirk Pranke <dpranke@google.com>
Bot-Commit: Rubber Stamper <rubber-stamper@appspot.gserviceaccount.com>
Commit-Queue: Josip Sokcevic <sokcevic@google.com>
diff --git a/gsutil.py b/gsutil.py
index ca2e80e..d5423ad 100755
--- a/gsutil.py
+++ b/gsutil.py
@@ -31,11 +31,11 @@
THIS_DIR = os.path.dirname(os.path.abspath(__file__))
DEFAULT_BIN_DIR = os.path.join(THIS_DIR, 'external_bin', 'gsutil')
+DEFAULT_FALLBACK_GSUTIL = os.path.join(
+ THIS_DIR, 'third_party', 'gsutil', 'gsutil')
IS_WINDOWS = os.name == 'nt'
-VERSION = '4.61'
-
class InvalidGsutilError(Exception):
pass
@@ -98,20 +98,7 @@
return gsutil_bin
if not os.path.exists(target):
- try:
- os.makedirs(target)
- except FileExistsError:
- # Another process is prepping workspace, so let's check if gsutil_bin is
- # present. If after several checks it's still not, continue with
- # downloading gsutil.
- delay = 2 # base delay, in seconds
- for _ in range(3): # make N attempts
- # sleep first as it's not expected to have file ready just yet.
- time.sleep(delay)
- delay *= 1.5 # next delay increased by that factor
- if os.path.isfile(gsutil_bin):
- return gsutil_bin
-
+ os.makedirs(target)
with temporary_directory(target) as instance_dir:
# Clean up if we're redownloading a corrupted gsutil.
cleanup_path = os.path.join(instance_dir, 'clean')
@@ -142,18 +129,13 @@
return gsutil_bin
-def run_gsutil(target, args, clean=False):
- gsutil_bin = ensure_gsutil(VERSION, target, clean)
- args_opt = ['-o', 'GSUtil:software_update_check_period=0']
+def run_gsutil(force_version, fallback, target, args, clean=False):
+ if force_version:
+ gsutil_bin = ensure_gsutil(force_version, target, clean)
+ else:
+ gsutil_bin = fallback
+ disable_update = ['-o', 'GSUtil:software_update_check_period=0']
- if sys.platform == 'darwin':
- # We are experiencing problems with multiprocessing on MacOS where gsutil.py
- # may hang.
- # This behavior is documented in gsutil codebase, and recommendation is to
- # set GSUtil:parallel_process_count=1.
- # https://github.com/GoogleCloudPlatform/gsutil/blob/06efc9dc23719fab4fd5fadb506d252bbd3fe0dd/gslib/command.py#L1331
- # https://github.com/GoogleCloudPlatform/gsutil/issues/1100
- args_opt.extend(['-o', 'GSUtil:parallel_process_count=1'])
if sys.platform == 'cygwin':
# This script requires Windows Python, so invoke with depot_tools'
# Python.
@@ -165,10 +147,15 @@
sys.exit(subprocess.call(cmd))
assert sys.platform != 'cygwin'
+ # Run "gsutil" through "vpython". We need to do this because on GCE instances,
+ # expectations are made about Python having access to "google-compute-engine"
+ # and "boto" packages that are not met with non-system Python (e.g., bundles).
cmd = [
- sys.executable,
+ 'vpython',
+ '-vpython-spec', os.path.join(THIS_DIR, 'gsutil.vpython'),
+ '--',
gsutil_bin
- ] + args_opt + args
+ ] + disable_update + args
return subprocess.call(cmd, shell=IS_WINDOWS)
@@ -176,19 +163,13 @@
bin_dir = os.environ.get('DEPOT_TOOLS_GSUTIL_BIN_DIR', DEFAULT_BIN_DIR)
parser = argparse.ArgumentParser()
-
+ parser.add_argument('--force-version', default='4.30')
parser.add_argument('--clean', action='store_true',
help='Clear any existing gsutil package, forcing a new download.')
+ parser.add_argument('--fallback', default=DEFAULT_FALLBACK_GSUTIL)
parser.add_argument('--target', default=bin_dir,
help='The target directory to download/store a gsutil version in. '
'(default is %(default)s).')
-
- # These two args exist for backwards-compatibility but are no-ops.
- parser.add_argument('--force-version', default=VERSION,
- help='(deprecated, this flag has no effect)')
- parser.add_argument('--fallback',
- help='(deprecated, this flag has no effect)')
-
parser.add_argument('args', nargs=argparse.REMAINDER)
args, extras = parser.parse_known_args()
@@ -201,7 +182,8 @@
def main():
args = parse_args()
- return run_gsutil(args.target, args.args, clean=args.clean)
+ return run_gsutil(args.force_version, args.fallback, args.target, args.args,
+ clean=args.clean)
if __name__ == '__main__':