Refactor gsutil and fix concurrecny issue on MacOS

This is a partial reland of commit
e8ef6259b967f7d76d96fe3039dc7c9614a52d6d, but without actually upgrading
gsutil.

This refactoring allows easier upgrade of gsutil.

R=dpranke@google.com

Bug: 1249003
Change-Id: I63433b411e640a9fdb455ce25a51d910be0818a9
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/3166676
Reviewed-by: Dirk Pranke <dpranke@google.com>
Commit-Queue: Josip Sokcevic <sokcevic@google.com>
diff --git a/gsutil.py b/gsutil.py
index d5423ad..59a0e14 100755
--- a/gsutil.py
+++ b/gsutil.py
@@ -31,11 +31,11 @@
 
 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
 DEFAULT_BIN_DIR = os.path.join(THIS_DIR, 'external_bin', 'gsutil')
-DEFAULT_FALLBACK_GSUTIL = os.path.join(
-    THIS_DIR, 'third_party', 'gsutil', 'gsutil')
 
 IS_WINDOWS = os.name == 'nt'
 
+VERSION = '4.28'
+
 
 class InvalidGsutilError(Exception):
   pass
@@ -98,7 +98,20 @@
     return gsutil_bin
 
   if not os.path.exists(target):
-    os.makedirs(target)
+    try:
+      os.makedirs(target)
+    except FileExistsError:
+      # Another process is prepping workspace, so let's check if gsutil_bin is
+      # present.  If after several checks it's still not, continue with
+      # downloading gsutil.
+      delay = 2  # base delay, in seconds
+      for _ in range(3):  # make N attempts
+        # sleep first as it's not expected to have file ready just yet.
+        time.sleep(delay)
+        delay *= 1.5  # next delay increased by that factor
+        if os.path.isfile(gsutil_bin):
+          return gsutil_bin
+
   with temporary_directory(target) as instance_dir:
     # Clean up if we're redownloading a corrupted gsutil.
     cleanup_path = os.path.join(instance_dir, 'clean')
@@ -129,13 +142,18 @@
   return gsutil_bin
 
 
-def run_gsutil(force_version, fallback, target, args, clean=False):
-  if force_version:
-    gsutil_bin = ensure_gsutil(force_version, target, clean)
-  else:
-    gsutil_bin = fallback
-  disable_update = ['-o', 'GSUtil:software_update_check_period=0']
+def run_gsutil(target, args, clean=False):
+  gsutil_bin = ensure_gsutil(VERSION, target, clean)
+  args_opt = ['-o', 'GSUtil:software_update_check_period=0']
 
+  if sys.platform == 'darwin':
+    # We are experiencing problems with multiprocessing on MacOS where gsutil.py
+    # may hang.
+    # This behavior is documented in gsutil codebase, and recommendation is to
+    # set GSUtil:parallel_process_count=1.
+    # https://github.com/GoogleCloudPlatform/gsutil/blob/06efc9dc23719fab4fd5fadb506d252bbd3fe0dd/gslib/command.py#L1331
+    # https://github.com/GoogleCloudPlatform/gsutil/issues/1100
+    args_opt.extend(['-o', 'GSUtil:parallel_process_count=1'])
   if sys.platform == 'cygwin':
     # This script requires Windows Python, so invoke with depot_tools'
     # Python.
@@ -147,15 +165,12 @@
     sys.exit(subprocess.call(cmd))
   assert sys.platform != 'cygwin'
 
-  # Run "gsutil" through "vpython". We need to do this because on GCE instances,
-  # expectations are made about Python having access to "google-compute-engine"
-  # and "boto" packages that are not met with non-system Python (e.g., bundles).
   cmd = [
       'vpython',
       '-vpython-spec', os.path.join(THIS_DIR, 'gsutil.vpython'),
       '--',
       gsutil_bin
-  ] + disable_update + args
+  ] + args_opt + args
   return subprocess.call(cmd, shell=IS_WINDOWS)
 
 
@@ -163,13 +178,19 @@
   bin_dir = os.environ.get('DEPOT_TOOLS_GSUTIL_BIN_DIR', DEFAULT_BIN_DIR)
 
   parser = argparse.ArgumentParser()
-  parser.add_argument('--force-version', default='4.30')
+
   parser.add_argument('--clean', action='store_true',
       help='Clear any existing gsutil package, forcing a new download.')
-  parser.add_argument('--fallback', default=DEFAULT_FALLBACK_GSUTIL)
   parser.add_argument('--target', default=bin_dir,
       help='The target directory to download/store a gsutil version in. '
            '(default is %(default)s).')
+
+  # These two args exist for backwards-compatibility but are no-ops.
+  parser.add_argument('--force-version', default=VERSION,
+                      help='(deprecated, this flag has no effect)')
+  parser.add_argument('--fallback',
+                      help='(deprecated, this flag has no effect)')
+
   parser.add_argument('args', nargs=argparse.REMAINDER)
 
   args, extras = parser.parse_known_args()
@@ -182,8 +203,7 @@
 
 def main():
   args = parse_args()
-  return run_gsutil(args.force_version, args.fallback, args.target, args.args,
-                    clean=args.clean)
+  return run_gsutil(args.target, args.args, clean=args.clean)
 
 
 if __name__ == '__main__':