Reland "Limit number of concurrent gsutil cp calls"
This is a reland of d3cfea0c78442f0d8e68f8e96d35efceaa782325
Original change's description:
> Limit number of concurrent gsutil cp calls
>
> gsutil utilizes as many cores as possible, and will create many threads
> too. In combination with gclient doing the same thing (utilize as many
> cores as possible), this results in creating way too many processes and
> some builders are running out of resources (e.g. can't create threads).
>
> This limits number of gsutil cp commands in git_cache to two.
>
> R=apolito@google.com, dpranke@google.com
>
> Bug: 1255228, 1249003
> Change-Id: Ifcccaacde6c9e1c60c1da0a0bc5a3e512e158dcc
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/3219900
> Reviewed-by: Dirk Pranke <dpranke@google.com>
> Commit-Queue: Josip Sokcevic <sokcevic@google.com>
Bug: 1255228, 1249003
Change-Id: I712bf5f625fd42992e7b5310ae005fad2633e1a4
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/3223155
Reviewed-by: Anthony Polito <apolito@google.com>
Commit-Queue: Josip Sokcevic <sokcevic@google.com>
diff --git a/git_cache.py b/git_cache.py
index 82a3108..786546c 100755
--- a/git_cache.py
+++ b/git_cache.py
@@ -35,6 +35,12 @@
GIT_CACHE_CORRUPT_MESSAGE = 'WARNING: The Git cache is corrupt.'
+# gsutil creates many processes and threads. Creating too many gsutil cp
+# processes may result in running out of resources, and may perform worse due to
+# contextr switching. This limits how many concurrent gsutil cp processes
+# git_cache runs.
+GSUTIL_CP_SEMAPHORE = threading.Semaphore(2)
+
try:
# pylint: disable=undefined-variable
WinErr = WindowsError
@@ -306,8 +312,9 @@
self.print('Downloading files in %s/* into %s.' %
(latest_dir, tempdir))
with self.print_duration_of('download'):
- code = gsutil.call('-m', 'cp', '-r', latest_dir + "/*",
- tempdir)
+ with GSUTIL_CP_SEMAPHORE:
+ code = gsutil.call('-m', 'cp', '-r', latest_dir + "/*",
+ tempdir)
if code:
return False
# Set HEAD to main.