commit 254f33f91cda63072638be2f53884513a21348e3
author Mike Frysinger <vapier@chromium.org> Wed Dec 11 13:54:29 2019 -0500
committer Commit Bot <commit-bot@chromium.org> Thu Dec 12 02:40:32 2019 +0000
tree f7ddc1e6be712dd10d67841ce619a5da57e7ad9a
parent b37d39e8c1fa6f1ece1e7779a1081fdeff473047

gerrit: run many operations in parallel

I don't convert every possible operation here, just the ones that
are trivial.  We can always add more later as needed.

I picked a parallel limit of 10 arbitrarily.  I have no idea if
there are guidelines somewhere that would recommend something else.

BUG=None
TEST=`gerrit verify <180 CLs> 1` is a lot faster

Change-Id: Iafdc8ab0d0f0b80e93f261d2551d6c71f637b333
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/chromite/+/1962718
Reviewed-by: Mike Nichols <mikenichols@chromium.org>
Reviewed-by: Sean Abraham <seanabraham@chromium.org>
Commit-Queue: Mike Frysinger <vapier@chromium.org>
Tested-by: Mike Frysinger <vapier@chromium.org>

scripts/gerrit.py

diff --git a/scripts/gerrit.py b/scripts/gerrit.py
index 179e647..39a38f7 100644
--- a/scripts/gerrit.py
+++ b/scripts/gerrit.py
@@ -24,6 +24,7 @@
 from chromite.lib import cros_logging as logging
 from chromite.lib import gerrit
 from chromite.lib import gob_util
+from chromite.lib import parallel
 from chromite.lib import terminal
 from chromite.lib import uri_lib
 from chromite.utils import memoize
@@ -34,6 +35,12 @@
 ACTION_PREFIX = 'UserAct'
 
 
+# How many connections we'll use in parallel.  We don't want this to be too high
+# so we don't go over our per-user quota.  Pick 10 somewhat arbitrarily as that
+# seems to be good enough for users.
+CONNECTION_LIMIT = 10
+
+
 COLOR = None
 
 # Map the internal names to the ones we normally show on the web ui.
@@ -70,6 +77,13 @@
   return COLOR.Color(terminal.Color.BLUE, s)
 
 
+def _run_parallel_tasks(task, *args):
+  """Small wrapper around BackgroundTaskRunner to enforce job count."""
+  with parallel.BackgroundTaskRunner(task, processes=CONNECTION_LIMIT) as q:
+    for arg in args:
+      q.put([arg])
+
+
 def limits(cls):
   """Given a dict of fields, calculate the longest string lengths
 
@@ -359,10 +373,11 @@
 def UserActAutosubmit(opts, *args):
   """Mark CLs with the Auto-Submit label"""
   num = args[-1]
-  for arg in args[:-1]:
+  def task(arg):
     helper, cl = GetGerrit(opts, arg)
     helper.SetReview(cl, labels={'Auto-Submit': num},
                      dryrun=opts.dryrun, notify=opts.notify)
+  _run_parallel_tasks(task, *args[:-1])
 UserActAutosubmit.arg_min = 2
 UserActAutosubmit.usage = '<CLs...> <0|1>'
 
@@ -370,10 +385,11 @@
 def UserActReview(opts, *args):
   """Mark CLs with a code review status"""
   num = args[-1]
-  for arg in args[:-1]:
+  def task(arg):
     helper, cl = GetGerrit(opts, arg)
     helper.SetReview(cl, labels={'Code-Review': num},
                      dryrun=opts.dryrun, notify=opts.notify)
+  _run_parallel_tasks(task, *args[:-1])
 UserActReview.arg_min = 2
 UserActReview.usage = '<CLs...> <-2|-1|0|1|2>'
 
@@ -381,10 +397,11 @@
 def UserActVerify(opts, *args):
   """Mark CLs with a verified status"""
   num = args[-1]
-  for arg in args[:-1]:
+  def task(arg):
     helper, cl = GetGerrit(opts, arg)
     helper.SetReview(cl, labels={'Verified': num},
                      dryrun=opts.dryrun, notify=opts.notify)
+  _run_parallel_tasks(task, *args[:-1])
 UserActVerify.arg_min = 2
 UserActVerify.usage = '<CLs...> <-1|0|1>'
 
@@ -392,35 +409,39 @@
 def UserActReady(opts, *args):
   """Mark CLs with CQ dryrun (1) or ready (2) status"""
   num = args[-1]
-  for arg in args[:-1]:
+  def task(arg):
     helper, cl = GetGerrit(opts, arg)
     helper.SetReview(cl, labels={'Commit-Queue': num},
                      dryrun=opts.dryrun, notify=opts.notify)
+  _run_parallel_tasks(task, *args[:-1])
 UserActReady.arg_min = 2
 UserActReady.usage = '<CLs...> <0|1|2>'
 
 
 def UserActSubmit(opts, *args):
   """Submit CLs"""
-  for arg in args:
+  def task(arg):
     helper, cl = GetGerrit(opts, arg)
     helper.SubmitChange(cl, dryrun=opts.dryrun)
+  _run_parallel_tasks(task, *args)
 UserActSubmit.usage = '<CLs...>'
 
 
 def UserActAbandon(opts, *args):
   """Abandon CLs"""
-  for arg in args:
+  def task(arg):
     helper, cl = GetGerrit(opts, arg)
     helper.AbandonChange(cl, dryrun=opts.dryrun)
+  _run_parallel_tasks(task, *args)
 UserActAbandon.usage = '<CLs...>'
 
 
 def UserActRestore(opts, *args):
   """Restore CLs that were abandoned"""
-  for arg in args:
+  def task(arg):
     helper, cl = GetGerrit(opts, arg)
     helper.RestoreChange(cl, dryrun=opts.dryrun)
+  _run_parallel_tasks(task, *args)
 UserActRestore.usage = '<CLs...>'
 
 
@@ -466,9 +487,10 @@
 
 def UserActTopic(opts, topic, *args):
   """Set a topic for one or more CLs"""
-  for arg in args:
+  def task(arg):
     helper, arg = GetGerrit(opts, arg)
     helper.SetTopic(arg, topic, dryrun=opts.dryrun)
+  _run_parallel_tasks(task, *args)
 UserActTopic.usage = '<topic> <CLs...>'
 
 
@@ -501,9 +523,10 @@
 
 def UserActDeletedraft(opts, *args):
   """Delete draft CLs"""
-  for arg in args:
+  def task(arg):
     helper, cl = GetGerrit(opts, arg)
     helper.DeleteDraft(cl, dryrun=opts.dryrun)
+  _run_parallel_tasks(task, *args)
 UserActDeletedraft.usage = '<CLs...>'