Added 'git-retry' bootstrap
Added the 'git-retry' bootstrap command. This can be used to wrap other 'git'
commands around a fault-tolerant retry wrapper.
BUG=295109
TEST=localtest
R=iannucci@chromium.org, petermayo@chromium.org
Review URL: https://codereview.chromium.org/401673003
git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@285939 0039d316-1c4b-4281-b951-d872f2087c98
diff --git a/git_retry.py b/git_retry.py
new file mode 100755
index 0000000..b40e6d2
--- /dev/null
+++ b/git_retry.py
@@ -0,0 +1,156 @@
+#!/usr/bin/env python
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import logging
+import optparse
+import subprocess
+import sys
+import threading
+import time
+
+from git_common import GIT_EXE, GIT_TRANSIENT_ERRORS_RE
+
+
+class TeeThread(threading.Thread):
+
+ def __init__(self, fd, out_fd, name):
+ super(TeeThread, self).__init__(name='git-retry.tee.%s' % (name,))
+ self.data = None
+ self.fd = fd
+ self.out_fd = out_fd
+
+ def run(self):
+ chunks = []
+ for line in self.fd:
+ chunks.append(line)
+ self.out_fd.write(line)
+ self.data = ''.join(chunks)
+
+
+class GitRetry(object):
+
+ logger = logging.getLogger('git-retry')
+ DEFAULT_DELAY_SECS = 3.0
+ DEFAULT_RETRY_COUNT = 5
+
+ def __init__(self, retry_count=None, delay=None, delay_factor=None):
+ self.retry_count = retry_count or self.DEFAULT_RETRY_COUNT
+ self.delay = max(delay, 0) if delay else 0
+ self.delay_factor = max(delay_factor, 0) if delay_factor else 0
+
+ def shouldRetry(self, stderr):
+ m = GIT_TRANSIENT_ERRORS_RE.search(stderr)
+ if not m:
+ return False
+ self.logger.info("Encountered known transient error: [%s]",
+ stderr[m.start(): m.end()])
+ return True
+
+ @staticmethod
+ def execute(*args):
+ args = (GIT_EXE,) + args
+ proc = subprocess.Popen(
+ args,
+ stderr=subprocess.PIPE,
+ )
+ stderr_tee = TeeThread(proc.stderr, sys.stderr, 'stderr')
+
+ # Start our process. Collect/tee 'stdout' and 'stderr'.
+ stderr_tee.start()
+ try:
+ proc.wait()
+ except KeyboardInterrupt:
+ proc.kill()
+ raise
+ finally:
+ stderr_tee.join()
+ return proc.returncode, None, stderr_tee.data
+
+ def computeDelay(self, iteration):
+ """Returns: the delay (in seconds) for a given iteration
+
+ The first iteration has a delay of '0'.
+
+ Args:
+ iteration: (int) The iteration index (starting with zero as the first
+ iteration)
+ """
+ if (not self.delay) or (iteration == 0):
+ return 0
+ if self.delay_factor == 0:
+ # Linear delay
+ return iteration * self.delay
+ # Exponential delay
+ return (self.delay_factor ** (iteration - 1)) * self.delay
+
+ def __call__(self, *args):
+ returncode = 0
+ for i in xrange(self.retry_count):
+ # If the previous run failed and a delay is configured, delay before the
+ # next run.
+ delay = self.computeDelay(i)
+ if delay > 0:
+ self.logger.info("Delaying for [%s second(s)] until next retry", delay)
+ time.sleep(delay)
+
+ self.logger.debug("Executing subprocess (%d/%d) with arguments: %s",
+ (i+1), self.retry_count, args)
+ returncode, _, stderr = self.execute(*args)
+
+ self.logger.debug("Process terminated with return code: %d", returncode)
+ if returncode == 0:
+ break
+
+ if not self.shouldRetry(stderr):
+ self.logger.error("Process failure was not known to be transient; "
+ "terminating with return code %d", returncode)
+ break
+ return returncode
+
+
+def main(args):
+ parser = optparse.OptionParser()
+ parser.disable_interspersed_args()
+ parser.add_option('-v', '--verbose',
+ action='count', default=0,
+ help="Increase verbosity; can be specified multiple times")
+ parser.add_option('-c', '--retry-count', metavar='COUNT',
+ type=int, default=GitRetry.DEFAULT_RETRY_COUNT,
+ help="Number of times to retry (default=%default)")
+ parser.add_option('-d', '--delay', metavar='SECONDS',
+ type=float, default=GitRetry.DEFAULT_DELAY_SECS,
+ help="Specifies the amount of time (in seconds) to wait "
+ "between successive retries (default=%default). This "
+ "can be zero.")
+ parser.add_option('-D', '--delay-factor', metavar='FACTOR',
+ type=int, default=2,
+ help="The exponential factor to apply to delays in between "
+ "successive failures (default=%default). If this is "
+ "zero, delays will increase linearly. Set this to "
+ "one to have a constant (non-increasing) delay.")
+
+ opts, args = parser.parse_args(args)
+
+ # Configure logging verbosity
+ if opts.verbose == 0:
+ logging.getLogger().setLevel(logging.WARNING)
+ elif opts.verbose == 1:
+ logging.getLogger().setLevel(logging.INFO)
+ else:
+ logging.getLogger().setLevel(logging.DEBUG)
+
+ # Execute retries
+ retry = GitRetry(
+ retry_count=opts.retry_count,
+ delay=opts.delay,
+ delay_factor=opts.delay_factor,
+ )
+ return retry(*args)
+
+
+if __name__ == '__main__':
+ logging.basicConfig()
+ logging.getLogger().setLevel(logging.WARNING)
+ sys.exit(main(sys.argv[2:]))