blob: f0d8cec6f49cb99637af1c11ac8e5e54d5ef8bb1 [file] [log] [blame]
maruel@chromium.org99ac1c52012-01-16 14:52:12 +00001# Copyright (c) 2012 The Chromium Authors. All rights reserved.
maruel@chromium.org06617272010-11-04 13:50:50 +00002# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
maruel@chromium.org5f3eee32009-09-17 00:34:30 +00004
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +00005"""Generic utils."""
6
maruel@chromium.orgdae209f2012-07-03 16:08:15 +00007import codecs
hinoka@google.com267f33e2014-02-28 22:02:32 +00008import cStringIO
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00009import datetime
maruel@chromium.orgd9141bf2009-12-23 16:13:32 +000010import logging
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000011import os
ilevy@chromium.orgc28d3772013-07-12 19:42:37 +000012import pipes
szager@chromium.orgfc616382014-03-18 20:32:04 +000013import platform
maruel@chromium.org3742c842010-09-09 19:27:14 +000014import Queue
msb@chromium.orgac915bb2009-11-13 17:03:01 +000015import re
bradnelson@google.com8f9c69f2009-09-17 00:48:28 +000016import stat
borenet@google.com6b4a2ab2013-04-18 15:50:27 +000017import subprocess
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000018import sys
maruel@chromium.org0e0436a2011-10-25 13:32:41 +000019import tempfile
maruel@chromium.org9e5317a2010-08-13 20:35:11 +000020import threading
maruel@chromium.org167b9e62009-09-17 17:41:02 +000021import time
maruel@chromium.orgeb5edbc2012-01-16 17:03:28 +000022import urlparse
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000023
maruel@chromium.orgca0f8392011-09-08 17:15:15 +000024import subprocess2
25
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000026
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +000027RETRY_MAX = 3
28RETRY_INITIAL_SLEEP = 0.5
szager@chromium.orgfe0d1902014-04-08 20:50:44 +000029START = datetime.datetime.now()
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +000030
31
borenet@google.com6a9b1682014-03-24 18:35:23 +000032_WARNINGS = []
33
34
szager@chromium.orgff113292014-03-25 06:02:08 +000035# These repos are known to cause OOM errors on 32-bit platforms, due the the
36# very large objects they contain. It is not safe to use threaded index-pack
37# when cloning/fetching them.
38THREADED_INDEX_PACK_BLACKLIST = [
39 'https://chromium.googlesource.com/chromium/reference_builds/chrome_win.git'
40]
41
42
maruel@chromium.org66c83e62010-09-07 14:18:45 +000043class Error(Exception):
44 """gclient exception class."""
szager@chromium.org4a3c17e2013-05-24 23:59:29 +000045 def __init__(self, msg, *args, **kwargs):
46 index = getattr(threading.currentThread(), 'index', 0)
47 if index:
48 msg = '\n'.join('%d> %s' % (index, l) for l in msg.splitlines())
49 super(Error, self).__init__(msg, *args, **kwargs)
maruel@chromium.org66c83e62010-09-07 14:18:45 +000050
nick@chromium.org3ac1c4e2014-01-16 02:44:42 +000051
szager@chromium.orgfe0d1902014-04-08 20:50:44 +000052def Elapsed(until=None):
53 if until is None:
54 until = datetime.datetime.now()
55 return str(until - START).partition('.')[0]
56
57
borenet@google.com6a9b1682014-03-24 18:35:23 +000058def PrintWarnings():
59 """Prints any accumulated warnings."""
60 if _WARNINGS:
61 print >> sys.stderr, '\n\nWarnings:'
62 for warning in _WARNINGS:
63 print >> sys.stderr, warning
64
65
66def AddWarning(msg):
67 """Adds the given warning message to the list of accumulated warnings."""
68 _WARNINGS.append(msg)
69
70
msb@chromium.orgac915bb2009-11-13 17:03:01 +000071def SplitUrlRevision(url):
72 """Splits url and returns a two-tuple: url, rev"""
73 if url.startswith('ssh:'):
maruel@chromium.org78b8cd12010-10-26 12:47:07 +000074 # Make sure ssh://user-name@example.com/~/test.git@stable works
kangil.han@samsung.com71b13572013-10-16 17:28:11 +000075 regex = r'(ssh://(?:[-.\w]+@)?[-\w:\.]+/[-~\w\./]+)(?:@(.+))?'
msb@chromium.orgac915bb2009-11-13 17:03:01 +000076 components = re.search(regex, url).groups()
77 else:
scr@chromium.orgf1eccaf2014-04-11 15:51:33 +000078 components = url.rsplit('@', 1)
79 if re.match(r'^\w+\@', url) and '@' not in components[0]:
80 components = [url]
81
msb@chromium.orgac915bb2009-11-13 17:03:01 +000082 if len(components) == 1:
83 components += [None]
84 return tuple(components)
85
86
primiano@chromium.org5439ea52014-08-06 17:18:18 +000087def IsGitSha(revision):
88 """Returns true if the given string is a valid hex-encoded sha"""
89 return re.match('^[a-fA-F0-9]{6,40}$', revision) is not None
90
91
floitsch@google.comeaab7842011-04-28 09:07:58 +000092def IsDateRevision(revision):
93 """Returns true if the given revision is of the form "{ ... }"."""
94 return bool(revision and re.match(r'^\{.+\}$', str(revision)))
95
96
97def MakeDateRevision(date):
98 """Returns a revision representing the latest revision before the given
99 date."""
100 return "{" + date + "}"
101
102
maruel@chromium.org5990f9d2010-07-07 18:02:58 +0000103def SyntaxErrorToError(filename, e):
104 """Raises a gclient_utils.Error exception with the human readable message"""
105 try:
106 # Try to construct a human readable error message
107 if filename:
108 error_message = 'There is a syntax error in %s\n' % filename
109 else:
110 error_message = 'There is a syntax error\n'
111 error_message += 'Line #%s, character %s: "%s"' % (
112 e.lineno, e.offset, re.sub(r'[\r\n]*$', '', e.text))
113 except:
114 # Something went wrong, re-raise the original exception
115 raise e
116 else:
117 raise Error(error_message)
118
119
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000120class PrintableObject(object):
121 def __str__(self):
122 output = ''
123 for i in dir(self):
124 if i.startswith('__'):
125 continue
126 output += '%s = %s\n' % (i, str(getattr(self, i, '')))
127 return output
128
129
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000130def FileRead(filename, mode='rU'):
maruel@chromium.org51e84fb2012-07-03 23:06:21 +0000131 with open(filename, mode=mode) as f:
maruel@chromium.orgc3cd5372012-07-11 17:39:24 +0000132 # codecs.open() has different behavior than open() on python 2.6 so use
133 # open() and decode manually.
chrisha@chromium.org2b99d432012-07-12 18:10:28 +0000134 s = f.read()
135 try:
136 return s.decode('utf-8')
137 except UnicodeDecodeError:
138 return s
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000139
140
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000141def FileWrite(filename, content, mode='w'):
maruel@chromium.orgdae209f2012-07-03 16:08:15 +0000142 with codecs.open(filename, mode=mode, encoding='utf-8') as f:
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000143 f.write(content)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000144
145
cyrille@nnamrak.orgef509e42013-09-20 13:19:08 +0000146def safe_rename(old, new):
147 """Renames a file reliably.
148
nick@chromium.org3ac1c4e2014-01-16 02:44:42 +0000149 Sometimes os.rename does not work because a dying git process keeps a handle
150 on it for a few seconds. An exception is then thrown, which make the program
cyrille@nnamrak.orgef509e42013-09-20 13:19:08 +0000151 give up what it was doing and remove what was deleted.
nick@chromium.org3ac1c4e2014-01-16 02:44:42 +0000152 The only solution is to catch the exception and try again until it works.
cyrille@nnamrak.orgef509e42013-09-20 13:19:08 +0000153 """
154 # roughly 10s
155 retries = 100
156 for i in range(retries):
157 try:
158 os.rename(old, new)
159 break
160 except OSError:
161 if i == (retries - 1):
162 # Give up.
163 raise
164 # retry
165 logging.debug("Renaming failed from %s to %s. Retrying ..." % (old, new))
166 time.sleep(0.1)
167
168
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000169def rmtree(path):
170 """shutil.rmtree() on steroids.
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000171
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000172 Recursively removes a directory, even if it's marked read-only.
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000173
174 shutil.rmtree() doesn't work on Windows if any of the files or directories
175 are read-only, which svn repositories and some .svn files are. We need to
176 be able to force the files to be writable (i.e., deletable) as we traverse
177 the tree.
178
179 Even with all this, Windows still sometimes fails to delete a file, citing
180 a permission error (maybe something to do with antivirus scans or disk
181 indexing). The best suggestion any of the user forums had was to wait a
182 bit and try again, so we do that too. It's hand-waving, but sometimes it
183 works. :/
184
185 On POSIX systems, things are a little bit simpler. The modes of the files
186 to be deleted doesn't matter, only the modes of the directories containing
187 them are significant. As the directory tree is traversed, each directory
188 has its mode set appropriately before descending into it. This should
189 result in the entire tree being removed, with the possible exception of
190 *path itself, because nothing attempts to change the mode of its parent.
191 Doing so would be hazardous, as it's not a directory slated for removal.
192 In the ordinary case, this is not a problem: for our purposes, the user
193 will never lack write permission on *path's parent.
194 """
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000195 if not os.path.exists(path):
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000196 return
197
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000198 if os.path.islink(path) or not os.path.isdir(path):
199 raise Error('Called rmtree(%s) in non-directory' % path)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000200
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000201 if sys.platform == 'win32':
borenet@google.com6b4a2ab2013-04-18 15:50:27 +0000202 # Give up and use cmd.exe's rd command.
203 path = os.path.normcase(path)
204 for _ in xrange(3):
205 exitcode = subprocess.call(['cmd.exe', '/c', 'rd', '/q', '/s', path])
206 if exitcode == 0:
207 return
208 else:
209 print >> sys.stderr, 'rd exited with code %d' % exitcode
210 time.sleep(3)
211 raise Exception('Failed to remove path %s' % path)
212
213 # On POSIX systems, we need the x-bit set on the directory to access it,
214 # the r-bit to see its contents, and the w-bit to remove files from it.
215 # The actual modes of the files within the directory is irrelevant.
216 os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000217
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000218 def remove(func, subpath):
borenet@google.com6b4a2ab2013-04-18 15:50:27 +0000219 func(subpath)
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000220
221 for fn in os.listdir(path):
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000222 # If fullpath is a symbolic link that points to a directory, isdir will
223 # be True, but we don't want to descend into that as a directory, we just
224 # want to remove the link. Check islink and treat links as ordinary files
225 # would be treated regardless of what they reference.
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000226 fullpath = os.path.join(path, fn)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000227 if os.path.islink(fullpath) or not os.path.isdir(fullpath):
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000228 remove(os.remove, fullpath)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000229 else:
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000230 # Recurse.
231 rmtree(fullpath)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000232
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000233 remove(os.rmdir, path)
234
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000235
maruel@chromium.org6c48a302011-10-20 23:44:20 +0000236def safe_makedirs(tree):
237 """Creates the directory in a safe manner.
238
239 Because multiple threads can create these directories concurently, trap the
240 exception and pass on.
241 """
242 count = 0
243 while not os.path.exists(tree):
244 count += 1
245 try:
246 os.makedirs(tree)
247 except OSError, e:
248 # 17 POSIX, 183 Windows
249 if e.errno not in (17, 183):
250 raise
251 if count > 40:
252 # Give up.
253 raise
254
255
ilevy@chromium.orgc28d3772013-07-12 19:42:37 +0000256def CommandToStr(args):
257 """Converts an arg list into a shell escaped string."""
258 return ' '.join(pipes.quote(arg) for arg in args)
259
260
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +0000261def CheckCallAndFilterAndHeader(args, always=False, header=None, **kwargs):
maruel@chromium.org17d01792010-09-01 18:07:10 +0000262 """Adds 'header' support to CheckCallAndFilter.
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000263
maruel@chromium.org17d01792010-09-01 18:07:10 +0000264 If |always| is True, a message indicating what is being done
265 is printed to stdout all the time even if not output is generated. Otherwise
266 the message header is printed only if the call generated any ouput.
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000267 """
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +0000268 stdout = kwargs.setdefault('stdout', sys.stdout)
269 if header is None:
270 header = "\n________ running '%s' in '%s'\n" % (
ilevy@chromium.org4aad1852013-07-12 21:32:51 +0000271 ' '.join(args), kwargs.get('cwd', '.'))
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +0000272
maruel@chromium.org17d01792010-09-01 18:07:10 +0000273 if always:
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +0000274 stdout.write(header)
maruel@chromium.org17d01792010-09-01 18:07:10 +0000275 else:
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +0000276 filter_fn = kwargs.get('filter_fn')
maruel@chromium.org17d01792010-09-01 18:07:10 +0000277 def filter_msg(line):
278 if line is None:
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +0000279 stdout.write(header)
maruel@chromium.org17d01792010-09-01 18:07:10 +0000280 elif filter_fn:
281 filter_fn(line)
282 kwargs['filter_fn'] = filter_msg
283 kwargs['call_filter_on_first_line'] = True
284 # Obviously.
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +0000285 kwargs.setdefault('print_stdout', True)
maruel@chromium.org17d01792010-09-01 18:07:10 +0000286 return CheckCallAndFilter(args, **kwargs)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000287
maruel@chromium.org17d01792010-09-01 18:07:10 +0000288
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000289class Wrapper(object):
290 """Wraps an object, acting as a transparent proxy for all properties by
291 default.
292 """
293 def __init__(self, wrapped):
294 self._wrapped = wrapped
295
296 def __getattr__(self, name):
297 return getattr(self._wrapped, name)
maruel@chromium.orgdb111f72010-09-08 13:36:53 +0000298
maruel@chromium.orge0de9cb2010-09-17 15:07:14 +0000299
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000300class AutoFlush(Wrapper):
maruel@chromium.orge0de9cb2010-09-17 15:07:14 +0000301 """Creates a file object clone to automatically flush after N seconds."""
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000302 def __init__(self, wrapped, delay):
303 super(AutoFlush, self).__init__(wrapped)
304 if not hasattr(self, 'lock'):
305 self.lock = threading.Lock()
306 self.__last_flushed_at = time.time()
307 self.delay = delay
maruel@chromium.orge0de9cb2010-09-17 15:07:14 +0000308
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000309 @property
310 def autoflush(self):
311 return self
maruel@chromium.orge0de9cb2010-09-17 15:07:14 +0000312
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000313 def write(self, out, *args, **kwargs):
314 self._wrapped.write(out, *args, **kwargs)
maruel@chromium.orgdb111f72010-09-08 13:36:53 +0000315 should_flush = False
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000316 self.lock.acquire()
maruel@chromium.org9c531262010-09-08 13:41:13 +0000317 try:
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000318 if self.delay and (time.time() - self.__last_flushed_at) > self.delay:
maruel@chromium.orgdb111f72010-09-08 13:36:53 +0000319 should_flush = True
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000320 self.__last_flushed_at = time.time()
maruel@chromium.org9c531262010-09-08 13:41:13 +0000321 finally:
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000322 self.lock.release()
maruel@chromium.orgdb111f72010-09-08 13:36:53 +0000323 if should_flush:
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000324 self.flush()
maruel@chromium.orgdb111f72010-09-08 13:36:53 +0000325
326
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000327class Annotated(Wrapper):
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000328 """Creates a file object clone to automatically prepends every line in worker
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000329 threads with a NN> prefix.
330 """
331 def __init__(self, wrapped, include_zero=False):
332 super(Annotated, self).__init__(wrapped)
333 if not hasattr(self, 'lock'):
334 self.lock = threading.Lock()
335 self.__output_buffers = {}
336 self.__include_zero = include_zero
maruel@chromium.orgcb1e97a2010-09-09 20:09:20 +0000337
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000338 @property
339 def annotated(self):
340 return self
maruel@chromium.orgcb1e97a2010-09-09 20:09:20 +0000341
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000342 def write(self, out):
343 index = getattr(threading.currentThread(), 'index', 0)
344 if not index and not self.__include_zero:
345 # Unindexed threads aren't buffered.
346 return self._wrapped.write(out)
maruel@chromium.orgcb1e97a2010-09-09 20:09:20 +0000347
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000348 self.lock.acquire()
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000349 try:
350 # Use a dummy array to hold the string so the code can be lockless.
351 # Strings are immutable, requiring to keep a lock for the whole dictionary
352 # otherwise. Using an array is faster than using a dummy object.
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000353 if not index in self.__output_buffers:
354 obj = self.__output_buffers[index] = ['']
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000355 else:
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000356 obj = self.__output_buffers[index]
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000357 finally:
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000358 self.lock.release()
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000359
360 # Continue lockless.
361 obj[0] += out
362 while '\n' in obj[0]:
363 line, remaining = obj[0].split('\n', 1)
nsylvain@google.come939bb52011-06-01 22:59:15 +0000364 if line:
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000365 self._wrapped.write('%d>%s\n' % (index, line))
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000366 obj[0] = remaining
367
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000368 def flush(self):
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000369 """Flush buffered output."""
370 orphans = []
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000371 self.lock.acquire()
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000372 try:
373 # Detect threads no longer existing.
374 indexes = (getattr(t, 'index', None) for t in threading.enumerate())
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000375 indexes = filter(None, indexes)
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000376 for index in self.__output_buffers:
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000377 if not index in indexes:
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000378 orphans.append((index, self.__output_buffers[index][0]))
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000379 for orphan in orphans:
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000380 del self.__output_buffers[orphan[0]]
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000381 finally:
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000382 self.lock.release()
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000383
384 # Don't keep the lock while writting. Will append \n when it shouldn't.
385 for orphan in orphans:
nsylvain@google.come939bb52011-06-01 22:59:15 +0000386 if orphan[1]:
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000387 self._wrapped.write('%d>%s\n' % (orphan[0], orphan[1]))
388 return self._wrapped.flush()
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000389
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000390
391def MakeFileAutoFlush(fileobj, delay=10):
392 autoflush = getattr(fileobj, 'autoflush', None)
393 if autoflush:
394 autoflush.delay = delay
395 return fileobj
396 return AutoFlush(fileobj, delay)
397
398
399def MakeFileAnnotated(fileobj, include_zero=False):
400 if getattr(fileobj, 'annotated', None):
401 return fileobj
402 return Annotated(fileobj)
maruel@chromium.orgcb1e97a2010-09-09 20:09:20 +0000403
404
xusydoc@chromium.org2fd6c3f2013-05-03 21:57:55 +0000405GCLIENT_CHILDREN = []
406GCLIENT_CHILDREN_LOCK = threading.Lock()
407
408
409class GClientChildren(object):
410 @staticmethod
411 def add(popen_obj):
412 with GCLIENT_CHILDREN_LOCK:
413 GCLIENT_CHILDREN.append(popen_obj)
414
415 @staticmethod
416 def remove(popen_obj):
417 with GCLIENT_CHILDREN_LOCK:
418 GCLIENT_CHILDREN.remove(popen_obj)
419
420 @staticmethod
421 def _attemptToKillChildren():
422 global GCLIENT_CHILDREN
423 with GCLIENT_CHILDREN_LOCK:
424 zombies = [c for c in GCLIENT_CHILDREN if c.poll() is None]
425
426 for zombie in zombies:
427 try:
428 zombie.kill()
429 except OSError:
430 pass
431
432 with GCLIENT_CHILDREN_LOCK:
433 GCLIENT_CHILDREN = [k for k in GCLIENT_CHILDREN if k.poll() is not None]
434
435 @staticmethod
436 def _areZombies():
437 with GCLIENT_CHILDREN_LOCK:
438 return bool(GCLIENT_CHILDREN)
439
440 @staticmethod
441 def KillAllRemainingChildren():
442 GClientChildren._attemptToKillChildren()
443
444 if GClientChildren._areZombies():
445 time.sleep(0.5)
446 GClientChildren._attemptToKillChildren()
447
448 with GCLIENT_CHILDREN_LOCK:
449 if GCLIENT_CHILDREN:
450 print >> sys.stderr, 'Could not kill the following subprocesses:'
451 for zombie in GCLIENT_CHILDREN:
452 print >> sys.stderr, ' ', zombie.pid
453
454
maruel@chromium.org17d01792010-09-01 18:07:10 +0000455def CheckCallAndFilter(args, stdout=None, filter_fn=None,
456 print_stdout=None, call_filter_on_first_line=False,
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +0000457 retry=False, **kwargs):
maruel@chromium.org17d01792010-09-01 18:07:10 +0000458 """Runs a command and calls back a filter function if needed.
459
maruel@chromium.org57bf78d2011-09-08 18:57:33 +0000460 Accepts all subprocess2.Popen() parameters plus:
maruel@chromium.org17d01792010-09-01 18:07:10 +0000461 print_stdout: If True, the command's stdout is forwarded to stdout.
462 filter_fn: A function taking a single string argument called with each line
maruel@chromium.org57bf78d2011-09-08 18:57:33 +0000463 of the subprocess2's output. Each line has the trailing newline
maruel@chromium.org17d01792010-09-01 18:07:10 +0000464 character trimmed.
465 stdout: Can be any bufferable output.
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +0000466 retry: If the process exits non-zero, sleep for a brief interval and try
467 again, up to RETRY_MAX times.
maruel@chromium.org17d01792010-09-01 18:07:10 +0000468
469 stderr is always redirected to stdout.
470 """
471 assert print_stdout or filter_fn
472 stdout = stdout or sys.stdout
hinoka@google.com267f33e2014-02-28 22:02:32 +0000473 output = cStringIO.StringIO()
maruel@chromium.org17d01792010-09-01 18:07:10 +0000474 filter_fn = filter_fn or (lambda x: None)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000475
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +0000476 sleep_interval = RETRY_INITIAL_SLEEP
477 run_cwd = kwargs.get('cwd', os.getcwd())
478 for _ in xrange(RETRY_MAX + 1):
479 kid = subprocess2.Popen(
480 args, bufsize=0, stdout=subprocess2.PIPE, stderr=subprocess2.STDOUT,
481 **kwargs)
xusydoc@chromium.org2fd6c3f2013-05-03 21:57:55 +0000482
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +0000483 GClientChildren.add(kid)
chase@chromium.org8ad1cee2010-08-16 19:12:27 +0000484
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +0000485 # Do a flush of stdout before we begin reading from the subprocess2's stdout
486 stdout.flush()
487
488 # Also, we need to forward stdout to prevent weird re-ordering of output.
489 # This has to be done on a per byte basis to make sure it is not buffered:
490 # normally buffering is done for each line, but if svn requests input, no
491 # end-of-line character is output after the prompt and it would not show up.
492 try:
493 in_byte = kid.stdout.read(1)
494 if in_byte:
495 if call_filter_on_first_line:
496 filter_fn(None)
497 in_line = ''
498 while in_byte:
hinoka@google.com267f33e2014-02-28 22:02:32 +0000499 output.write(in_byte)
500 if print_stdout:
501 stdout.write(in_byte)
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000502 if in_byte not in ['\r', '\n']:
503 in_line += in_byte
maruel@chromium.org109cb9d2011-09-14 20:03:11 +0000504 else:
505 filter_fn(in_line)
506 in_line = ''
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +0000507 in_byte = kid.stdout.read(1)
508 # Flush the rest of buffered output. This is only an issue with
509 # stdout/stderr not ending with a \n.
510 if len(in_line):
szager@google.com85d3e3a2011-10-07 17:12:00 +0000511 filter_fn(in_line)
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +0000512 rv = kid.wait()
xusydoc@chromium.org2fd6c3f2013-05-03 21:57:55 +0000513
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +0000514 # Don't put this in a 'finally,' since the child may still run if we get
515 # an exception.
516 GClientChildren.remove(kid)
xusydoc@chromium.org2fd6c3f2013-05-03 21:57:55 +0000517
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +0000518 except KeyboardInterrupt:
519 print >> sys.stderr, 'Failed while running "%s"' % ' '.join(args)
520 raise
maruel@chromium.org109cb9d2011-09-14 20:03:11 +0000521
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +0000522 if rv == 0:
hinoka@google.com267f33e2014-02-28 22:02:32 +0000523 return output.getvalue()
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +0000524 if not retry:
525 break
526 print ("WARNING: subprocess '%s' in %s failed; will retry after a short "
527 'nap...' % (' '.join('"%s"' % x for x in args), run_cwd))
raphael.kubo.da.costa@intel.com91507f72013-10-22 12:18:25 +0000528 time.sleep(sleep_interval)
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +0000529 sleep_interval *= 2
530 raise subprocess2.CalledProcessError(
531 rv, args, kwargs.get('cwd', None), None, None)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000532
533
agable@chromium.org5a306a22014-02-24 22:13:59 +0000534class GitFilter(object):
535 """A filter_fn implementation for quieting down git output messages.
536
537 Allows a custom function to skip certain lines (predicate), and will throttle
538 the output of percentage completed lines to only output every X seconds.
539 """
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000540 PERCENT_RE = re.compile('(.*) ([0-9]{1,3})% .*')
agable@chromium.org5a306a22014-02-24 22:13:59 +0000541
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000542 def __init__(self, time_throttle=0, predicate=None, out_fh=None):
agable@chromium.org5a306a22014-02-24 22:13:59 +0000543 """
544 Args:
545 time_throttle (int): GitFilter will throttle 'noisy' output (such as the
546 XX% complete messages) to only be printed at least |time_throttle|
547 seconds apart.
548 predicate (f(line)): An optional function which is invoked for every line.
549 The line will be skipped if predicate(line) returns False.
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000550 out_fh: File handle to write output to.
agable@chromium.org5a306a22014-02-24 22:13:59 +0000551 """
552 self.last_time = 0
553 self.time_throttle = time_throttle
554 self.predicate = predicate
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000555 self.out_fh = out_fh or sys.stdout
556 self.progress_prefix = None
agable@chromium.org5a306a22014-02-24 22:13:59 +0000557
558 def __call__(self, line):
559 # git uses an escape sequence to clear the line; elide it.
560 esc = line.find(unichr(033))
561 if esc > -1:
562 line = line[:esc]
563 if self.predicate and not self.predicate(line):
564 return
565 now = time.time()
566 match = self.PERCENT_RE.match(line)
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000567 if match:
568 if match.group(1) != self.progress_prefix:
569 self.progress_prefix = match.group(1)
570 elif now - self.last_time < self.time_throttle:
571 return
572 self.last_time = now
573 self.out_fh.write('[%s] ' % Elapsed())
574 print >> self.out_fh, line
agable@chromium.org5a306a22014-02-24 22:13:59 +0000575
576
maruel@chromium.org9eda4112010-06-11 18:56:10 +0000577def FindGclientRoot(from_dir, filename='.gclient'):
maruel@chromium.orga9371762009-12-22 18:27:38 +0000578 """Tries to find the gclient root."""
jochen@chromium.org20760a52010-09-08 08:47:28 +0000579 real_from_dir = os.path.realpath(from_dir)
580 path = real_from_dir
maruel@chromium.org9eda4112010-06-11 18:56:10 +0000581 while not os.path.exists(os.path.join(path, filename)):
maruel@chromium.org3a292682010-08-23 18:54:55 +0000582 split_path = os.path.split(path)
583 if not split_path[1]:
maruel@chromium.orga9371762009-12-22 18:27:38 +0000584 return None
maruel@chromium.org3a292682010-08-23 18:54:55 +0000585 path = split_path[0]
jochen@chromium.org20760a52010-09-08 08:47:28 +0000586
587 # If we did not find the file in the current directory, make sure we are in a
588 # sub directory that is controlled by this configuration.
589 if path != real_from_dir:
590 entries_filename = os.path.join(path, filename + '_entries')
591 if not os.path.exists(entries_filename):
592 # If .gclient_entries does not exist, a previous call to gclient sync
593 # might have failed. In that case, we cannot verify that the .gclient
594 # is the one we want to use. In order to not to cause too much trouble,
595 # just issue a warning and return the path anyway.
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000596 print >> sys.stderr, ("%s file in parent directory %s might not be the "
jochen@chromium.org20760a52010-09-08 08:47:28 +0000597 "file you want to use" % (filename, path))
598 return path
599 scope = {}
600 try:
601 exec(FileRead(entries_filename), scope)
602 except SyntaxError, e:
603 SyntaxErrorToError(filename, e)
604 all_directories = scope['entries'].keys()
605 path_to_check = real_from_dir[len(path)+1:]
606 while path_to_check:
607 if path_to_check in all_directories:
608 return path
609 path_to_check = os.path.dirname(path_to_check)
610 return None
maruel@chromium.org3742c842010-09-09 19:27:14 +0000611
maruel@chromium.orgd9141bf2009-12-23 16:13:32 +0000612 logging.info('Found gclient root at ' + path)
maruel@chromium.orga9371762009-12-22 18:27:38 +0000613 return path
maruel@chromium.org3ccbf7e2009-12-22 20:46:42 +0000614
maruel@chromium.org9eda4112010-06-11 18:56:10 +0000615
maruel@chromium.org3ccbf7e2009-12-22 20:46:42 +0000616def PathDifference(root, subpath):
617 """Returns the difference subpath minus root."""
618 root = os.path.realpath(root)
619 subpath = os.path.realpath(subpath)
620 if not subpath.startswith(root):
621 return None
622 # If the root does not have a trailing \ or /, we add it so the returned
623 # path starts immediately after the seperator regardless of whether it is
624 # provided.
625 root = os.path.join(root, '')
626 return subpath[len(root):]
piman@chromium.orgf43d0192010-04-15 02:36:04 +0000627
628
629def FindFileUpwards(filename, path=None):
rcui@google.com13595ff2011-10-13 01:25:07 +0000630 """Search upwards from the a directory (default: current) to find a file.
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +0000631
rcui@google.com13595ff2011-10-13 01:25:07 +0000632 Returns nearest upper-level directory with the passed in file.
633 """
piman@chromium.orgf43d0192010-04-15 02:36:04 +0000634 if not path:
635 path = os.getcwd()
636 path = os.path.realpath(path)
637 while True:
638 file_path = os.path.join(path, filename)
rcui@google.com13595ff2011-10-13 01:25:07 +0000639 if os.path.exists(file_path):
640 return path
piman@chromium.orgf43d0192010-04-15 02:36:04 +0000641 (new_path, _) = os.path.split(path)
642 if new_path == path:
643 return None
644 path = new_path
645
646
nick@chromium.org3ac1c4e2014-01-16 02:44:42 +0000647def GetMacWinOrLinux():
648 """Returns 'mac', 'win', or 'linux', matching the current platform."""
649 if sys.platform.startswith(('cygwin', 'win')):
650 return 'win'
651 elif sys.platform.startswith('linux'):
652 return 'linux'
653 elif sys.platform == 'darwin':
654 return 'mac'
655 raise Error('Unknown platform: ' + sys.platform)
656
657
brettw@chromium.orgcc968fe2014-06-23 17:30:32 +0000658def GetBuildtoolsPath():
659 """Returns the full path to the buildtools directory.
660 This is based on the root of the checkout containing the current directory."""
661 gclient_root = FindGclientRoot(os.getcwd())
662 if not gclient_root:
jochen@chromium.orgaaee92f2014-07-02 07:35:31 +0000663 # Some projects might not use .gclient. Try to see whether we're in a git
664 # checkout.
665 top_dir = [os.getcwd()]
666 def filter_fn(line):
667 top_dir[0] = os.path.normpath(line.rstrip('\n'))
668 try:
669 CheckCallAndFilter(["git", "rev-parse", "--show-toplevel"],
670 print_stdout=False, filter_fn=filter_fn)
671 except Exception:
672 pass
673 top_dir = top_dir[0]
674 if os.path.exists(os.path.join(top_dir, 'buildtools')):
675 return os.path.join(top_dir, 'buildtools')
brettw@chromium.orgcc968fe2014-06-23 17:30:32 +0000676 return None
677 return os.path.join(gclient_root, 'src', 'buildtools')
678
679
680def GetBuildtoolsPlatformBinaryPath():
681 """Returns the full path to the binary directory for the current platform."""
682 # Mac and Windows just have one directory, Linux has two according to whether
683 # it's 32 or 64 bits.
684 buildtools_path = GetBuildtoolsPath()
685 if not buildtools_path:
686 return None
687
688 if sys.platform.startswith(('cygwin', 'win')):
689 subdir = 'win'
690 elif sys.platform == 'darwin':
691 subdir = 'mac'
692 elif sys.platform.startswith('linux'):
693 if sys.maxsize > 2**32:
694 subdir = 'linux64'
695 else:
696 subdir = 'linux32'
697 else:
698 raise Error('Unknown platform: ' + sys.platform)
699 return os.path.join(buildtools_path, subdir)
700
701
nick@chromium.org3ac1c4e2014-01-16 02:44:42 +0000702def GetExeSuffix():
703 """Returns '' or '.exe' depending on how executables work on this platform."""
704 if sys.platform.startswith(('cygwin', 'win')):
705 return '.exe'
706 return ''
707
708
piman@chromium.orgf43d0192010-04-15 02:36:04 +0000709def GetGClientRootAndEntries(path=None):
710 """Returns the gclient root and the dict of entries."""
711 config_file = '.gclient_entries'
maruel@chromium.org93a9ee02011-10-18 18:23:58 +0000712 root = FindFileUpwards(config_file, path)
713 if not root:
maruel@chromium.org116704f2010-06-11 17:34:38 +0000714 print "Can't find %s" % config_file
piman@chromium.orgf43d0192010-04-15 02:36:04 +0000715 return None
maruel@chromium.org93a9ee02011-10-18 18:23:58 +0000716 config_path = os.path.join(root, config_file)
piman@chromium.orgf43d0192010-04-15 02:36:04 +0000717 env = {}
718 execfile(config_path, env)
719 config_dir = os.path.dirname(config_path)
720 return config_dir, env['entries']
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000721
722
maruel@chromium.org6ca8bf82011-09-19 23:04:30 +0000723def lockedmethod(method):
724 """Method decorator that holds self.lock for the duration of the call."""
725 def inner(self, *args, **kwargs):
726 try:
727 try:
728 self.lock.acquire()
729 except KeyboardInterrupt:
730 print >> sys.stderr, 'Was deadlocked'
731 raise
732 return method(self, *args, **kwargs)
733 finally:
734 self.lock.release()
735 return inner
736
737
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000738class WorkItem(object):
739 """One work item."""
maruel@chromium.org4901daf2011-10-20 14:34:47 +0000740 # On cygwin, creating a lock throwing randomly when nearing ~100 locks.
741 # As a workaround, use a single lock. Yep you read it right. Single lock for
742 # all the 100 objects.
743 lock = threading.Lock()
744
maruel@chromium.org6ca8bf82011-09-19 23:04:30 +0000745 def __init__(self, name):
maruel@chromium.org485dcab2011-09-14 12:48:47 +0000746 # A unique string representing this work item.
maruel@chromium.org6ca8bf82011-09-19 23:04:30 +0000747 self._name = name
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000748 self.outbuf = cStringIO.StringIO()
749 self.start = self.finish = None
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000750
maruel@chromium.org77e4eca2010-09-21 13:23:07 +0000751 def run(self, work_queue):
752 """work_queue is passed as keyword argument so it should be
maruel@chromium.org3742c842010-09-09 19:27:14 +0000753 the last parameters of the function when you override it."""
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000754 pass
755
maruel@chromium.org6ca8bf82011-09-19 23:04:30 +0000756 @property
757 def name(self):
758 return self._name
759
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000760
761class ExecutionQueue(object):
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000762 """Runs a set of WorkItem that have interdependencies and were WorkItem are
763 added as they are processed.
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000764
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000765 In gclient's case, Dependencies sometime needs to be run out of order due to
766 From() keyword. This class manages that all the required dependencies are run
767 before running each one.
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000768
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000769 Methods of this class are thread safe.
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000770 """
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000771 def __init__(self, jobs, progress, ignore_requirements, verbose=False):
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000772 """jobs specifies the number of concurrent tasks to allow. progress is a
773 Progress instance."""
774 # Set when a thread is done or a new item is enqueued.
775 self.ready_cond = threading.Condition()
776 # Maximum number of concurrent tasks.
777 self.jobs = jobs
778 # List of WorkItem, for gclient, these are Dependency instances.
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000779 self.queued = []
780 # List of strings representing each Dependency.name that was run.
781 self.ran = []
782 # List of items currently running.
783 self.running = []
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000784 # Exceptions thrown if any.
maruel@chromium.org3742c842010-09-09 19:27:14 +0000785 self.exceptions = Queue.Queue()
786 # Progress status
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000787 self.progress = progress
788 if self.progress:
maruel@chromium.org3742c842010-09-09 19:27:14 +0000789 self.progress.update(0)
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000790
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +0000791 self.ignore_requirements = ignore_requirements
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000792 self.verbose = verbose
793 self.last_join = None
794 self.last_subproc_output = None
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +0000795
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000796 def enqueue(self, d):
797 """Enqueue one Dependency to be executed later once its requirements are
798 satisfied.
799 """
800 assert isinstance(d, WorkItem)
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000801 self.ready_cond.acquire()
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000802 try:
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000803 self.queued.append(d)
804 total = len(self.queued) + len(self.ran) + len(self.running)
szager@chromium.orge98e04c2014-07-25 00:28:06 +0000805 if self.jobs == 1:
806 total += 1
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000807 logging.debug('enqueued(%s)' % d.name)
808 if self.progress:
szager@chromium.orge98e04c2014-07-25 00:28:06 +0000809 self.progress._total = total
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000810 self.progress.update(0)
811 self.ready_cond.notifyAll()
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000812 finally:
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000813 self.ready_cond.release()
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000814
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000815 def out_cb(self, _):
816 self.last_subproc_output = datetime.datetime.now()
817 return True
818
819 @staticmethod
820 def format_task_output(task, comment=''):
821 if comment:
822 comment = ' (%s)' % comment
823 if task.start and task.finish:
824 elapsed = ' (Elapsed: %s)' % (
825 str(task.finish - task.start).partition('.')[0])
826 else:
827 elapsed = ''
828 return """
829%s%s%s
830----------------------------------------
831%s
832----------------------------------------""" % (
szager@chromium.org1f4e71b2014-04-09 19:45:40 +0000833 task.name, comment, elapsed, task.outbuf.getvalue().strip())
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000834
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000835 def flush(self, *args, **kwargs):
836 """Runs all enqueued items until all are executed."""
maruel@chromium.org3742c842010-09-09 19:27:14 +0000837 kwargs['work_queue'] = self
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000838 self.last_subproc_output = self.last_join = datetime.datetime.now()
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000839 self.ready_cond.acquire()
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000840 try:
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000841 while True:
842 # Check for task to run first, then wait.
843 while True:
maruel@chromium.org3742c842010-09-09 19:27:14 +0000844 if not self.exceptions.empty():
845 # Systematically flush the queue when an exception logged.
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000846 self.queued = []
maruel@chromium.org3742c842010-09-09 19:27:14 +0000847 self._flush_terminated_threads()
848 if (not self.queued and not self.running or
849 self.jobs == len(self.running)):
maruel@chromium.org1333cb32011-10-04 23:40:16 +0000850 logging.debug('No more worker threads or can\'t queue anything.')
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000851 break
maruel@chromium.org3742c842010-09-09 19:27:14 +0000852
853 # Check for new tasks to start.
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000854 for i in xrange(len(self.queued)):
855 # Verify its requirements.
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +0000856 if (self.ignore_requirements or
857 not (set(self.queued[i].requirements) - set(self.ran))):
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000858 # Start one work item: all its requirements are satisfied.
maruel@chromium.org3742c842010-09-09 19:27:14 +0000859 self._run_one_task(self.queued.pop(i), args, kwargs)
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000860 break
861 else:
862 # Couldn't find an item that could run. Break out the outher loop.
863 break
maruel@chromium.org3742c842010-09-09 19:27:14 +0000864
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000865 if not self.queued and not self.running:
maruel@chromium.org3742c842010-09-09 19:27:14 +0000866 # We're done.
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000867 break
868 # We need to poll here otherwise Ctrl-C isn't processed.
maruel@chromium.org485dcab2011-09-14 12:48:47 +0000869 try:
870 self.ready_cond.wait(10)
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000871 # If we haven't printed to terminal for a while, but we have received
872 # spew from a suprocess, let the user know we're still progressing.
873 now = datetime.datetime.now()
874 if (now - self.last_join > datetime.timedelta(seconds=60) and
875 self.last_subproc_output > self.last_join):
876 if self.progress:
877 print >> sys.stdout, ''
hinoka@google.com4dfb8662014-04-25 22:21:24 +0000878 sys.stdout.flush()
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000879 elapsed = Elapsed()
880 print >> sys.stdout, '[%s] Still working on:' % elapsed
hinoka@google.com4dfb8662014-04-25 22:21:24 +0000881 sys.stdout.flush()
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000882 for task in self.running:
883 print >> sys.stdout, '[%s] %s' % (elapsed, task.item.name)
hinoka@google.com4dfb8662014-04-25 22:21:24 +0000884 sys.stdout.flush()
maruel@chromium.org485dcab2011-09-14 12:48:47 +0000885 except KeyboardInterrupt:
886 # Help debugging by printing some information:
887 print >> sys.stderr, (
888 ('\nAllowed parallel jobs: %d\n# queued: %d\nRan: %s\n'
889 'Running: %d') % (
890 self.jobs,
891 len(self.queued),
892 ', '.join(self.ran),
893 len(self.running)))
894 for i in self.queued:
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000895 print >> sys.stderr, '%s (not started): %s' % (
896 i.name, ', '.join(i.requirements))
897 for i in self.running:
898 print >> sys.stderr, self.format_task_output(i.item, 'interrupted')
maruel@chromium.org485dcab2011-09-14 12:48:47 +0000899 raise
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000900 # Something happened: self.enqueue() or a thread terminated. Loop again.
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000901 finally:
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000902 self.ready_cond.release()
maruel@chromium.org3742c842010-09-09 19:27:14 +0000903
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000904 assert not self.running, 'Now guaranteed to be single-threaded'
maruel@chromium.org3742c842010-09-09 19:27:14 +0000905 if not self.exceptions.empty():
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000906 if self.progress:
907 print >> sys.stdout, ''
maruel@chromium.orgc8d064b2010-08-16 16:46:14 +0000908 # To get back the stack location correctly, the raise a, b, c form must be
909 # used, passing a tuple as the first argument doesn't work.
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000910 e, task = self.exceptions.get()
911 print >> sys.stderr, self.format_task_output(task.item, 'ERROR')
maruel@chromium.orgc8d064b2010-08-16 16:46:14 +0000912 raise e[0], e[1], e[2]
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000913 elif self.progress:
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000914 self.progress.end()
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000915
maruel@chromium.org3742c842010-09-09 19:27:14 +0000916 def _flush_terminated_threads(self):
917 """Flush threads that have terminated."""
918 running = self.running
919 self.running = []
920 for t in running:
921 if t.isAlive():
922 self.running.append(t)
923 else:
924 t.join()
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000925 self.last_join = datetime.datetime.now()
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000926 sys.stdout.flush()
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000927 if self.verbose:
928 print >> sys.stdout, self.format_task_output(t.item)
maruel@chromium.org3742c842010-09-09 19:27:14 +0000929 if self.progress:
maruel@chromium.org55a2eb82010-10-06 23:35:18 +0000930 self.progress.update(1, t.item.name)
maruel@chromium.orgf36c0ee2011-09-14 19:16:47 +0000931 if t.item.name in self.ran:
932 raise Error(
933 'gclient is confused, "%s" is already in "%s"' % (
934 t.item.name, ', '.join(self.ran)))
maruel@chromium.orgacc45672010-09-09 21:21:21 +0000935 if not t.item.name in self.ran:
936 self.ran.append(t.item.name)
maruel@chromium.org3742c842010-09-09 19:27:14 +0000937
938 def _run_one_task(self, task_item, args, kwargs):
939 if self.jobs > 1:
940 # Start the thread.
941 index = len(self.ran) + len(self.running) + 1
maruel@chromium.org77e4eca2010-09-21 13:23:07 +0000942 new_thread = self._Worker(task_item, index, args, kwargs)
maruel@chromium.org3742c842010-09-09 19:27:14 +0000943 self.running.append(new_thread)
944 new_thread.start()
945 else:
946 # Run the 'thread' inside the main thread. Don't try to catch any
947 # exception.
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000948 try:
949 task_item.start = datetime.datetime.now()
950 print >> task_item.outbuf, '[%s] Started.' % Elapsed(task_item.start)
951 task_item.run(*args, **kwargs)
952 task_item.finish = datetime.datetime.now()
953 print >> task_item.outbuf, '[%s] Finished.' % Elapsed(task_item.finish)
954 self.ran.append(task_item.name)
955 if self.verbose:
956 if self.progress:
957 print >> sys.stdout, ''
958 print >> sys.stdout, self.format_task_output(task_item)
959 if self.progress:
960 self.progress.update(1, ', '.join(t.item.name for t in self.running))
961 except KeyboardInterrupt:
962 print >> sys.stderr, self.format_task_output(task_item, 'interrupted')
963 raise
964 except Exception:
965 print >> sys.stderr, self.format_task_output(task_item, 'ERROR')
966 raise
967
maruel@chromium.org3742c842010-09-09 19:27:14 +0000968
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000969 class _Worker(threading.Thread):
970 """One thread to execute one WorkItem."""
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000971 def __init__(self, item, index, args, kwargs):
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000972 threading.Thread.__init__(self, name=item.name or 'Worker')
maruel@chromium.org1333cb32011-10-04 23:40:16 +0000973 logging.info('_Worker(%s) reqs:%s' % (item.name, item.requirements))
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000974 self.item = item
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000975 self.index = index
maruel@chromium.org3742c842010-09-09 19:27:14 +0000976 self.args = args
977 self.kwargs = kwargs
xusydoc@chromium.org2fd6c3f2013-05-03 21:57:55 +0000978 self.daemon = True
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000979
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000980 def run(self):
981 """Runs in its own thread."""
maruel@chromium.org1333cb32011-10-04 23:40:16 +0000982 logging.debug('_Worker.run(%s)' % self.item.name)
maruel@chromium.org3742c842010-09-09 19:27:14 +0000983 work_queue = self.kwargs['work_queue']
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000984 try:
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000985 self.item.start = datetime.datetime.now()
986 print >> self.item.outbuf, '[%s] Started.' % Elapsed(self.item.start)
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000987 self.item.run(*self.args, **self.kwargs)
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000988 self.item.finish = datetime.datetime.now()
989 print >> self.item.outbuf, '[%s] Finished.' % Elapsed(self.item.finish)
xusydoc@chromium.org2fd6c3f2013-05-03 21:57:55 +0000990 except KeyboardInterrupt:
xusydoc@chromium.orgc144e062013-05-03 23:23:53 +0000991 logging.info('Caught KeyboardInterrupt in thread %s', self.item.name)
xusydoc@chromium.org2fd6c3f2013-05-03 21:57:55 +0000992 logging.info(str(sys.exc_info()))
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000993 work_queue.exceptions.put((sys.exc_info(), self))
xusydoc@chromium.org2fd6c3f2013-05-03 21:57:55 +0000994 raise
maruel@chromium.orgc8d064b2010-08-16 16:46:14 +0000995 except Exception:
996 # Catch exception location.
xusydoc@chromium.orgc144e062013-05-03 23:23:53 +0000997 logging.info('Caught exception in thread %s', self.item.name)
maruel@chromium.org3742c842010-09-09 19:27:14 +0000998 logging.info(str(sys.exc_info()))
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000999 work_queue.exceptions.put((sys.exc_info(), self))
maruel@chromium.org9e5317a2010-08-13 20:35:11 +00001000 finally:
xusydoc@chromium.orgc144e062013-05-03 23:23:53 +00001001 logging.info('_Worker.run(%s) done', self.item.name)
xusydoc@chromium.org2fd6c3f2013-05-03 21:57:55 +00001002 work_queue.ready_cond.acquire()
1003 try:
1004 work_queue.ready_cond.notifyAll()
1005 finally:
1006 work_queue.ready_cond.release()
maruel@chromium.org0e0436a2011-10-25 13:32:41 +00001007
1008
jbroman@chromium.org615a2622013-05-03 13:20:14 +00001009def GetEditor(git, git_editor=None):
1010 """Returns the most plausible editor to use.
1011
1012 In order of preference:
1013 - GIT_EDITOR/SVN_EDITOR environment variable
1014 - core.editor git configuration variable (if supplied by git-cl)
1015 - VISUAL environment variable
1016 - EDITOR environment variable
bratell@opera.com65621c72013-12-09 15:05:32 +00001017 - vi (non-Windows) or notepad (Windows)
jbroman@chromium.org615a2622013-05-03 13:20:14 +00001018
1019 In the case of git-cl, this matches git's behaviour, except that it does not
1020 include dumb terminal detection.
1021
1022 In the case of gcl, this matches svn's behaviour, except that it does not
1023 accept a command-line flag or check the editor-cmd configuration variable.
1024 """
maruel@chromium.org0e0436a2011-10-25 13:32:41 +00001025 if git:
jbroman@chromium.org615a2622013-05-03 13:20:14 +00001026 editor = os.environ.get('GIT_EDITOR') or git_editor
maruel@chromium.org0e0436a2011-10-25 13:32:41 +00001027 else:
1028 editor = os.environ.get('SVN_EDITOR')
1029 if not editor:
jbroman@chromium.org615a2622013-05-03 13:20:14 +00001030 editor = os.environ.get('VISUAL')
1031 if not editor:
maruel@chromium.org0e0436a2011-10-25 13:32:41 +00001032 editor = os.environ.get('EDITOR')
1033 if not editor:
1034 if sys.platform.startswith('win'):
1035 editor = 'notepad'
1036 else:
bratell@opera.com65621c72013-12-09 15:05:32 +00001037 editor = 'vi'
maruel@chromium.org0e0436a2011-10-25 13:32:41 +00001038 return editor
1039
1040
jbroman@chromium.org615a2622013-05-03 13:20:14 +00001041def RunEditor(content, git, git_editor=None):
maruel@chromium.org0e0436a2011-10-25 13:32:41 +00001042 """Opens up the default editor in the system to get the CL description."""
maruel@chromium.orgcbd760f2013-07-23 13:02:48 +00001043 file_handle, filename = tempfile.mkstemp(text=True, prefix='cl_description')
maruel@chromium.org0e0436a2011-10-25 13:32:41 +00001044 # Make sure CRLF is handled properly by requiring none.
1045 if '\r' in content:
asvitkine@chromium.org0eff22d2011-10-25 16:11:16 +00001046 print >> sys.stderr, (
1047 '!! Please remove \\r from your change description !!')
maruel@chromium.org0e0436a2011-10-25 13:32:41 +00001048 fileobj = os.fdopen(file_handle, 'w')
1049 # Still remove \r if present.
1050 fileobj.write(re.sub('\r?\n', '\n', content))
1051 fileobj.close()
1052
1053 try:
jbroman@chromium.org615a2622013-05-03 13:20:14 +00001054 editor = GetEditor(git, git_editor=git_editor)
1055 if not editor:
1056 return None
1057 cmd = '%s %s' % (editor, filename)
maruel@chromium.org0e0436a2011-10-25 13:32:41 +00001058 if sys.platform == 'win32' and os.environ.get('TERM') == 'msys':
1059 # Msysgit requires the usage of 'env' to be present.
1060 cmd = 'env ' + cmd
1061 try:
1062 # shell=True to allow the shell to handle all forms of quotes in
1063 # $EDITOR.
1064 subprocess2.check_call(cmd, shell=True)
1065 except subprocess2.CalledProcessError:
1066 return None
1067 return FileRead(filename)
1068 finally:
1069 os.remove(filename)
maruel@chromium.org99ac1c52012-01-16 14:52:12 +00001070
1071
maruel@chromium.orgeb5edbc2012-01-16 17:03:28 +00001072def UpgradeToHttps(url):
1073 """Upgrades random urls to https://.
1074
1075 Do not touch unknown urls like ssh:// or git://.
1076 Do not touch http:// urls with a port number,
1077 Fixes invalid GAE url.
1078 """
1079 if not url:
1080 return url
1081 if not re.match(r'[a-z\-]+\://.*', url):
1082 # Make sure it is a valid uri. Otherwise, urlparse() will consider it a
1083 # relative url and will use http:///foo. Note that it defaults to http://
1084 # for compatibility with naked url like "localhost:8080".
1085 url = 'http://%s' % url
1086 parsed = list(urlparse.urlparse(url))
1087 # Do not automatically upgrade http to https if a port number is provided.
1088 if parsed[0] == 'http' and not re.match(r'^.+?\:\d+$', parsed[1]):
1089 parsed[0] = 'https'
maruel@chromium.orgeb5edbc2012-01-16 17:03:28 +00001090 return urlparse.urlunparse(parsed)
1091
1092
maruel@chromium.org99ac1c52012-01-16 14:52:12 +00001093def ParseCodereviewSettingsContent(content):
1094 """Process a codereview.settings file properly."""
1095 lines = (l for l in content.splitlines() if not l.strip().startswith("#"))
1096 try:
1097 keyvals = dict([x.strip() for x in l.split(':', 1)] for l in lines if l)
1098 except ValueError:
1099 raise Error(
1100 'Failed to process settings, please fix. Content:\n\n%s' % content)
maruel@chromium.orgeb5edbc2012-01-16 17:03:28 +00001101 def fix_url(key):
1102 if keyvals.get(key):
1103 keyvals[key] = UpgradeToHttps(keyvals[key])
1104 fix_url('CODE_REVIEW_SERVER')
1105 fix_url('VIEW_VC')
maruel@chromium.org99ac1c52012-01-16 14:52:12 +00001106 return keyvals
ilevy@chromium.org13691502012-10-16 04:26:37 +00001107
1108
1109def NumLocalCpus():
1110 """Returns the number of processors.
1111
1112 Python on OSX 10.6 raises a NotImplementedError exception.
1113 """
1114 try:
1115 import multiprocessing
1116 return multiprocessing.cpu_count()
1117 except: # pylint: disable=W0702
1118 # Mac OS 10.6 only
1119 # pylint: disable=E1101
1120 return int(os.sysconf('SC_NPROCESSORS_ONLN'))
szager@chromium.orgfc616382014-03-18 20:32:04 +00001121
1122def DefaultDeltaBaseCacheLimit():
1123 """Return a reasonable default for the git config core.deltaBaseCacheLimit.
1124
1125 The primary constraint is the address space of virtual memory. The cache
1126 size limit is per-thread, and 32-bit systems can hit OOM errors if this
1127 parameter is set too high.
1128 """
1129 if platform.architecture()[0].startswith('64'):
1130 return '2g'
1131 else:
1132 return '512m'
1133
szager@chromium.orgff113292014-03-25 06:02:08 +00001134def DefaultIndexPackConfig(url=''):
szager@chromium.orgfc616382014-03-18 20:32:04 +00001135 """Return reasonable default values for configuring git-index-pack.
1136
1137 Experiments suggest that higher values for pack.threads don't improve
1138 performance."""
szager@chromium.orgff113292014-03-25 06:02:08 +00001139 cache_limit = DefaultDeltaBaseCacheLimit()
1140 result = ['-c', 'core.deltaBaseCacheLimit=%s' % cache_limit]
1141 if url in THREADED_INDEX_PACK_BLACKLIST:
1142 result.extend(['-c', 'pack.threads=1'])
1143 return result