blob: f9cf30ca2af1c23e8c9eff092c392980f2cf70a6 [file] [log] [blame]
maruel@chromium.orgca0f8392011-09-08 17:15:15 +00001# Copyright (c) 2011 The Chromium Authors. All rights reserved.
maruel@chromium.org06617272010-11-04 13:50:50 +00002# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
maruel@chromium.org5f3eee32009-09-17 00:34:30 +00004
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +00005"""Generic utils."""
6
maruel@chromium.org167b9e62009-09-17 17:41:02 +00007import errno
maruel@chromium.orgd9141bf2009-12-23 16:13:32 +00008import logging
maruel@chromium.org5f3eee32009-09-17 00:34:30 +00009import os
maruel@chromium.org3742c842010-09-09 19:27:14 +000010import Queue
msb@chromium.orgac915bb2009-11-13 17:03:01 +000011import re
bradnelson@google.com8f9c69f2009-09-17 00:48:28 +000012import stat
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000013import subprocess
14import sys
maruel@chromium.org9e5317a2010-08-13 20:35:11 +000015import threading
maruel@chromium.org167b9e62009-09-17 17:41:02 +000016import time
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000017
maruel@chromium.orgca0f8392011-09-08 17:15:15 +000018import subprocess2
19
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000020
maruel@chromium.org58ef2972011-04-01 21:00:11 +000021def hack_subprocess():
22 """subprocess functions may throw exceptions when used in multiple threads.
23
24 See http://bugs.python.org/issue1731717 for more information.
25 """
26 subprocess._cleanup = lambda: None
maruel@chromium.org06617272010-11-04 13:50:50 +000027
28
maruel@chromium.org66c83e62010-09-07 14:18:45 +000029class Error(Exception):
30 """gclient exception class."""
31 pass
32
33
maruel@chromium.orgca0f8392011-09-08 17:15:15 +000034class CheckCallError(subprocess2.CalledProcessError, Error):
maruel@chromium.org9a2f37e2009-12-19 16:03:28 +000035 """CheckCall() returned non-0."""
maruel@chromium.orgca0f8392011-09-08 17:15:15 +000036 def __init__(self, cmd, cwd, returncode, stdout, stderr=None):
37 subprocess2.CalledProcessError.__init__(
38 self, returncode, cmd, cwd, stdout, stderr)
39 Error.__init__(self, cmd)
maruel@chromium.org9a2f37e2009-12-19 16:03:28 +000040
maruel@chromium.org7b194c12010-09-07 20:57:09 +000041 def __str__(self):
maruel@chromium.orgca0f8392011-09-08 17:15:15 +000042 return subprocess2.CalledProcessError.__str__(self)
maruel@chromium.org7b194c12010-09-07 20:57:09 +000043
maruel@chromium.org9a2f37e2009-12-19 16:03:28 +000044
maruel@chromium.org58ef2972011-04-01 21:00:11 +000045def Popen(args, **kwargs):
46 """Calls subprocess.Popen() with hacks to work around certain behaviors.
47
48 Ensure English outpout for svn and make it work reliably on Windows.
49 """
50 logging.debug(u'%s, cwd=%s' % (u' '.join(args), kwargs.get('cwd', '')))
51 if not 'env' in kwargs:
52 # It's easier to parse the stdout if it is always in English.
53 kwargs['env'] = os.environ.copy()
54 kwargs['env']['LANGUAGE'] = 'en'
55 if not 'shell' in kwargs:
56 # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for the
57 # executable, but shell=True makes subprocess on Linux fail when it's called
58 # with a list because it only tries to execute the first item in the list.
59 kwargs['shell'] = (sys.platform=='win32')
60 try:
61 return subprocess.Popen(args, **kwargs)
62 except OSError, e:
63 if e.errno == errno.EAGAIN and sys.platform == 'cygwin':
64 raise Error(
65 'Visit '
66 'http://code.google.com/p/chromium/wiki/CygwinDllRemappingFailure to '
67 'learn how to fix this error; you need to rebase your cygwin dlls')
68 raise
69
70
maruel@chromium.orgac610232010-10-13 14:01:31 +000071def CheckCall(command, print_error=True, **kwargs):
maruel@chromium.org3a292682010-08-23 18:54:55 +000072 """Similar subprocess.check_call() but redirects stdout and
73 returns (stdout, stderr).
maruel@chromium.org9a2f37e2009-12-19 16:03:28 +000074
75 Works on python 2.4
76 """
maruel@chromium.org18111352009-12-20 17:21:28 +000077 try:
maruel@chromium.orgea8c1a92009-12-20 17:21:59 +000078 stderr = None
79 if not print_error:
80 stderr = subprocess.PIPE
maruel@chromium.orgac610232010-10-13 14:01:31 +000081 process = Popen(command, stdout=subprocess.PIPE, stderr=stderr, **kwargs)
maruel@chromium.org7be5ef22010-01-30 22:31:50 +000082 std_out, std_err = process.communicate()
maruel@chromium.org18111352009-12-20 17:21:28 +000083 except OSError, e:
maruel@chromium.orgac610232010-10-13 14:01:31 +000084 raise CheckCallError(command, kwargs.get('cwd', None), e.errno, None)
maruel@chromium.org18111352009-12-20 17:21:28 +000085 if process.returncode:
maruel@chromium.orgac610232010-10-13 14:01:31 +000086 raise CheckCallError(command, kwargs.get('cwd', None), process.returncode,
87 std_out, std_err)
maruel@chromium.org7be5ef22010-01-30 22:31:50 +000088 return std_out, std_err
maruel@chromium.org9a2f37e2009-12-19 16:03:28 +000089
90
msb@chromium.orgac915bb2009-11-13 17:03:01 +000091def SplitUrlRevision(url):
92 """Splits url and returns a two-tuple: url, rev"""
93 if url.startswith('ssh:'):
maruel@chromium.org78b8cd12010-10-26 12:47:07 +000094 # Make sure ssh://user-name@example.com/~/test.git@stable works
95 regex = r'(ssh://(?:[-\w]+@)?[-\w:\.]+/[-~\w\./]+)(?:@(.+))?'
msb@chromium.orgac915bb2009-11-13 17:03:01 +000096 components = re.search(regex, url).groups()
97 else:
maruel@chromium.org116704f2010-06-11 17:34:38 +000098 components = url.split('@', 1)
msb@chromium.orgac915bb2009-11-13 17:03:01 +000099 if len(components) == 1:
100 components += [None]
101 return tuple(components)
102
103
floitsch@google.comeaab7842011-04-28 09:07:58 +0000104def IsDateRevision(revision):
105 """Returns true if the given revision is of the form "{ ... }"."""
106 return bool(revision and re.match(r'^\{.+\}$', str(revision)))
107
108
109def MakeDateRevision(date):
110 """Returns a revision representing the latest revision before the given
111 date."""
112 return "{" + date + "}"
113
114
maruel@chromium.org5990f9d2010-07-07 18:02:58 +0000115def SyntaxErrorToError(filename, e):
116 """Raises a gclient_utils.Error exception with the human readable message"""
117 try:
118 # Try to construct a human readable error message
119 if filename:
120 error_message = 'There is a syntax error in %s\n' % filename
121 else:
122 error_message = 'There is a syntax error\n'
123 error_message += 'Line #%s, character %s: "%s"' % (
124 e.lineno, e.offset, re.sub(r'[\r\n]*$', '', e.text))
125 except:
126 # Something went wrong, re-raise the original exception
127 raise e
128 else:
129 raise Error(error_message)
130
131
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000132class PrintableObject(object):
133 def __str__(self):
134 output = ''
135 for i in dir(self):
136 if i.startswith('__'):
137 continue
138 output += '%s = %s\n' % (i, str(getattr(self, i, '')))
139 return output
140
141
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000142def FileRead(filename, mode='rU'):
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000143 content = None
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000144 f = open(filename, mode)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000145 try:
146 content = f.read()
147 finally:
148 f.close()
149 return content
150
151
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000152def FileWrite(filename, content, mode='w'):
153 f = open(filename, mode)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000154 try:
155 f.write(content)
156 finally:
157 f.close()
158
159
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000160def rmtree(path):
161 """shutil.rmtree() on steroids.
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000162
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000163 Recursively removes a directory, even if it's marked read-only.
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000164
165 shutil.rmtree() doesn't work on Windows if any of the files or directories
166 are read-only, which svn repositories and some .svn files are. We need to
167 be able to force the files to be writable (i.e., deletable) as we traverse
168 the tree.
169
170 Even with all this, Windows still sometimes fails to delete a file, citing
171 a permission error (maybe something to do with antivirus scans or disk
172 indexing). The best suggestion any of the user forums had was to wait a
173 bit and try again, so we do that too. It's hand-waving, but sometimes it
174 works. :/
175
176 On POSIX systems, things are a little bit simpler. The modes of the files
177 to be deleted doesn't matter, only the modes of the directories containing
178 them are significant. As the directory tree is traversed, each directory
179 has its mode set appropriately before descending into it. This should
180 result in the entire tree being removed, with the possible exception of
181 *path itself, because nothing attempts to change the mode of its parent.
182 Doing so would be hazardous, as it's not a directory slated for removal.
183 In the ordinary case, this is not a problem: for our purposes, the user
184 will never lack write permission on *path's parent.
185 """
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000186 if not os.path.exists(path):
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000187 return
188
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000189 if os.path.islink(path) or not os.path.isdir(path):
190 raise Error('Called rmtree(%s) in non-directory' % path)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000191
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000192 if sys.platform == 'win32':
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000193 # Some people don't have the APIs installed. In that case we'll do without.
maruel@chromium.org1edee692011-03-12 19:39:13 +0000194 win32api = None
195 win32con = None
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000196 try:
maruel@chromium.org1edee692011-03-12 19:39:13 +0000197 # Unable to import 'XX'
198 # pylint: disable=F0401
199 import win32api, win32con
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000200 except ImportError:
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000201 pass
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000202 else:
203 # On POSIX systems, we need the x-bit set on the directory to access it,
204 # the r-bit to see its contents, and the w-bit to remove files from it.
205 # The actual modes of the files within the directory is irrelevant.
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000206 os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000207
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000208 def remove(func, subpath):
209 if sys.platform == 'win32':
210 os.chmod(subpath, stat.S_IWRITE)
211 if win32api and win32con:
212 win32api.SetFileAttributes(subpath, win32con.FILE_ATTRIBUTE_NORMAL)
213 try:
214 func(subpath)
215 except OSError, e:
216 if e.errno != errno.EACCES or sys.platform != 'win32':
217 raise
218 # Failed to delete, try again after a 100ms sleep.
219 time.sleep(0.1)
220 func(subpath)
221
222 for fn in os.listdir(path):
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000223 # If fullpath is a symbolic link that points to a directory, isdir will
224 # be True, but we don't want to descend into that as a directory, we just
225 # want to remove the link. Check islink and treat links as ordinary files
226 # would be treated regardless of what they reference.
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000227 fullpath = os.path.join(path, fn)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000228 if os.path.islink(fullpath) or not os.path.isdir(fullpath):
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000229 remove(os.remove, fullpath)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000230 else:
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000231 # Recurse.
232 rmtree(fullpath)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000233
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000234 remove(os.rmdir, path)
235
236# TODO(maruel): Rename the references.
237RemoveDirectory = rmtree
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000238
239
maruel@chromium.org17d01792010-09-01 18:07:10 +0000240def CheckCallAndFilterAndHeader(args, always=False, **kwargs):
241 """Adds 'header' support to CheckCallAndFilter.
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000242
maruel@chromium.org17d01792010-09-01 18:07:10 +0000243 If |always| is True, a message indicating what is being done
244 is printed to stdout all the time even if not output is generated. Otherwise
245 the message header is printed only if the call generated any ouput.
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000246 """
maruel@chromium.org17d01792010-09-01 18:07:10 +0000247 stdout = kwargs.get('stdout', None) or sys.stdout
248 if always:
maruel@chromium.org559c3f82010-08-23 19:26:08 +0000249 stdout.write('\n________ running \'%s\' in \'%s\'\n'
maruel@chromium.org17d01792010-09-01 18:07:10 +0000250 % (' '.join(args), kwargs.get('cwd', '.')))
251 else:
252 filter_fn = kwargs.get('filter_fn', None)
253 def filter_msg(line):
254 if line is None:
255 stdout.write('\n________ running \'%s\' in \'%s\'\n'
256 % (' '.join(args), kwargs.get('cwd', '.')))
257 elif filter_fn:
258 filter_fn(line)
259 kwargs['filter_fn'] = filter_msg
260 kwargs['call_filter_on_first_line'] = True
261 # Obviously.
262 kwargs['print_stdout'] = True
263 return CheckCallAndFilter(args, **kwargs)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000264
maruel@chromium.org17d01792010-09-01 18:07:10 +0000265
maruel@chromium.orge0de9cb2010-09-17 15:07:14 +0000266def SoftClone(obj):
267 """Clones an object. copy.copy() doesn't work on 'file' objects."""
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000268 if obj.__class__.__name__ == 'SoftCloned':
269 return obj
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000270 class SoftCloned(object):
271 pass
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000272 new_obj = SoftCloned()
maruel@chromium.orge0de9cb2010-09-17 15:07:14 +0000273 for member in dir(obj):
274 if member.startswith('_'):
275 continue
276 setattr(new_obj, member, getattr(obj, member))
277 return new_obj
maruel@chromium.orgdb111f72010-09-08 13:36:53 +0000278
maruel@chromium.orge0de9cb2010-09-17 15:07:14 +0000279
280def MakeFileAutoFlush(fileobj, delay=10):
281 """Creates a file object clone to automatically flush after N seconds."""
282 if hasattr(fileobj, 'last_flushed_at'):
283 # Already patched. Just update delay.
284 fileobj.delay = delay
285 return fileobj
286
maruel@chromium.orgb17b55b2010-11-03 14:42:37 +0000287 # Attribute 'XXX' defined outside __init__
288 # pylint: disable=W0201
maruel@chromium.orge0de9cb2010-09-17 15:07:14 +0000289 new_fileobj = SoftClone(fileobj)
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000290 if not hasattr(new_fileobj, 'lock'):
291 new_fileobj.lock = threading.Lock()
maruel@chromium.orge0de9cb2010-09-17 15:07:14 +0000292 new_fileobj.last_flushed_at = time.time()
293 new_fileobj.delay = delay
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000294 new_fileobj.old_auto_flush_write = new_fileobj.write
maruel@chromium.orge0de9cb2010-09-17 15:07:14 +0000295 # Silence pylint.
296 new_fileobj.flush = fileobj.flush
297
298 def auto_flush_write(out):
299 new_fileobj.old_auto_flush_write(out)
maruel@chromium.orgdb111f72010-09-08 13:36:53 +0000300 should_flush = False
maruel@chromium.orge0de9cb2010-09-17 15:07:14 +0000301 new_fileobj.lock.acquire()
maruel@chromium.org9c531262010-09-08 13:41:13 +0000302 try:
maruel@chromium.orge0de9cb2010-09-17 15:07:14 +0000303 if (new_fileobj.delay and
304 (time.time() - new_fileobj.last_flushed_at) > new_fileobj.delay):
maruel@chromium.orgdb111f72010-09-08 13:36:53 +0000305 should_flush = True
maruel@chromium.orge0de9cb2010-09-17 15:07:14 +0000306 new_fileobj.last_flushed_at = time.time()
maruel@chromium.org9c531262010-09-08 13:41:13 +0000307 finally:
maruel@chromium.orge0de9cb2010-09-17 15:07:14 +0000308 new_fileobj.lock.release()
maruel@chromium.orgdb111f72010-09-08 13:36:53 +0000309 if should_flush:
maruel@chromium.orge0de9cb2010-09-17 15:07:14 +0000310 new_fileobj.flush()
maruel@chromium.orgdb111f72010-09-08 13:36:53 +0000311
maruel@chromium.orge0de9cb2010-09-17 15:07:14 +0000312 new_fileobj.write = auto_flush_write
313 return new_fileobj
maruel@chromium.orgdb111f72010-09-08 13:36:53 +0000314
315
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000316def MakeFileAnnotated(fileobj):
317 """Creates a file object clone to automatically prepends every line in worker
318 threads with a NN> prefix."""
319 if hasattr(fileobj, 'output_buffers'):
320 # Already patched.
321 return fileobj
maruel@chromium.orgcb1e97a2010-09-09 20:09:20 +0000322
maruel@chromium.orgb17b55b2010-11-03 14:42:37 +0000323 # Attribute 'XXX' defined outside __init__
324 # pylint: disable=W0201
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000325 new_fileobj = SoftClone(fileobj)
326 if not hasattr(new_fileobj, 'lock'):
327 new_fileobj.lock = threading.Lock()
328 new_fileobj.output_buffers = {}
329 new_fileobj.old_annotated_write = new_fileobj.write
maruel@chromium.orgcb1e97a2010-09-09 20:09:20 +0000330
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000331 def annotated_write(out):
332 index = getattr(threading.currentThread(), 'index', None)
333 if index is None:
334 # Undexed threads aren't buffered.
335 new_fileobj.old_annotated_write(out)
336 return
maruel@chromium.orgcb1e97a2010-09-09 20:09:20 +0000337
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000338 new_fileobj.lock.acquire()
339 try:
340 # Use a dummy array to hold the string so the code can be lockless.
341 # Strings are immutable, requiring to keep a lock for the whole dictionary
342 # otherwise. Using an array is faster than using a dummy object.
343 if not index in new_fileobj.output_buffers:
344 obj = new_fileobj.output_buffers[index] = ['']
345 else:
346 obj = new_fileobj.output_buffers[index]
347 finally:
348 new_fileobj.lock.release()
349
350 # Continue lockless.
351 obj[0] += out
352 while '\n' in obj[0]:
353 line, remaining = obj[0].split('\n', 1)
nsylvain@google.come939bb52011-06-01 22:59:15 +0000354 if line:
355 new_fileobj.old_annotated_write('%d>%s\n' % (index, line))
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000356 obj[0] = remaining
357
358 def full_flush():
359 """Flush buffered output."""
360 orphans = []
361 new_fileobj.lock.acquire()
362 try:
363 # Detect threads no longer existing.
364 indexes = (getattr(t, 'index', None) for t in threading.enumerate())
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000365 indexes = filter(None, indexes)
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000366 for index in new_fileobj.output_buffers:
367 if not index in indexes:
368 orphans.append((index, new_fileobj.output_buffers[index][0]))
369 for orphan in orphans:
370 del new_fileobj.output_buffers[orphan[0]]
371 finally:
372 new_fileobj.lock.release()
373
374 # Don't keep the lock while writting. Will append \n when it shouldn't.
375 for orphan in orphans:
nsylvain@google.come939bb52011-06-01 22:59:15 +0000376 if orphan[1]:
377 new_fileobj.old_annotated_write('%d>%s\n' % (orphan[0], orphan[1]))
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000378
379 new_fileobj.write = annotated_write
380 new_fileobj.full_flush = full_flush
381 return new_fileobj
maruel@chromium.orgcb1e97a2010-09-09 20:09:20 +0000382
383
maruel@chromium.org17d01792010-09-01 18:07:10 +0000384def CheckCallAndFilter(args, stdout=None, filter_fn=None,
385 print_stdout=None, call_filter_on_first_line=False,
386 **kwargs):
387 """Runs a command and calls back a filter function if needed.
388
389 Accepts all subprocess.Popen() parameters plus:
390 print_stdout: If True, the command's stdout is forwarded to stdout.
391 filter_fn: A function taking a single string argument called with each line
392 of the subprocess's output. Each line has the trailing newline
393 character trimmed.
394 stdout: Can be any bufferable output.
395
396 stderr is always redirected to stdout.
397 """
398 assert print_stdout or filter_fn
399 stdout = stdout or sys.stdout
400 filter_fn = filter_fn or (lambda x: None)
401 assert not 'stderr' in kwargs
maruel@chromium.org2b9aa8e2010-08-25 20:01:42 +0000402 kid = Popen(args, bufsize=0,
403 stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
404 **kwargs)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000405
maruel@chromium.org17d01792010-09-01 18:07:10 +0000406 # Do a flush of stdout before we begin reading from the subprocess's stdout
maruel@chromium.org559c3f82010-08-23 19:26:08 +0000407 stdout.flush()
chase@chromium.org8ad1cee2010-08-16 19:12:27 +0000408
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000409 # Also, we need to forward stdout to prevent weird re-ordering of output.
410 # This has to be done on a per byte basis to make sure it is not buffered:
411 # normally buffering is done for each line, but if svn requests input, no
412 # end-of-line character is output after the prompt and it would not show up.
413 in_byte = kid.stdout.read(1)
maruel@chromium.org17d01792010-09-01 18:07:10 +0000414 if in_byte:
415 if call_filter_on_first_line:
416 filter_fn(None)
417 in_line = ''
418 while in_byte:
419 if in_byte != '\r':
420 if print_stdout:
421 stdout.write(in_byte)
422 if in_byte != '\n':
423 in_line += in_byte
424 else:
425 filter_fn(in_line)
426 in_line = ''
maruel@chromium.org17d01792010-09-01 18:07:10 +0000427 in_byte = kid.stdout.read(1)
428 # Flush the rest of buffered output. This is only an issue with
429 # stdout/stderr not ending with a \n.
430 if len(in_line):
431 filter_fn(in_line)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000432 rv = kid.wait()
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000433 if rv:
maruel@chromium.org7b194c12010-09-07 20:57:09 +0000434 raise CheckCallError(args, kwargs.get('cwd', None), rv, None)
maruel@chromium.org17d01792010-09-01 18:07:10 +0000435 return 0
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000436
437
maruel@chromium.org9eda4112010-06-11 18:56:10 +0000438def FindGclientRoot(from_dir, filename='.gclient'):
maruel@chromium.orga9371762009-12-22 18:27:38 +0000439 """Tries to find the gclient root."""
jochen@chromium.org20760a52010-09-08 08:47:28 +0000440 real_from_dir = os.path.realpath(from_dir)
441 path = real_from_dir
maruel@chromium.org9eda4112010-06-11 18:56:10 +0000442 while not os.path.exists(os.path.join(path, filename)):
maruel@chromium.org3a292682010-08-23 18:54:55 +0000443 split_path = os.path.split(path)
444 if not split_path[1]:
maruel@chromium.orga9371762009-12-22 18:27:38 +0000445 return None
maruel@chromium.org3a292682010-08-23 18:54:55 +0000446 path = split_path[0]
jochen@chromium.org20760a52010-09-08 08:47:28 +0000447
448 # If we did not find the file in the current directory, make sure we are in a
449 # sub directory that is controlled by this configuration.
450 if path != real_from_dir:
451 entries_filename = os.path.join(path, filename + '_entries')
452 if not os.path.exists(entries_filename):
453 # If .gclient_entries does not exist, a previous call to gclient sync
454 # might have failed. In that case, we cannot verify that the .gclient
455 # is the one we want to use. In order to not to cause too much trouble,
456 # just issue a warning and return the path anyway.
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000457 print >> sys.stderr, ("%s file in parent directory %s might not be the "
jochen@chromium.org20760a52010-09-08 08:47:28 +0000458 "file you want to use" % (filename, path))
459 return path
460 scope = {}
461 try:
462 exec(FileRead(entries_filename), scope)
463 except SyntaxError, e:
464 SyntaxErrorToError(filename, e)
465 all_directories = scope['entries'].keys()
466 path_to_check = real_from_dir[len(path)+1:]
467 while path_to_check:
468 if path_to_check in all_directories:
469 return path
470 path_to_check = os.path.dirname(path_to_check)
471 return None
maruel@chromium.org3742c842010-09-09 19:27:14 +0000472
maruel@chromium.orgd9141bf2009-12-23 16:13:32 +0000473 logging.info('Found gclient root at ' + path)
maruel@chromium.orga9371762009-12-22 18:27:38 +0000474 return path
maruel@chromium.org3ccbf7e2009-12-22 20:46:42 +0000475
maruel@chromium.org9eda4112010-06-11 18:56:10 +0000476
maruel@chromium.org3ccbf7e2009-12-22 20:46:42 +0000477def PathDifference(root, subpath):
478 """Returns the difference subpath minus root."""
479 root = os.path.realpath(root)
480 subpath = os.path.realpath(subpath)
481 if not subpath.startswith(root):
482 return None
483 # If the root does not have a trailing \ or /, we add it so the returned
484 # path starts immediately after the seperator regardless of whether it is
485 # provided.
486 root = os.path.join(root, '')
487 return subpath[len(root):]
piman@chromium.orgf43d0192010-04-15 02:36:04 +0000488
489
490def FindFileUpwards(filename, path=None):
491 """Search upwards from the a directory (default: current) to find a file."""
492 if not path:
493 path = os.getcwd()
494 path = os.path.realpath(path)
495 while True:
496 file_path = os.path.join(path, filename)
497 if os.path.isfile(file_path):
498 return file_path
499 (new_path, _) = os.path.split(path)
500 if new_path == path:
501 return None
502 path = new_path
503
504
505def GetGClientRootAndEntries(path=None):
506 """Returns the gclient root and the dict of entries."""
507 config_file = '.gclient_entries'
508 config_path = FindFileUpwards(config_file, path)
509
510 if not config_path:
maruel@chromium.org116704f2010-06-11 17:34:38 +0000511 print "Can't find %s" % config_file
piman@chromium.orgf43d0192010-04-15 02:36:04 +0000512 return None
513
514 env = {}
515 execfile(config_path, env)
516 config_dir = os.path.dirname(config_path)
517 return config_dir, env['entries']
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000518
519
520class WorkItem(object):
521 """One work item."""
522 # A list of string, each being a WorkItem name.
523 requirements = []
524 # A unique string representing this work item.
525 name = None
526
maruel@chromium.org77e4eca2010-09-21 13:23:07 +0000527 def run(self, work_queue):
528 """work_queue is passed as keyword argument so it should be
maruel@chromium.org3742c842010-09-09 19:27:14 +0000529 the last parameters of the function when you override it."""
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000530 pass
531
532
533class ExecutionQueue(object):
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000534 """Runs a set of WorkItem that have interdependencies and were WorkItem are
535 added as they are processed.
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000536
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000537 In gclient's case, Dependencies sometime needs to be run out of order due to
538 From() keyword. This class manages that all the required dependencies are run
539 before running each one.
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000540
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000541 Methods of this class are thread safe.
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000542 """
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000543 def __init__(self, jobs, progress):
544 """jobs specifies the number of concurrent tasks to allow. progress is a
545 Progress instance."""
maruel@chromium.org58ef2972011-04-01 21:00:11 +0000546 hack_subprocess()
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000547 # Set when a thread is done or a new item is enqueued.
548 self.ready_cond = threading.Condition()
549 # Maximum number of concurrent tasks.
550 self.jobs = jobs
551 # List of WorkItem, for gclient, these are Dependency instances.
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000552 self.queued = []
553 # List of strings representing each Dependency.name that was run.
554 self.ran = []
555 # List of items currently running.
556 self.running = []
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000557 # Exceptions thrown if any.
maruel@chromium.org3742c842010-09-09 19:27:14 +0000558 self.exceptions = Queue.Queue()
559 # Progress status
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000560 self.progress = progress
561 if self.progress:
maruel@chromium.org3742c842010-09-09 19:27:14 +0000562 self.progress.update(0)
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000563
564 def enqueue(self, d):
565 """Enqueue one Dependency to be executed later once its requirements are
566 satisfied.
567 """
568 assert isinstance(d, WorkItem)
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000569 self.ready_cond.acquire()
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000570 try:
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000571 self.queued.append(d)
572 total = len(self.queued) + len(self.ran) + len(self.running)
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000573 logging.debug('enqueued(%s)' % d.name)
574 if self.progress:
575 self.progress._total = total + 1
576 self.progress.update(0)
577 self.ready_cond.notifyAll()
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000578 finally:
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000579 self.ready_cond.release()
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000580
581 def flush(self, *args, **kwargs):
582 """Runs all enqueued items until all are executed."""
maruel@chromium.org3742c842010-09-09 19:27:14 +0000583 kwargs['work_queue'] = self
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000584 self.ready_cond.acquire()
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000585 try:
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000586 while True:
587 # Check for task to run first, then wait.
588 while True:
maruel@chromium.org3742c842010-09-09 19:27:14 +0000589 if not self.exceptions.empty():
590 # Systematically flush the queue when an exception logged.
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000591 self.queued = []
maruel@chromium.org3742c842010-09-09 19:27:14 +0000592 self._flush_terminated_threads()
593 if (not self.queued and not self.running or
594 self.jobs == len(self.running)):
595 # No more worker threads or can't queue anything.
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000596 break
maruel@chromium.org3742c842010-09-09 19:27:14 +0000597
598 # Check for new tasks to start.
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000599 for i in xrange(len(self.queued)):
600 # Verify its requirements.
601 for r in self.queued[i].requirements:
602 if not r in self.ran:
603 # Requirement not met.
604 break
605 else:
606 # Start one work item: all its requirements are satisfied.
maruel@chromium.org3742c842010-09-09 19:27:14 +0000607 self._run_one_task(self.queued.pop(i), args, kwargs)
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000608 break
609 else:
610 # Couldn't find an item that could run. Break out the outher loop.
611 break
maruel@chromium.org3742c842010-09-09 19:27:14 +0000612
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000613 if not self.queued and not self.running:
maruel@chromium.org3742c842010-09-09 19:27:14 +0000614 # We're done.
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000615 break
616 # We need to poll here otherwise Ctrl-C isn't processed.
617 self.ready_cond.wait(10)
618 # Something happened: self.enqueue() or a thread terminated. Loop again.
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000619 finally:
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000620 self.ready_cond.release()
maruel@chromium.org3742c842010-09-09 19:27:14 +0000621
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000622 assert not self.running, 'Now guaranteed to be single-threaded'
maruel@chromium.org3742c842010-09-09 19:27:14 +0000623 if not self.exceptions.empty():
maruel@chromium.orgc8d064b2010-08-16 16:46:14 +0000624 # To get back the stack location correctly, the raise a, b, c form must be
625 # used, passing a tuple as the first argument doesn't work.
maruel@chromium.org3742c842010-09-09 19:27:14 +0000626 e = self.exceptions.get()
maruel@chromium.orgc8d064b2010-08-16 16:46:14 +0000627 raise e[0], e[1], e[2]
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000628 if self.progress:
629 self.progress.end()
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000630
maruel@chromium.org3742c842010-09-09 19:27:14 +0000631 def _flush_terminated_threads(self):
632 """Flush threads that have terminated."""
633 running = self.running
634 self.running = []
635 for t in running:
636 if t.isAlive():
637 self.running.append(t)
638 else:
639 t.join()
dpranke@chromium.org97ae58e2011-03-18 00:29:20 +0000640 sys.stdout.full_flush() # pylint: disable=E1101
maruel@chromium.org3742c842010-09-09 19:27:14 +0000641 if self.progress:
maruel@chromium.org55a2eb82010-10-06 23:35:18 +0000642 self.progress.update(1, t.item.name)
maruel@chromium.orgacc45672010-09-09 21:21:21 +0000643 assert not t.item.name in self.ran
644 if not t.item.name in self.ran:
645 self.ran.append(t.item.name)
maruel@chromium.org3742c842010-09-09 19:27:14 +0000646
647 def _run_one_task(self, task_item, args, kwargs):
648 if self.jobs > 1:
649 # Start the thread.
650 index = len(self.ran) + len(self.running) + 1
maruel@chromium.org77e4eca2010-09-21 13:23:07 +0000651 new_thread = self._Worker(task_item, index, args, kwargs)
maruel@chromium.org3742c842010-09-09 19:27:14 +0000652 self.running.append(new_thread)
653 new_thread.start()
654 else:
655 # Run the 'thread' inside the main thread. Don't try to catch any
656 # exception.
657 task_item.run(*args, **kwargs)
658 self.ran.append(task_item.name)
659 if self.progress:
maruel@chromium.org55a2eb82010-10-06 23:35:18 +0000660 self.progress.update(1, ', '.join(t.item.name for t in self.running))
maruel@chromium.org3742c842010-09-09 19:27:14 +0000661
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000662 class _Worker(threading.Thread):
663 """One thread to execute one WorkItem."""
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000664 def __init__(self, item, index, args, kwargs):
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000665 threading.Thread.__init__(self, name=item.name or 'Worker')
maruel@chromium.org3742c842010-09-09 19:27:14 +0000666 logging.info(item.name)
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000667 self.item = item
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000668 self.index = index
maruel@chromium.org3742c842010-09-09 19:27:14 +0000669 self.args = args
670 self.kwargs = kwargs
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000671
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000672 def run(self):
673 """Runs in its own thread."""
674 logging.debug('running(%s)' % self.item.name)
maruel@chromium.org3742c842010-09-09 19:27:14 +0000675 work_queue = self.kwargs['work_queue']
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000676 try:
677 self.item.run(*self.args, **self.kwargs)
maruel@chromium.orgc8d064b2010-08-16 16:46:14 +0000678 except Exception:
679 # Catch exception location.
maruel@chromium.org3742c842010-09-09 19:27:14 +0000680 logging.info('Caught exception in thread %s' % self.item.name)
681 logging.info(str(sys.exc_info()))
682 work_queue.exceptions.put(sys.exc_info())
683 logging.info('Task %s done' % self.item.name)
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000684
maruel@chromium.org3742c842010-09-09 19:27:14 +0000685 work_queue.ready_cond.acquire()
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000686 try:
maruel@chromium.org3742c842010-09-09 19:27:14 +0000687 work_queue.ready_cond.notifyAll()
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000688 finally:
maruel@chromium.org3742c842010-09-09 19:27:14 +0000689 work_queue.ready_cond.release()