blob: ae958cde5d0753d185c0102c46b08233992dad04 [file] [log] [blame]
maruel@chromium.org99ac1c52012-01-16 14:52:12 +00001# Copyright (c) 2012 The Chromium Authors. All rights reserved.
maruel@chromium.org06617272010-11-04 13:50:50 +00002# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
maruel@chromium.org5f3eee32009-09-17 00:34:30 +00004
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +00005"""Generic utils."""
6
Raul Tambreb946b232019-03-26 14:48:46 +00007from __future__ import print_function
8
maruel@chromium.orgdae209f2012-07-03 16:08:15 +00009import codecs
Paweł Hajdan, Jr7e502612017-06-12 16:58:38 +020010import collections
Andrii Shyshkalov351c61d2017-01-21 20:40:16 +000011import contextlib
szager@chromium.orgfe0d1902014-04-08 20:50:44 +000012import datetime
Ben Pastened410c662020-08-26 17:07:03 +000013import errno
Raul Tambreb946b232019-03-26 14:48:46 +000014import functools
15import io
maruel@chromium.orgd9141bf2009-12-23 16:13:32 +000016import logging
Paweł Hajdan, Jr7e502612017-06-12 16:58:38 +020017import operator
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000018import os
ilevy@chromium.orgc28d3772013-07-12 19:42:37 +000019import pipes
szager@chromium.orgfc616382014-03-18 20:32:04 +000020import platform
msb@chromium.orgac915bb2009-11-13 17:03:01 +000021import re
bradnelson@google.com8f9c69f2009-09-17 00:48:28 +000022import stat
borenet@google.com6b4a2ab2013-04-18 15:50:27 +000023import subprocess
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000024import sys
maruel@chromium.org0e0436a2011-10-25 13:32:41 +000025import tempfile
maruel@chromium.org9e5317a2010-08-13 20:35:11 +000026import threading
maruel@chromium.org167b9e62009-09-17 17:41:02 +000027import time
maruel@chromium.orgca0f8392011-09-08 17:15:15 +000028import subprocess2
29
Raul Tambreb946b232019-03-26 14:48:46 +000030if sys.version_info.major == 2:
31 from cStringIO import StringIO
Raul Tambre6693d092020-02-19 20:36:45 +000032 import collections as collections_abc
Edward Lemura8145022020-01-06 18:47:54 +000033 import Queue as queue
34 import urlparse
Raul Tambreb946b232019-03-26 14:48:46 +000035else:
Raul Tambre6693d092020-02-19 20:36:45 +000036 from collections import abc as collections_abc
Raul Tambreb946b232019-03-26 14:48:46 +000037 from io import StringIO
Edward Lemura8145022020-01-06 18:47:54 +000038 import queue
39 import urllib.parse as urlparse
Raul Tambreb946b232019-03-26 14:48:46 +000040
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000041
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +000042RETRY_MAX = 3
43RETRY_INITIAL_SLEEP = 0.5
szager@chromium.orgfe0d1902014-04-08 20:50:44 +000044START = datetime.datetime.now()
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +000045
46
borenet@google.com6a9b1682014-03-24 18:35:23 +000047_WARNINGS = []
48
49
szager@chromium.orgff113292014-03-25 06:02:08 +000050# These repos are known to cause OOM errors on 32-bit platforms, due the the
51# very large objects they contain. It is not safe to use threaded index-pack
52# when cloning/fetching them.
Ayu Ishii09858612020-06-26 18:00:52 +000053THREADED_INDEX_PACK_BLOCKLIST = [
szager@chromium.orgff113292014-03-25 06:02:08 +000054 'https://chromium.googlesource.com/chromium/reference_builds/chrome_win.git'
55]
56
Raul Tambreb946b232019-03-26 14:48:46 +000057"""To support rethrowing exceptions with tracebacks on both Py2 and 3."""
58if sys.version_info.major == 2:
59 # We have to use exec to avoid a SyntaxError in Python 3.
60 exec("def reraise(typ, value, tb=None):\n raise typ, value, tb\n")
61else:
62 def reraise(typ, value, tb=None):
63 if value is None:
64 value = typ()
65 if value.__traceback__ is not tb:
66 raise value.with_traceback(tb)
67 raise value
68
szager@chromium.orgff113292014-03-25 06:02:08 +000069
maruel@chromium.org66c83e62010-09-07 14:18:45 +000070class Error(Exception):
71 """gclient exception class."""
szager@chromium.org4a3c17e2013-05-24 23:59:29 +000072 def __init__(self, msg, *args, **kwargs):
73 index = getattr(threading.currentThread(), 'index', 0)
74 if index:
75 msg = '\n'.join('%d> %s' % (index, l) for l in msg.splitlines())
76 super(Error, self).__init__(msg, *args, **kwargs)
maruel@chromium.org66c83e62010-09-07 14:18:45 +000077
nick@chromium.org3ac1c4e2014-01-16 02:44:42 +000078
szager@chromium.orgfe0d1902014-04-08 20:50:44 +000079def Elapsed(until=None):
80 if until is None:
81 until = datetime.datetime.now()
82 return str(until - START).partition('.')[0]
83
84
borenet@google.com6a9b1682014-03-24 18:35:23 +000085def PrintWarnings():
86 """Prints any accumulated warnings."""
87 if _WARNINGS:
Raul Tambreb946b232019-03-26 14:48:46 +000088 print('\n\nWarnings:', file=sys.stderr)
borenet@google.com6a9b1682014-03-24 18:35:23 +000089 for warning in _WARNINGS:
Raul Tambreb946b232019-03-26 14:48:46 +000090 print(warning, file=sys.stderr)
borenet@google.com6a9b1682014-03-24 18:35:23 +000091
92
93def AddWarning(msg):
94 """Adds the given warning message to the list of accumulated warnings."""
95 _WARNINGS.append(msg)
96
97
msb@chromium.orgac915bb2009-11-13 17:03:01 +000098def SplitUrlRevision(url):
99 """Splits url and returns a two-tuple: url, rev"""
100 if url.startswith('ssh:'):
maruel@chromium.org78b8cd12010-10-26 12:47:07 +0000101 # Make sure ssh://user-name@example.com/~/test.git@stable works
kangil.han@samsung.com71b13572013-10-16 17:28:11 +0000102 regex = r'(ssh://(?:[-.\w]+@)?[-\w:\.]+/[-~\w\./]+)(?:@(.+))?'
msb@chromium.orgac915bb2009-11-13 17:03:01 +0000103 components = re.search(regex, url).groups()
104 else:
scr@chromium.orgf1eccaf2014-04-11 15:51:33 +0000105 components = url.rsplit('@', 1)
106 if re.match(r'^\w+\@', url) and '@' not in components[0]:
107 components = [url]
108
msb@chromium.orgac915bb2009-11-13 17:03:01 +0000109 if len(components) == 1:
110 components += [None]
111 return tuple(components)
112
113
primiano@chromium.org5439ea52014-08-06 17:18:18 +0000114def IsGitSha(revision):
115 """Returns true if the given string is a valid hex-encoded sha"""
116 return re.match('^[a-fA-F0-9]{6,40}$', revision) is not None
117
118
Paweł Hajdan, Jr5ec77132017-08-16 19:21:06 +0200119def IsFullGitSha(revision):
120 """Returns true if the given string is a valid hex-encoded full sha"""
121 return re.match('^[a-fA-F0-9]{40}$', revision) is not None
122
123
floitsch@google.comeaab7842011-04-28 09:07:58 +0000124def IsDateRevision(revision):
125 """Returns true if the given revision is of the form "{ ... }"."""
126 return bool(revision and re.match(r'^\{.+\}$', str(revision)))
127
128
129def MakeDateRevision(date):
130 """Returns a revision representing the latest revision before the given
131 date."""
132 return "{" + date + "}"
133
134
maruel@chromium.org5990f9d2010-07-07 18:02:58 +0000135def SyntaxErrorToError(filename, e):
136 """Raises a gclient_utils.Error exception with the human readable message"""
137 try:
138 # Try to construct a human readable error message
139 if filename:
140 error_message = 'There is a syntax error in %s\n' % filename
141 else:
142 error_message = 'There is a syntax error\n'
143 error_message += 'Line #%s, character %s: "%s"' % (
144 e.lineno, e.offset, re.sub(r'[\r\n]*$', '', e.text))
145 except:
146 # Something went wrong, re-raise the original exception
147 raise e
148 else:
149 raise Error(error_message)
150
151
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000152class PrintableObject(object):
153 def __str__(self):
154 output = ''
155 for i in dir(self):
156 if i.startswith('__'):
157 continue
158 output += '%s = %s\n' % (i, str(getattr(self, i, '')))
159 return output
160
161
Edward Lesmesae3586b2020-03-23 21:21:14 +0000162def AskForData(message):
Christian Flache6855432021-12-01 08:10:05 +0000163 # Try to load the readline module, so that "elaborate line editing" features
164 # such as backspace work for `raw_input` / `input`.
165 try:
166 import readline
167 except ImportError:
168 # The readline module does not exist in all Python distributions, e.g. on
169 # Windows. Fall back to simple input handling.
170 pass
171
Edward Lesmesae3586b2020-03-23 21:21:14 +0000172 # Use this so that it can be mocked in tests on Python 2 and 3.
173 try:
174 if sys.version_info.major == 2:
175 return raw_input(message)
176 return input(message)
177 except KeyboardInterrupt:
178 # Hide the exception.
179 sys.exit(1)
180
181
Edward Lemur419c92f2019-10-25 22:17:49 +0000182def FileRead(filename, mode='rbU'):
Josip Sokcevic8b3bc252021-06-04 18:44:11 +0000183 # mode is ignored now; we always return unicode strings.
184 with open(filename, mode='rb') as f:
chrisha@chromium.org2b99d432012-07-12 18:10:28 +0000185 s = f.read()
Josip Sokcevic8b3bc252021-06-04 18:44:11 +0000186 try:
187 return s.decode('utf-8', 'replace')
188 except (UnicodeDecodeError, AttributeError):
Edward Lemur419c92f2019-10-25 22:17:49 +0000189 return s
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000190
191
Edward Lemur1773f372020-02-22 00:27:14 +0000192def FileWrite(filename, content, mode='w', encoding='utf-8'):
193 with codecs.open(filename, mode=mode, encoding=encoding) as f:
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000194 f.write(content)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000195
196
Andrii Shyshkalov351c61d2017-01-21 20:40:16 +0000197@contextlib.contextmanager
198def temporary_directory(**kwargs):
199 tdir = tempfile.mkdtemp(**kwargs)
200 try:
201 yield tdir
202 finally:
203 if tdir:
204 rmtree(tdir)
205
206
Edward Lemur1773f372020-02-22 00:27:14 +0000207@contextlib.contextmanager
208def temporary_file():
209 """Creates a temporary file.
210
211 On Windows, a file must be closed before it can be opened again. This function
212 allows to write something like:
213
214 with gclient_utils.temporary_file() as tmp:
215 gclient_utils.FileWrite(tmp, foo)
216 useful_stuff(tmp)
217
218 Instead of something like:
219
220 with tempfile.NamedTemporaryFile(delete=False) as tmp:
221 tmp.write(foo)
222 tmp.close()
223 try:
224 useful_stuff(tmp)
225 finally:
226 os.remove(tmp.name)
227 """
228 handle, name = tempfile.mkstemp()
229 os.close(handle)
230 try:
231 yield name
232 finally:
233 os.remove(name)
234
235
cyrille@nnamrak.orgef509e42013-09-20 13:19:08 +0000236def safe_rename(old, new):
237 """Renames a file reliably.
238
nick@chromium.org3ac1c4e2014-01-16 02:44:42 +0000239 Sometimes os.rename does not work because a dying git process keeps a handle
240 on it for a few seconds. An exception is then thrown, which make the program
cyrille@nnamrak.orgef509e42013-09-20 13:19:08 +0000241 give up what it was doing and remove what was deleted.
nick@chromium.org3ac1c4e2014-01-16 02:44:42 +0000242 The only solution is to catch the exception and try again until it works.
cyrille@nnamrak.orgef509e42013-09-20 13:19:08 +0000243 """
244 # roughly 10s
245 retries = 100
246 for i in range(retries):
247 try:
248 os.rename(old, new)
249 break
250 except OSError:
251 if i == (retries - 1):
252 # Give up.
253 raise
254 # retry
255 logging.debug("Renaming failed from %s to %s. Retrying ..." % (old, new))
256 time.sleep(0.1)
257
258
loislo@chromium.org67b59e92014-12-25 13:48:37 +0000259def rm_file_or_tree(path):
Ben Pastene1906f402019-10-24 15:36:00 +0000260 if os.path.isfile(path) or os.path.islink(path):
loislo@chromium.org67b59e92014-12-25 13:48:37 +0000261 os.remove(path)
262 else:
263 rmtree(path)
264
265
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000266def rmtree(path):
267 """shutil.rmtree() on steroids.
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000268
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000269 Recursively removes a directory, even if it's marked read-only.
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000270
271 shutil.rmtree() doesn't work on Windows if any of the files or directories
agable41e3a6c2016-10-20 11:36:56 -0700272 are read-only. We need to be able to force the files to be writable (i.e.,
273 deletable) as we traverse the tree.
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000274
275 Even with all this, Windows still sometimes fails to delete a file, citing
276 a permission error (maybe something to do with antivirus scans or disk
277 indexing). The best suggestion any of the user forums had was to wait a
278 bit and try again, so we do that too. It's hand-waving, but sometimes it
279 works. :/
280
281 On POSIX systems, things are a little bit simpler. The modes of the files
282 to be deleted doesn't matter, only the modes of the directories containing
283 them are significant. As the directory tree is traversed, each directory
284 has its mode set appropriately before descending into it. This should
285 result in the entire tree being removed, with the possible exception of
286 *path itself, because nothing attempts to change the mode of its parent.
287 Doing so would be hazardous, as it's not a directory slated for removal.
288 In the ordinary case, this is not a problem: for our purposes, the user
289 will never lack write permission on *path's parent.
290 """
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000291 if not os.path.exists(path):
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000292 return
293
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000294 if os.path.islink(path) or not os.path.isdir(path):
295 raise Error('Called rmtree(%s) in non-directory' % path)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000296
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000297 if sys.platform == 'win32':
borenet@google.com6b4a2ab2013-04-18 15:50:27 +0000298 # Give up and use cmd.exe's rd command.
299 path = os.path.normcase(path)
Raul Tambrec2f74c12019-03-19 05:55:53 +0000300 for _ in range(3):
borenet@google.com6b4a2ab2013-04-18 15:50:27 +0000301 exitcode = subprocess.call(['cmd.exe', '/c', 'rd', '/q', '/s', path])
302 if exitcode == 0:
303 return
304 else:
Raul Tambreb946b232019-03-26 14:48:46 +0000305 print('rd exited with code %d' % exitcode, file=sys.stderr)
borenet@google.com6b4a2ab2013-04-18 15:50:27 +0000306 time.sleep(3)
307 raise Exception('Failed to remove path %s' % path)
308
309 # On POSIX systems, we need the x-bit set on the directory to access it,
310 # the r-bit to see its contents, and the w-bit to remove files from it.
311 # The actual modes of the files within the directory is irrelevant.
312 os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000313
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000314 def remove(func, subpath):
borenet@google.com6b4a2ab2013-04-18 15:50:27 +0000315 func(subpath)
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000316
317 for fn in os.listdir(path):
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000318 # If fullpath is a symbolic link that points to a directory, isdir will
319 # be True, but we don't want to descend into that as a directory, we just
320 # want to remove the link. Check islink and treat links as ordinary files
321 # would be treated regardless of what they reference.
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000322 fullpath = os.path.join(path, fn)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000323 if os.path.islink(fullpath) or not os.path.isdir(fullpath):
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000324 remove(os.remove, fullpath)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000325 else:
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000326 # Recurse.
327 rmtree(fullpath)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000328
maruel@chromium.orgf9040722011-03-09 14:47:51 +0000329 remove(os.rmdir, path)
330
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000331
maruel@chromium.org6c48a302011-10-20 23:44:20 +0000332def safe_makedirs(tree):
333 """Creates the directory in a safe manner.
334
Quinten Yearsley925cedb2020-04-13 17:49:39 +0000335 Because multiple threads can create these directories concurrently, trap the
maruel@chromium.org6c48a302011-10-20 23:44:20 +0000336 exception and pass on.
337 """
338 count = 0
339 while not os.path.exists(tree):
340 count += 1
341 try:
342 os.makedirs(tree)
Raul Tambreb946b232019-03-26 14:48:46 +0000343 except OSError as e:
maruel@chromium.org6c48a302011-10-20 23:44:20 +0000344 # 17 POSIX, 183 Windows
345 if e.errno not in (17, 183):
346 raise
347 if count > 40:
348 # Give up.
349 raise
350
351
ilevy@chromium.orgc28d3772013-07-12 19:42:37 +0000352def CommandToStr(args):
353 """Converts an arg list into a shell escaped string."""
354 return ' '.join(pipes.quote(arg) for arg in args)
355
356
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000357class Wrapper(object):
358 """Wraps an object, acting as a transparent proxy for all properties by
359 default.
360 """
361 def __init__(self, wrapped):
362 self._wrapped = wrapped
363
364 def __getattr__(self, name):
365 return getattr(self._wrapped, name)
maruel@chromium.orgdb111f72010-09-08 13:36:53 +0000366
maruel@chromium.orge0de9cb2010-09-17 15:07:14 +0000367
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000368class AutoFlush(Wrapper):
maruel@chromium.orge0de9cb2010-09-17 15:07:14 +0000369 """Creates a file object clone to automatically flush after N seconds."""
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000370 def __init__(self, wrapped, delay):
371 super(AutoFlush, self).__init__(wrapped)
372 if not hasattr(self, 'lock'):
373 self.lock = threading.Lock()
374 self.__last_flushed_at = time.time()
375 self.delay = delay
maruel@chromium.orge0de9cb2010-09-17 15:07:14 +0000376
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000377 @property
378 def autoflush(self):
379 return self
maruel@chromium.orge0de9cb2010-09-17 15:07:14 +0000380
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000381 def write(self, out, *args, **kwargs):
382 self._wrapped.write(out, *args, **kwargs)
maruel@chromium.orgdb111f72010-09-08 13:36:53 +0000383 should_flush = False
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000384 self.lock.acquire()
maruel@chromium.org9c531262010-09-08 13:41:13 +0000385 try:
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000386 if self.delay and (time.time() - self.__last_flushed_at) > self.delay:
maruel@chromium.orgdb111f72010-09-08 13:36:53 +0000387 should_flush = True
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000388 self.__last_flushed_at = time.time()
maruel@chromium.org9c531262010-09-08 13:41:13 +0000389 finally:
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000390 self.lock.release()
maruel@chromium.orgdb111f72010-09-08 13:36:53 +0000391 if should_flush:
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000392 self.flush()
maruel@chromium.orgdb111f72010-09-08 13:36:53 +0000393
394
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000395class Annotated(Wrapper):
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000396 """Creates a file object clone to automatically prepends every line in worker
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000397 threads with a NN> prefix.
398 """
399 def __init__(self, wrapped, include_zero=False):
400 super(Annotated, self).__init__(wrapped)
401 if not hasattr(self, 'lock'):
402 self.lock = threading.Lock()
403 self.__output_buffers = {}
404 self.__include_zero = include_zero
Edward Lemurcb1eb482019-10-09 18:03:14 +0000405 self._wrapped_write = getattr(self._wrapped, 'buffer', self._wrapped).write
maruel@chromium.orgcb1e97a2010-09-09 20:09:20 +0000406
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000407 @property
408 def annotated(self):
409 return self
maruel@chromium.orgcb1e97a2010-09-09 20:09:20 +0000410
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000411 def write(self, out):
Edward Lemurcb1eb482019-10-09 18:03:14 +0000412 # Store as bytes to ensure Unicode characters get output correctly.
413 if not isinstance(out, bytes):
414 out = out.encode('utf-8')
415
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000416 index = getattr(threading.currentThread(), 'index', 0)
417 if not index and not self.__include_zero:
418 # Unindexed threads aren't buffered.
Edward Lemurcb1eb482019-10-09 18:03:14 +0000419 return self._wrapped_write(out)
maruel@chromium.orgcb1e97a2010-09-09 20:09:20 +0000420
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000421 self.lock.acquire()
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000422 try:
423 # Use a dummy array to hold the string so the code can be lockless.
424 # Strings are immutable, requiring to keep a lock for the whole dictionary
425 # otherwise. Using an array is faster than using a dummy object.
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000426 if not index in self.__output_buffers:
Edward Lemurcb1eb482019-10-09 18:03:14 +0000427 obj = self.__output_buffers[index] = [b'']
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000428 else:
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000429 obj = self.__output_buffers[index]
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000430 finally:
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000431 self.lock.release()
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000432
433 # Continue lockless.
434 obj[0] += out
Raul Tambre25eb8e42019-05-14 16:39:45 +0000435 while True:
Edward Lemurcb1eb482019-10-09 18:03:14 +0000436 cr_loc = obj[0].find(b'\r')
437 lf_loc = obj[0].find(b'\n')
Raul Tambre25eb8e42019-05-14 16:39:45 +0000438 if cr_loc == lf_loc == -1:
439 break
440 elif cr_loc == -1 or (lf_loc >= 0 and lf_loc < cr_loc):
Edward Lemurcb1eb482019-10-09 18:03:14 +0000441 line, remaining = obj[0].split(b'\n', 1)
Raul Tambre25eb8e42019-05-14 16:39:45 +0000442 if line:
Edward Lemurcb1eb482019-10-09 18:03:14 +0000443 self._wrapped_write(b'%d>%s\n' % (index, line))
Raul Tambre25eb8e42019-05-14 16:39:45 +0000444 elif lf_loc == -1 or (cr_loc >= 0 and cr_loc < lf_loc):
Edward Lemurcb1eb482019-10-09 18:03:14 +0000445 line, remaining = obj[0].split(b'\r', 1)
Raul Tambre25eb8e42019-05-14 16:39:45 +0000446 if line:
Edward Lemurcb1eb482019-10-09 18:03:14 +0000447 self._wrapped_write(b'%d>%s\r' % (index, line))
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000448 obj[0] = remaining
449
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000450 def flush(self):
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000451 """Flush buffered output."""
452 orphans = []
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000453 self.lock.acquire()
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000454 try:
455 # Detect threads no longer existing.
456 indexes = (getattr(t, 'index', None) for t in threading.enumerate())
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000457 indexes = filter(None, indexes)
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000458 for index in self.__output_buffers:
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000459 if not index in indexes:
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000460 orphans.append((index, self.__output_buffers[index][0]))
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000461 for orphan in orphans:
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000462 del self.__output_buffers[orphan[0]]
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000463 finally:
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000464 self.lock.release()
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000465
Quinten Yearsley925cedb2020-04-13 17:49:39 +0000466 # Don't keep the lock while writing. Will append \n when it shouldn't.
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000467 for orphan in orphans:
nsylvain@google.come939bb52011-06-01 22:59:15 +0000468 if orphan[1]:
Edward Lemurcb1eb482019-10-09 18:03:14 +0000469 self._wrapped_write(b'%d>%s\n' % (orphan[0], orphan[1]))
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000470 return self._wrapped.flush()
maruel@chromium.org4ed34182010-09-17 15:57:47 +0000471
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000472
473def MakeFileAutoFlush(fileobj, delay=10):
474 autoflush = getattr(fileobj, 'autoflush', None)
475 if autoflush:
476 autoflush.delay = delay
477 return fileobj
478 return AutoFlush(fileobj, delay)
479
480
481def MakeFileAnnotated(fileobj, include_zero=False):
482 if getattr(fileobj, 'annotated', None):
483 return fileobj
Raul Tambre383f6cf2019-09-21 14:40:59 +0000484 return Annotated(fileobj, include_zero)
maruel@chromium.orgcb1e97a2010-09-09 20:09:20 +0000485
486
xusydoc@chromium.org2fd6c3f2013-05-03 21:57:55 +0000487GCLIENT_CHILDREN = []
488GCLIENT_CHILDREN_LOCK = threading.Lock()
489
490
491class GClientChildren(object):
492 @staticmethod
493 def add(popen_obj):
494 with GCLIENT_CHILDREN_LOCK:
495 GCLIENT_CHILDREN.append(popen_obj)
496
497 @staticmethod
498 def remove(popen_obj):
499 with GCLIENT_CHILDREN_LOCK:
500 GCLIENT_CHILDREN.remove(popen_obj)
501
502 @staticmethod
503 def _attemptToKillChildren():
504 global GCLIENT_CHILDREN
505 with GCLIENT_CHILDREN_LOCK:
506 zombies = [c for c in GCLIENT_CHILDREN if c.poll() is None]
507
508 for zombie in zombies:
509 try:
510 zombie.kill()
511 except OSError:
512 pass
513
514 with GCLIENT_CHILDREN_LOCK:
515 GCLIENT_CHILDREN = [k for k in GCLIENT_CHILDREN if k.poll() is not None]
516
517 @staticmethod
518 def _areZombies():
519 with GCLIENT_CHILDREN_LOCK:
520 return bool(GCLIENT_CHILDREN)
521
522 @staticmethod
523 def KillAllRemainingChildren():
524 GClientChildren._attemptToKillChildren()
525
526 if GClientChildren._areZombies():
527 time.sleep(0.5)
528 GClientChildren._attemptToKillChildren()
529
530 with GCLIENT_CHILDREN_LOCK:
531 if GCLIENT_CHILDREN:
Raul Tambreb946b232019-03-26 14:48:46 +0000532 print('Could not kill the following subprocesses:', file=sys.stderr)
xusydoc@chromium.org2fd6c3f2013-05-03 21:57:55 +0000533 for zombie in GCLIENT_CHILDREN:
Raul Tambreb946b232019-03-26 14:48:46 +0000534 print(' ', zombie.pid, file=sys.stderr)
xusydoc@chromium.org2fd6c3f2013-05-03 21:57:55 +0000535
536
Edward Lemur24146be2019-08-01 21:44:52 +0000537def CheckCallAndFilter(args, print_stdout=False, filter_fn=None,
538 show_header=False, always_show_header=False, retry=False,
539 **kwargs):
maruel@chromium.org17d01792010-09-01 18:07:10 +0000540 """Runs a command and calls back a filter function if needed.
541
maruel@chromium.org57bf78d2011-09-08 18:57:33 +0000542 Accepts all subprocess2.Popen() parameters plus:
maruel@chromium.org17d01792010-09-01 18:07:10 +0000543 print_stdout: If True, the command's stdout is forwarded to stdout.
544 filter_fn: A function taking a single string argument called with each line
maruel@chromium.org57bf78d2011-09-08 18:57:33 +0000545 of the subprocess2's output. Each line has the trailing newline
maruel@chromium.org17d01792010-09-01 18:07:10 +0000546 character trimmed.
Edward Lemur24146be2019-08-01 21:44:52 +0000547 show_header: Whether to display a header before the command output.
548 always_show_header: Show header even when the command produced no output.
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +0000549 retry: If the process exits non-zero, sleep for a brief interval and try
550 again, up to RETRY_MAX times.
maruel@chromium.org17d01792010-09-01 18:07:10 +0000551
552 stderr is always redirected to stdout.
Edward Lemur24146be2019-08-01 21:44:52 +0000553
554 Returns the output of the command as a binary string.
maruel@chromium.org17d01792010-09-01 18:07:10 +0000555 """
Edward Lemur24146be2019-08-01 21:44:52 +0000556 def show_header_if_necessary(needs_header, attempt):
557 """Show the header at most once."""
558 if not needs_header[0]:
559 return
560
561 needs_header[0] = False
562 # Automatically generated header. We only prepend a newline if
563 # always_show_header is false, since it usually indicates there's an
564 # external progress display, and it's better not to clobber it in that case.
565 header = '' if always_show_header else '\n'
566 header += '________ running \'%s\' in \'%s\'' % (
567 ' '.join(args), kwargs.get('cwd', '.'))
568 if attempt:
569 header += ' attempt %s / %s' % (attempt + 1, RETRY_MAX + 1)
570 header += '\n'
571
572 if print_stdout:
Edward Lemurefce0d12019-09-07 03:36:37 +0000573 stdout_write = getattr(sys.stdout, 'buffer', sys.stdout).write
574 stdout_write(header.encode())
Edward Lemur24146be2019-08-01 21:44:52 +0000575 if filter_fn:
576 filter_fn(header)
577
578 def filter_line(command_output, line_start):
579 """Extract the last line from command output and filter it."""
580 if not filter_fn or line_start is None:
581 return
582 command_output.seek(line_start)
583 filter_fn(command_output.read().decode('utf-8'))
584
585 # Initialize stdout writer if needed. On Python 3, sys.stdout does not accept
586 # byte inputs and sys.stdout.buffer must be used instead.
587 if print_stdout:
588 sys.stdout.flush()
589 stdout_write = getattr(sys.stdout, 'buffer', sys.stdout).write
590 else:
591 stdout_write = lambda _: None
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000592
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +0000593 sleep_interval = RETRY_INITIAL_SLEEP
594 run_cwd = kwargs.get('cwd', os.getcwd())
Josip Sokcevic740825e2021-05-12 18:28:34 +0000595
596 # Store the output of the command regardless of the value of print_stdout or
597 # filter_fn.
598 command_output = io.BytesIO()
Edward Lemur24146be2019-08-01 21:44:52 +0000599 for attempt in range(RETRY_MAX + 1):
Ben Pastened410c662020-08-26 17:07:03 +0000600 # If our stdout is a terminal, then pass in a psuedo-tty pipe to our
601 # subprocess when filtering its output. This makes the subproc believe
602 # it was launched from a terminal, which will preserve ANSI color codes.
Milad Fad949c912020-09-18 00:26:08 +0000603 os_type = GetMacWinAixOrLinux()
604 if sys.stdout.isatty() and os_type != 'win' and os_type != 'aix':
Ben Pastened410c662020-08-26 17:07:03 +0000605 pipe_reader, pipe_writer = os.openpty()
606 else:
607 pipe_reader, pipe_writer = os.pipe()
608
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +0000609 kid = subprocess2.Popen(
Ben Pastened410c662020-08-26 17:07:03 +0000610 args, bufsize=0, stdout=pipe_writer, stderr=subprocess2.STDOUT,
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +0000611 **kwargs)
Ben Pastened410c662020-08-26 17:07:03 +0000612 # Close the write end of the pipe once we hand it off to the child proc.
613 os.close(pipe_writer)
xusydoc@chromium.org2fd6c3f2013-05-03 21:57:55 +0000614
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +0000615 GClientChildren.add(kid)
chase@chromium.org8ad1cee2010-08-16 19:12:27 +0000616
Edward Lemur24146be2019-08-01 21:44:52 +0000617 # Passed as a list for "by ref" semantics.
618 needs_header = [show_header]
619 if always_show_header:
620 show_header_if_necessary(needs_header, attempt)
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +0000621
622 # Also, we need to forward stdout to prevent weird re-ordering of output.
623 # This has to be done on a per byte basis to make sure it is not buffered:
agable41e3a6c2016-10-20 11:36:56 -0700624 # normally buffering is done for each line, but if the process requests
625 # input, no end-of-line character is output after the prompt and it would
626 # not show up.
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +0000627 try:
Edward Lemur24146be2019-08-01 21:44:52 +0000628 line_start = None
629 while True:
Ben Pastened410c662020-08-26 17:07:03 +0000630 try:
631 in_byte = os.read(pipe_reader, 1)
632 except (IOError, OSError) as e:
633 if e.errno == errno.EIO:
634 # An errno.EIO means EOF?
635 in_byte = None
636 else:
637 raise e
Edward Lemur24146be2019-08-01 21:44:52 +0000638 is_newline = in_byte in (b'\n', b'\r')
639 if not in_byte:
640 break
641
642 show_header_if_necessary(needs_header, attempt)
643
644 if is_newline:
645 filter_line(command_output, line_start)
646 line_start = None
647 elif line_start is None:
648 line_start = command_output.tell()
649
650 stdout_write(in_byte)
651 command_output.write(in_byte)
652
653 # Flush the rest of buffered output.
654 sys.stdout.flush()
655 if line_start is not None:
656 filter_line(command_output, line_start)
657
Ben Pastened410c662020-08-26 17:07:03 +0000658 os.close(pipe_reader)
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +0000659 rv = kid.wait()
xusydoc@chromium.org2fd6c3f2013-05-03 21:57:55 +0000660
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +0000661 # Don't put this in a 'finally,' since the child may still run if we get
662 # an exception.
663 GClientChildren.remove(kid)
xusydoc@chromium.org2fd6c3f2013-05-03 21:57:55 +0000664
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +0000665 except KeyboardInterrupt:
Raul Tambreb946b232019-03-26 14:48:46 +0000666 print('Failed while running "%s"' % ' '.join(args), file=sys.stderr)
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +0000667 raise
maruel@chromium.org109cb9d2011-09-14 20:03:11 +0000668
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +0000669 if rv == 0:
Edward Lemur24146be2019-08-01 21:44:52 +0000670 return command_output.getvalue()
671
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +0000672 if not retry:
673 break
Edward Lemur24146be2019-08-01 21:44:52 +0000674
Raul Tambreb946b232019-03-26 14:48:46 +0000675 print("WARNING: subprocess '%s' in %s failed; will retry after a short "
676 'nap...' % (' '.join('"%s"' % x for x in args), run_cwd))
Josip Sokcevic740825e2021-05-12 18:28:34 +0000677 command_output = io.BytesIO()
raphael.kubo.da.costa@intel.com91507f72013-10-22 12:18:25 +0000678 time.sleep(sleep_interval)
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +0000679 sleep_interval *= 2
Edward Lemur24146be2019-08-01 21:44:52 +0000680
szager@chromium.orgf2d7d6b2013-10-17 20:41:43 +0000681 raise subprocess2.CalledProcessError(
Josip Sokcevic740825e2021-05-12 18:28:34 +0000682 rv, args, kwargs.get('cwd', None), command_output.getvalue(), None)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000683
684
agable@chromium.org5a306a22014-02-24 22:13:59 +0000685class GitFilter(object):
686 """A filter_fn implementation for quieting down git output messages.
687
688 Allows a custom function to skip certain lines (predicate), and will throttle
689 the output of percentage completed lines to only output every X seconds.
690 """
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000691 PERCENT_RE = re.compile('(.*) ([0-9]{1,3})% .*')
agable@chromium.org5a306a22014-02-24 22:13:59 +0000692
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000693 def __init__(self, time_throttle=0, predicate=None, out_fh=None):
agable@chromium.org5a306a22014-02-24 22:13:59 +0000694 """
695 Args:
696 time_throttle (int): GitFilter will throttle 'noisy' output (such as the
697 XX% complete messages) to only be printed at least |time_throttle|
698 seconds apart.
699 predicate (f(line)): An optional function which is invoked for every line.
700 The line will be skipped if predicate(line) returns False.
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000701 out_fh: File handle to write output to.
agable@chromium.org5a306a22014-02-24 22:13:59 +0000702 """
Edward Lemur24146be2019-08-01 21:44:52 +0000703 self.first_line = True
agable@chromium.org5a306a22014-02-24 22:13:59 +0000704 self.last_time = 0
705 self.time_throttle = time_throttle
706 self.predicate = predicate
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000707 self.out_fh = out_fh or sys.stdout
708 self.progress_prefix = None
agable@chromium.org5a306a22014-02-24 22:13:59 +0000709
710 def __call__(self, line):
711 # git uses an escape sequence to clear the line; elide it.
Christian Biesinger1b4c7e92019-08-08 19:33:16 +0000712 esc = line.find(chr(0o33))
agable@chromium.org5a306a22014-02-24 22:13:59 +0000713 if esc > -1:
714 line = line[:esc]
715 if self.predicate and not self.predicate(line):
716 return
717 now = time.time()
Christian Biesinger1b4c7e92019-08-08 19:33:16 +0000718 match = self.PERCENT_RE.match(line)
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000719 if match:
720 if match.group(1) != self.progress_prefix:
721 self.progress_prefix = match.group(1)
722 elif now - self.last_time < self.time_throttle:
723 return
724 self.last_time = now
Edward Lemur24146be2019-08-01 21:44:52 +0000725 if not self.first_line:
726 self.out_fh.write('[%s] ' % Elapsed())
727 self.first_line = False
Raul Tambreb946b232019-03-26 14:48:46 +0000728 print(line, file=self.out_fh)
agable@chromium.org5a306a22014-02-24 22:13:59 +0000729
730
piman@chromium.orgf43d0192010-04-15 02:36:04 +0000731def FindFileUpwards(filename, path=None):
rcui@google.com13595ff2011-10-13 01:25:07 +0000732 """Search upwards from the a directory (default: current) to find a file.
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +0000733
rcui@google.com13595ff2011-10-13 01:25:07 +0000734 Returns nearest upper-level directory with the passed in file.
735 """
piman@chromium.orgf43d0192010-04-15 02:36:04 +0000736 if not path:
737 path = os.getcwd()
738 path = os.path.realpath(path)
739 while True:
740 file_path = os.path.join(path, filename)
rcui@google.com13595ff2011-10-13 01:25:07 +0000741 if os.path.exists(file_path):
742 return path
piman@chromium.orgf43d0192010-04-15 02:36:04 +0000743 (new_path, _) = os.path.split(path)
744 if new_path == path:
745 return None
746 path = new_path
747
748
Milad Fa52fdd1f2020-09-15 21:24:46 +0000749def GetMacWinAixOrLinux():
nick@chromium.org3ac1c4e2014-01-16 02:44:42 +0000750 """Returns 'mac', 'win', or 'linux', matching the current platform."""
751 if sys.platform.startswith(('cygwin', 'win')):
752 return 'win'
753 elif sys.platform.startswith('linux'):
754 return 'linux'
755 elif sys.platform == 'darwin':
756 return 'mac'
Milad Fa52fdd1f2020-09-15 21:24:46 +0000757 elif sys.platform.startswith('aix'):
758 return 'aix'
nick@chromium.org3ac1c4e2014-01-16 02:44:42 +0000759 raise Error('Unknown platform: ' + sys.platform)
760
761
piman@chromium.orgf43d0192010-04-15 02:36:04 +0000762def GetGClientRootAndEntries(path=None):
763 """Returns the gclient root and the dict of entries."""
764 config_file = '.gclient_entries'
maruel@chromium.org93a9ee02011-10-18 18:23:58 +0000765 root = FindFileUpwards(config_file, path)
766 if not root:
Raul Tambreb946b232019-03-26 14:48:46 +0000767 print("Can't find %s" % config_file)
piman@chromium.orgf43d0192010-04-15 02:36:04 +0000768 return None
maruel@chromium.org93a9ee02011-10-18 18:23:58 +0000769 config_path = os.path.join(root, config_file)
piman@chromium.orgf43d0192010-04-15 02:36:04 +0000770 env = {}
Raphael Kubo da Costa107c97c2019-10-07 18:04:26 +0000771 with open(config_path) as config:
772 exec(config.read(), env)
piman@chromium.orgf43d0192010-04-15 02:36:04 +0000773 config_dir = os.path.dirname(config_path)
774 return config_dir, env['entries']
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000775
776
maruel@chromium.org6ca8bf82011-09-19 23:04:30 +0000777def lockedmethod(method):
778 """Method decorator that holds self.lock for the duration of the call."""
779 def inner(self, *args, **kwargs):
780 try:
781 try:
782 self.lock.acquire()
783 except KeyboardInterrupt:
Raul Tambreb946b232019-03-26 14:48:46 +0000784 print('Was deadlocked', file=sys.stderr)
maruel@chromium.org6ca8bf82011-09-19 23:04:30 +0000785 raise
786 return method(self, *args, **kwargs)
787 finally:
788 self.lock.release()
789 return inner
790
791
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000792class WorkItem(object):
793 """One work item."""
maruel@chromium.org4901daf2011-10-20 14:34:47 +0000794 # On cygwin, creating a lock throwing randomly when nearing ~100 locks.
795 # As a workaround, use a single lock. Yep you read it right. Single lock for
796 # all the 100 objects.
797 lock = threading.Lock()
798
maruel@chromium.org6ca8bf82011-09-19 23:04:30 +0000799 def __init__(self, name):
maruel@chromium.org485dcab2011-09-14 12:48:47 +0000800 # A unique string representing this work item.
maruel@chromium.org6ca8bf82011-09-19 23:04:30 +0000801 self._name = name
Raul Tambreb946b232019-03-26 14:48:46 +0000802 self.outbuf = StringIO()
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000803 self.start = self.finish = None
hinoka885e5b12016-06-08 14:40:09 -0700804 self.resources = [] # List of resources this work item requires.
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000805
maruel@chromium.org77e4eca2010-09-21 13:23:07 +0000806 def run(self, work_queue):
807 """work_queue is passed as keyword argument so it should be
maruel@chromium.org3742c842010-09-09 19:27:14 +0000808 the last parameters of the function when you override it."""
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000809 pass
810
maruel@chromium.org6ca8bf82011-09-19 23:04:30 +0000811 @property
812 def name(self):
813 return self._name
814
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000815
816class ExecutionQueue(object):
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000817 """Runs a set of WorkItem that have interdependencies and were WorkItem are
818 added as they are processed.
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000819
Paweł Hajdan, Jr7e9303b2017-05-23 14:38:27 +0200820 This class manages that all the required dependencies are run
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000821 before running each one.
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000822
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000823 Methods of this class are thread safe.
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000824 """
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000825 def __init__(self, jobs, progress, ignore_requirements, verbose=False):
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000826 """jobs specifies the number of concurrent tasks to allow. progress is a
827 Progress instance."""
828 # Set when a thread is done or a new item is enqueued.
829 self.ready_cond = threading.Condition()
830 # Maximum number of concurrent tasks.
831 self.jobs = jobs
832 # List of WorkItem, for gclient, these are Dependency instances.
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000833 self.queued = []
834 # List of strings representing each Dependency.name that was run.
835 self.ran = []
836 # List of items currently running.
837 self.running = []
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000838 # Exceptions thrown if any.
Raul Tambreb946b232019-03-26 14:48:46 +0000839 self.exceptions = queue.Queue()
maruel@chromium.org3742c842010-09-09 19:27:14 +0000840 # Progress status
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000841 self.progress = progress
842 if self.progress:
maruel@chromium.org3742c842010-09-09 19:27:14 +0000843 self.progress.update(0)
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000844
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +0000845 self.ignore_requirements = ignore_requirements
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000846 self.verbose = verbose
847 self.last_join = None
848 self.last_subproc_output = None
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +0000849
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000850 def enqueue(self, d):
851 """Enqueue one Dependency to be executed later once its requirements are
852 satisfied.
853 """
854 assert isinstance(d, WorkItem)
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000855 self.ready_cond.acquire()
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000856 try:
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000857 self.queued.append(d)
858 total = len(self.queued) + len(self.ran) + len(self.running)
szager@chromium.orge98e04c2014-07-25 00:28:06 +0000859 if self.jobs == 1:
860 total += 1
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000861 logging.debug('enqueued(%s)' % d.name)
862 if self.progress:
szager@chromium.orge98e04c2014-07-25 00:28:06 +0000863 self.progress._total = total
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000864 self.progress.update(0)
865 self.ready_cond.notifyAll()
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000866 finally:
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000867 self.ready_cond.release()
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000868
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000869 def out_cb(self, _):
870 self.last_subproc_output = datetime.datetime.now()
871 return True
872
873 @staticmethod
874 def format_task_output(task, comment=''):
875 if comment:
876 comment = ' (%s)' % comment
877 if task.start and task.finish:
878 elapsed = ' (Elapsed: %s)' % (
879 str(task.finish - task.start).partition('.')[0])
880 else:
881 elapsed = ''
882 return """
883%s%s%s
884----------------------------------------
885%s
886----------------------------------------""" % (
szager@chromium.org1f4e71b2014-04-09 19:45:40 +0000887 task.name, comment, elapsed, task.outbuf.getvalue().strip())
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000888
hinoka885e5b12016-06-08 14:40:09 -0700889 def _is_conflict(self, job):
890 """Checks to see if a job will conflict with another running job."""
891 for running_job in self.running:
892 for used_resource in running_job.item.resources:
893 logging.debug('Checking resource %s' % used_resource)
894 if used_resource in job.resources:
895 return True
896 return False
897
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000898 def flush(self, *args, **kwargs):
899 """Runs all enqueued items until all are executed."""
maruel@chromium.org3742c842010-09-09 19:27:14 +0000900 kwargs['work_queue'] = self
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000901 self.last_subproc_output = self.last_join = datetime.datetime.now()
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000902 self.ready_cond.acquire()
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000903 try:
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000904 while True:
905 # Check for task to run first, then wait.
906 while True:
maruel@chromium.org3742c842010-09-09 19:27:14 +0000907 if not self.exceptions.empty():
908 # Systematically flush the queue when an exception logged.
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000909 self.queued = []
maruel@chromium.org3742c842010-09-09 19:27:14 +0000910 self._flush_terminated_threads()
911 if (not self.queued and not self.running or
912 self.jobs == len(self.running)):
maruel@chromium.org1333cb32011-10-04 23:40:16 +0000913 logging.debug('No more worker threads or can\'t queue anything.')
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000914 break
maruel@chromium.org3742c842010-09-09 19:27:14 +0000915
916 # Check for new tasks to start.
Raul Tambreb946b232019-03-26 14:48:46 +0000917 for i in range(len(self.queued)):
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000918 # Verify its requirements.
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +0000919 if (self.ignore_requirements or
920 not (set(self.queued[i].requirements) - set(self.ran))):
hinoka885e5b12016-06-08 14:40:09 -0700921 if not self._is_conflict(self.queued[i]):
922 # Start one work item: all its requirements are satisfied.
923 self._run_one_task(self.queued.pop(i), args, kwargs)
924 break
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000925 else:
926 # Couldn't find an item that could run. Break out the outher loop.
927 break
maruel@chromium.org3742c842010-09-09 19:27:14 +0000928
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000929 if not self.queued and not self.running:
maruel@chromium.org3742c842010-09-09 19:27:14 +0000930 # We're done.
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000931 break
932 # We need to poll here otherwise Ctrl-C isn't processed.
maruel@chromium.org485dcab2011-09-14 12:48:47 +0000933 try:
934 self.ready_cond.wait(10)
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000935 # If we haven't printed to terminal for a while, but we have received
936 # spew from a suprocess, let the user know we're still progressing.
937 now = datetime.datetime.now()
938 if (now - self.last_join > datetime.timedelta(seconds=60) and
939 self.last_subproc_output > self.last_join):
940 if self.progress:
Raul Tambreb946b232019-03-26 14:48:46 +0000941 print('')
hinoka@google.com4dfb8662014-04-25 22:21:24 +0000942 sys.stdout.flush()
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000943 elapsed = Elapsed()
Raul Tambreb946b232019-03-26 14:48:46 +0000944 print('[%s] Still working on:' % elapsed)
hinoka@google.com4dfb8662014-04-25 22:21:24 +0000945 sys.stdout.flush()
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000946 for task in self.running:
Raul Tambreb946b232019-03-26 14:48:46 +0000947 print('[%s] %s' % (elapsed, task.item.name))
hinoka@google.com4dfb8662014-04-25 22:21:24 +0000948 sys.stdout.flush()
maruel@chromium.org485dcab2011-09-14 12:48:47 +0000949 except KeyboardInterrupt:
950 # Help debugging by printing some information:
Raul Tambreb946b232019-03-26 14:48:46 +0000951 print(
maruel@chromium.org485dcab2011-09-14 12:48:47 +0000952 ('\nAllowed parallel jobs: %d\n# queued: %d\nRan: %s\n'
Raul Tambreb946b232019-03-26 14:48:46 +0000953 'Running: %d') % (self.jobs, len(self.queued), ', '.join(
954 self.ran), len(self.running)),
955 file=sys.stderr)
maruel@chromium.org485dcab2011-09-14 12:48:47 +0000956 for i in self.queued:
Raul Tambreb946b232019-03-26 14:48:46 +0000957 print(
958 '%s (not started): %s' % (i.name, ', '.join(i.requirements)),
959 file=sys.stderr)
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000960 for i in self.running:
Raul Tambreb946b232019-03-26 14:48:46 +0000961 print(
962 self.format_task_output(i.item, 'interrupted'), file=sys.stderr)
maruel@chromium.org485dcab2011-09-14 12:48:47 +0000963 raise
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000964 # Something happened: self.enqueue() or a thread terminated. Loop again.
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000965 finally:
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000966 self.ready_cond.release()
maruel@chromium.org3742c842010-09-09 19:27:14 +0000967
maruel@chromium.org9e5317a2010-08-13 20:35:11 +0000968 assert not self.running, 'Now guaranteed to be single-threaded'
maruel@chromium.org3742c842010-09-09 19:27:14 +0000969 if not self.exceptions.empty():
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000970 if self.progress:
Raul Tambreb946b232019-03-26 14:48:46 +0000971 print('')
maruel@chromium.orgc8d064b2010-08-16 16:46:14 +0000972 # To get back the stack location correctly, the raise a, b, c form must be
973 # used, passing a tuple as the first argument doesn't work.
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000974 e, task = self.exceptions.get()
Raul Tambreb946b232019-03-26 14:48:46 +0000975 print(self.format_task_output(task.item, 'ERROR'), file=sys.stderr)
976 reraise(e[0], e[1], e[2])
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000977 elif self.progress:
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000978 self.progress.end()
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +0000979
maruel@chromium.org3742c842010-09-09 19:27:14 +0000980 def _flush_terminated_threads(self):
981 """Flush threads that have terminated."""
982 running = self.running
983 self.running = []
984 for t in running:
Edward Lemura877ee62019-09-03 20:23:17 +0000985 if t.is_alive():
maruel@chromium.org3742c842010-09-09 19:27:14 +0000986 self.running.append(t)
987 else:
988 t.join()
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000989 self.last_join = datetime.datetime.now()
maruel@chromium.org042f0e72011-10-23 00:04:35 +0000990 sys.stdout.flush()
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000991 if self.verbose:
Raul Tambreb946b232019-03-26 14:48:46 +0000992 print(self.format_task_output(t.item))
maruel@chromium.org3742c842010-09-09 19:27:14 +0000993 if self.progress:
maruel@chromium.org55a2eb82010-10-06 23:35:18 +0000994 self.progress.update(1, t.item.name)
maruel@chromium.orgf36c0ee2011-09-14 19:16:47 +0000995 if t.item.name in self.ran:
996 raise Error(
997 'gclient is confused, "%s" is already in "%s"' % (
998 t.item.name, ', '.join(self.ran)))
maruel@chromium.orgacc45672010-09-09 21:21:21 +0000999 if not t.item.name in self.ran:
1000 self.ran.append(t.item.name)
maruel@chromium.org3742c842010-09-09 19:27:14 +00001001
1002 def _run_one_task(self, task_item, args, kwargs):
1003 if self.jobs > 1:
1004 # Start the thread.
1005 index = len(self.ran) + len(self.running) + 1
maruel@chromium.org77e4eca2010-09-21 13:23:07 +00001006 new_thread = self._Worker(task_item, index, args, kwargs)
maruel@chromium.org3742c842010-09-09 19:27:14 +00001007 self.running.append(new_thread)
1008 new_thread.start()
1009 else:
1010 # Run the 'thread' inside the main thread. Don't try to catch any
1011 # exception.
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001012 try:
1013 task_item.start = datetime.datetime.now()
Raul Tambreb946b232019-03-26 14:48:46 +00001014 print('[%s] Started.' % Elapsed(task_item.start), file=task_item.outbuf)
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001015 task_item.run(*args, **kwargs)
1016 task_item.finish = datetime.datetime.now()
Raul Tambreb946b232019-03-26 14:48:46 +00001017 print(
1018 '[%s] Finished.' % Elapsed(task_item.finish), file=task_item.outbuf)
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001019 self.ran.append(task_item.name)
1020 if self.verbose:
1021 if self.progress:
Raul Tambreb946b232019-03-26 14:48:46 +00001022 print('')
1023 print(self.format_task_output(task_item))
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001024 if self.progress:
1025 self.progress.update(1, ', '.join(t.item.name for t in self.running))
1026 except KeyboardInterrupt:
Raul Tambreb946b232019-03-26 14:48:46 +00001027 print(
1028 self.format_task_output(task_item, 'interrupted'), file=sys.stderr)
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001029 raise
1030 except Exception:
Raul Tambreb946b232019-03-26 14:48:46 +00001031 print(self.format_task_output(task_item, 'ERROR'), file=sys.stderr)
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001032 raise
1033
maruel@chromium.org3742c842010-09-09 19:27:14 +00001034
maruel@chromium.org9e5317a2010-08-13 20:35:11 +00001035 class _Worker(threading.Thread):
1036 """One thread to execute one WorkItem."""
maruel@chromium.org4ed34182010-09-17 15:57:47 +00001037 def __init__(self, item, index, args, kwargs):
maruel@chromium.org9e5317a2010-08-13 20:35:11 +00001038 threading.Thread.__init__(self, name=item.name or 'Worker')
maruel@chromium.org1333cb32011-10-04 23:40:16 +00001039 logging.info('_Worker(%s) reqs:%s' % (item.name, item.requirements))
maruel@chromium.org9e5317a2010-08-13 20:35:11 +00001040 self.item = item
maruel@chromium.org4ed34182010-09-17 15:57:47 +00001041 self.index = index
maruel@chromium.org3742c842010-09-09 19:27:14 +00001042 self.args = args
1043 self.kwargs = kwargs
xusydoc@chromium.org2fd6c3f2013-05-03 21:57:55 +00001044 self.daemon = True
maruel@chromium.org80cbe8b2010-08-13 13:53:07 +00001045
maruel@chromium.org9e5317a2010-08-13 20:35:11 +00001046 def run(self):
1047 """Runs in its own thread."""
maruel@chromium.org1333cb32011-10-04 23:40:16 +00001048 logging.debug('_Worker.run(%s)' % self.item.name)
maruel@chromium.org3742c842010-09-09 19:27:14 +00001049 work_queue = self.kwargs['work_queue']
maruel@chromium.org9e5317a2010-08-13 20:35:11 +00001050 try:
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001051 self.item.start = datetime.datetime.now()
Raul Tambreb946b232019-03-26 14:48:46 +00001052 print('[%s] Started.' % Elapsed(self.item.start), file=self.item.outbuf)
maruel@chromium.org9e5317a2010-08-13 20:35:11 +00001053 self.item.run(*self.args, **self.kwargs)
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001054 self.item.finish = datetime.datetime.now()
Raul Tambreb946b232019-03-26 14:48:46 +00001055 print(
1056 '[%s] Finished.' % Elapsed(self.item.finish), file=self.item.outbuf)
xusydoc@chromium.org2fd6c3f2013-05-03 21:57:55 +00001057 except KeyboardInterrupt:
xusydoc@chromium.orgc144e062013-05-03 23:23:53 +00001058 logging.info('Caught KeyboardInterrupt in thread %s', self.item.name)
xusydoc@chromium.org2fd6c3f2013-05-03 21:57:55 +00001059 logging.info(str(sys.exc_info()))
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001060 work_queue.exceptions.put((sys.exc_info(), self))
xusydoc@chromium.org2fd6c3f2013-05-03 21:57:55 +00001061 raise
maruel@chromium.orgc8d064b2010-08-16 16:46:14 +00001062 except Exception:
1063 # Catch exception location.
xusydoc@chromium.orgc144e062013-05-03 23:23:53 +00001064 logging.info('Caught exception in thread %s', self.item.name)
maruel@chromium.org3742c842010-09-09 19:27:14 +00001065 logging.info(str(sys.exc_info()))
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001066 work_queue.exceptions.put((sys.exc_info(), self))
maruel@chromium.org9e5317a2010-08-13 20:35:11 +00001067 finally:
xusydoc@chromium.orgc144e062013-05-03 23:23:53 +00001068 logging.info('_Worker.run(%s) done', self.item.name)
xusydoc@chromium.org2fd6c3f2013-05-03 21:57:55 +00001069 work_queue.ready_cond.acquire()
1070 try:
1071 work_queue.ready_cond.notifyAll()
1072 finally:
1073 work_queue.ready_cond.release()
maruel@chromium.org0e0436a2011-10-25 13:32:41 +00001074
1075
agable92bec4f2016-08-24 09:27:27 -07001076def GetEditor(git_editor=None):
jbroman@chromium.org615a2622013-05-03 13:20:14 +00001077 """Returns the most plausible editor to use.
1078
1079 In order of preference:
agable41e3a6c2016-10-20 11:36:56 -07001080 - GIT_EDITOR environment variable
jbroman@chromium.org615a2622013-05-03 13:20:14 +00001081 - core.editor git configuration variable (if supplied by git-cl)
1082 - VISUAL environment variable
1083 - EDITOR environment variable
bratell@opera.com65621c72013-12-09 15:05:32 +00001084 - vi (non-Windows) or notepad (Windows)
jbroman@chromium.org615a2622013-05-03 13:20:14 +00001085
1086 In the case of git-cl, this matches git's behaviour, except that it does not
1087 include dumb terminal detection.
jbroman@chromium.org615a2622013-05-03 13:20:14 +00001088 """
agable92bec4f2016-08-24 09:27:27 -07001089 editor = os.environ.get('GIT_EDITOR') or git_editor
maruel@chromium.org0e0436a2011-10-25 13:32:41 +00001090 if not editor:
jbroman@chromium.org615a2622013-05-03 13:20:14 +00001091 editor = os.environ.get('VISUAL')
1092 if not editor:
maruel@chromium.org0e0436a2011-10-25 13:32:41 +00001093 editor = os.environ.get('EDITOR')
1094 if not editor:
1095 if sys.platform.startswith('win'):
1096 editor = 'notepad'
1097 else:
bratell@opera.com65621c72013-12-09 15:05:32 +00001098 editor = 'vi'
maruel@chromium.org0e0436a2011-10-25 13:32:41 +00001099 return editor
1100
1101
jbroman@chromium.org615a2622013-05-03 13:20:14 +00001102def RunEditor(content, git, git_editor=None):
maruel@chromium.org0e0436a2011-10-25 13:32:41 +00001103 """Opens up the default editor in the system to get the CL description."""
maruel@chromium.orgcbd760f2013-07-23 13:02:48 +00001104 file_handle, filename = tempfile.mkstemp(text=True, prefix='cl_description')
maruel@chromium.org0e0436a2011-10-25 13:32:41 +00001105 # Make sure CRLF is handled properly by requiring none.
1106 if '\r' in content:
Raul Tambreb946b232019-03-26 14:48:46 +00001107 print(
1108 '!! Please remove \\r from your change description !!', file=sys.stderr)
sokcevic07152802021-08-18 00:06:34 +00001109 fileobj = os.fdopen(file_handle, 'wb')
maruel@chromium.org0e0436a2011-10-25 13:32:41 +00001110 # Still remove \r if present.
gab@chromium.orga3fe2902016-04-20 18:31:37 +00001111 content = re.sub('\r?\n', '\n', content)
1112 # Some editors complain when the file doesn't end in \n.
1113 if not content.endswith('\n'):
1114 content += '\n'
sokcevic07152802021-08-18 00:06:34 +00001115 fileobj.write(content.encode('utf-8'))
maruel@chromium.org0e0436a2011-10-25 13:32:41 +00001116 fileobj.close()
1117
1118 try:
agable92bec4f2016-08-24 09:27:27 -07001119 editor = GetEditor(git_editor=git_editor)
jbroman@chromium.org615a2622013-05-03 13:20:14 +00001120 if not editor:
1121 return None
1122 cmd = '%s %s' % (editor, filename)
maruel@chromium.org0e0436a2011-10-25 13:32:41 +00001123 if sys.platform == 'win32' and os.environ.get('TERM') == 'msys':
1124 # Msysgit requires the usage of 'env' to be present.
1125 cmd = 'env ' + cmd
1126 try:
1127 # shell=True to allow the shell to handle all forms of quotes in
1128 # $EDITOR.
1129 subprocess2.check_call(cmd, shell=True)
1130 except subprocess2.CalledProcessError:
1131 return None
1132 return FileRead(filename)
1133 finally:
1134 os.remove(filename)
maruel@chromium.org99ac1c52012-01-16 14:52:12 +00001135
1136
maruel@chromium.orgeb5edbc2012-01-16 17:03:28 +00001137def UpgradeToHttps(url):
1138 """Upgrades random urls to https://.
1139
1140 Do not touch unknown urls like ssh:// or git://.
1141 Do not touch http:// urls with a port number,
1142 Fixes invalid GAE url.
1143 """
1144 if not url:
1145 return url
1146 if not re.match(r'[a-z\-]+\://.*', url):
1147 # Make sure it is a valid uri. Otherwise, urlparse() will consider it a
1148 # relative url and will use http:///foo. Note that it defaults to http://
1149 # for compatibility with naked url like "localhost:8080".
1150 url = 'http://%s' % url
1151 parsed = list(urlparse.urlparse(url))
1152 # Do not automatically upgrade http to https if a port number is provided.
1153 if parsed[0] == 'http' and not re.match(r'^.+?\:\d+$', parsed[1]):
1154 parsed[0] = 'https'
maruel@chromium.orgeb5edbc2012-01-16 17:03:28 +00001155 return urlparse.urlunparse(parsed)
1156
1157
maruel@chromium.org99ac1c52012-01-16 14:52:12 +00001158def ParseCodereviewSettingsContent(content):
1159 """Process a codereview.settings file properly."""
1160 lines = (l for l in content.splitlines() if not l.strip().startswith("#"))
1161 try:
1162 keyvals = dict([x.strip() for x in l.split(':', 1)] for l in lines if l)
1163 except ValueError:
1164 raise Error(
1165 'Failed to process settings, please fix. Content:\n\n%s' % content)
maruel@chromium.orgeb5edbc2012-01-16 17:03:28 +00001166 def fix_url(key):
1167 if keyvals.get(key):
1168 keyvals[key] = UpgradeToHttps(keyvals[key])
1169 fix_url('CODE_REVIEW_SERVER')
1170 fix_url('VIEW_VC')
maruel@chromium.org99ac1c52012-01-16 14:52:12 +00001171 return keyvals
ilevy@chromium.org13691502012-10-16 04:26:37 +00001172
1173
1174def NumLocalCpus():
1175 """Returns the number of processors.
1176
dnj@chromium.org530523b2015-01-07 19:54:57 +00001177 multiprocessing.cpu_count() is permitted to raise NotImplementedError, and
1178 is known to do this on some Windows systems and OSX 10.6. If we can't get the
1179 CPU count, we will fall back to '1'.
ilevy@chromium.org13691502012-10-16 04:26:37 +00001180 """
dnj@chromium.org530523b2015-01-07 19:54:57 +00001181 # Surround the entire thing in try/except; no failure here should stop gclient
1182 # from working.
ilevy@chromium.org13691502012-10-16 04:26:37 +00001183 try:
dnj@chromium.org530523b2015-01-07 19:54:57 +00001184 # Use multiprocessing to get CPU count. This may raise
1185 # NotImplementedError.
1186 try:
1187 import multiprocessing
1188 return multiprocessing.cpu_count()
Quinten Yearsleyb2cc4a92016-12-15 13:53:26 -08001189 except NotImplementedError: # pylint: disable=bare-except
dnj@chromium.org530523b2015-01-07 19:54:57 +00001190 # (UNIX) Query 'os.sysconf'.
Quinten Yearsleyb2cc4a92016-12-15 13:53:26 -08001191 # pylint: disable=no-member
dnj@chromium.org530523b2015-01-07 19:54:57 +00001192 if hasattr(os, 'sysconf') and 'SC_NPROCESSORS_ONLN' in os.sysconf_names:
1193 return int(os.sysconf('SC_NPROCESSORS_ONLN'))
1194
1195 # (Windows) Query 'NUMBER_OF_PROCESSORS' environment variable.
1196 if 'NUMBER_OF_PROCESSORS' in os.environ:
1197 return int(os.environ['NUMBER_OF_PROCESSORS'])
1198 except Exception as e:
1199 logging.exception("Exception raised while probing CPU count: %s", e)
1200
1201 logging.debug('Failed to get CPU count. Defaulting to 1.')
1202 return 1
szager@chromium.orgfc616382014-03-18 20:32:04 +00001203
sbc@chromium.org9d0644d2015-06-05 23:16:54 +00001204
szager@chromium.orgfc616382014-03-18 20:32:04 +00001205def DefaultDeltaBaseCacheLimit():
1206 """Return a reasonable default for the git config core.deltaBaseCacheLimit.
1207
1208 The primary constraint is the address space of virtual memory. The cache
1209 size limit is per-thread, and 32-bit systems can hit OOM errors if this
1210 parameter is set too high.
1211 """
1212 if platform.architecture()[0].startswith('64'):
1213 return '2g'
1214 else:
1215 return '512m'
1216
sbc@chromium.org9d0644d2015-06-05 23:16:54 +00001217
szager@chromium.orgff113292014-03-25 06:02:08 +00001218def DefaultIndexPackConfig(url=''):
szager@chromium.orgfc616382014-03-18 20:32:04 +00001219 """Return reasonable default values for configuring git-index-pack.
1220
1221 Experiments suggest that higher values for pack.threads don't improve
1222 performance."""
szager@chromium.orgff113292014-03-25 06:02:08 +00001223 cache_limit = DefaultDeltaBaseCacheLimit()
1224 result = ['-c', 'core.deltaBaseCacheLimit=%s' % cache_limit]
Ayu Ishii09858612020-06-26 18:00:52 +00001225 if url in THREADED_INDEX_PACK_BLOCKLIST:
szager@chromium.orgff113292014-03-25 06:02:08 +00001226 result.extend(['-c', 'pack.threads=1'])
1227 return result
sbc@chromium.org9d0644d2015-06-05 23:16:54 +00001228
1229
1230def FindExecutable(executable):
1231 """This mimics the "which" utility."""
1232 path_folders = os.environ.get('PATH').split(os.pathsep)
1233
1234 for path_folder in path_folders:
1235 target = os.path.join(path_folder, executable)
Quinten Yearsley925cedb2020-04-13 17:49:39 +00001236 # Just in case we have some ~/blah paths.
sbc@chromium.org9d0644d2015-06-05 23:16:54 +00001237 target = os.path.abspath(os.path.expanduser(target))
1238 if os.path.isfile(target) and os.access(target, os.X_OK):
1239 return target
1240 if sys.platform.startswith('win'):
1241 for suffix in ('.bat', '.cmd', '.exe'):
1242 alt_target = target + suffix
1243 if os.path.isfile(alt_target) and os.access(alt_target, os.X_OK):
1244 return alt_target
1245 return None
Paweł Hajdan, Jr7e502612017-06-12 16:58:38 +02001246
1247
1248def freeze(obj):
1249 """Takes a generic object ``obj``, and returns an immutable version of it.
1250
1251 Supported types:
1252 * dict / OrderedDict -> FrozenDict
1253 * list -> tuple
1254 * set -> frozenset
1255 * any object with a working __hash__ implementation (assumes that hashable
1256 means immutable)
1257
1258 Will raise TypeError if you pass an object which is not hashable.
1259 """
Raul Tambre6693d092020-02-19 20:36:45 +00001260 if isinstance(obj, collections_abc.Mapping):
Raul Tambreb946b232019-03-26 14:48:46 +00001261 return FrozenDict((freeze(k), freeze(v)) for k, v in obj.items())
Paweł Hajdan, Jr7e502612017-06-12 16:58:38 +02001262 elif isinstance(obj, (list, tuple)):
1263 return tuple(freeze(i) for i in obj)
1264 elif isinstance(obj, set):
1265 return frozenset(freeze(i) for i in obj)
1266 else:
1267 hash(obj)
1268 return obj
1269
1270
Raul Tambre6693d092020-02-19 20:36:45 +00001271class FrozenDict(collections_abc.Mapping):
Paweł Hajdan, Jr7e502612017-06-12 16:58:38 +02001272 """An immutable OrderedDict.
1273
1274 Modified From: http://stackoverflow.com/a/2704866
1275 """
1276 def __init__(self, *args, **kwargs):
1277 self._d = collections.OrderedDict(*args, **kwargs)
1278
1279 # Calculate the hash immediately so that we know all the items are
1280 # hashable too.
Raul Tambreb946b232019-03-26 14:48:46 +00001281 self._hash = functools.reduce(
1282 operator.xor, (hash(i) for i in enumerate(self._d.items())), 0)
Paweł Hajdan, Jr7e502612017-06-12 16:58:38 +02001283
1284 def __eq__(self, other):
Raul Tambre6693d092020-02-19 20:36:45 +00001285 if not isinstance(other, collections_abc.Mapping):
Paweł Hajdan, Jr7e502612017-06-12 16:58:38 +02001286 return NotImplemented
1287 if self is other:
1288 return True
1289 if len(self) != len(other):
1290 return False
Marc-Antoine Ruel8e57b4b2019-10-11 01:01:36 +00001291 for k, v in self.items():
Paweł Hajdan, Jr7e502612017-06-12 16:58:38 +02001292 if k not in other or other[k] != v:
1293 return False
1294 return True
1295
1296 def __iter__(self):
1297 return iter(self._d)
1298
1299 def __len__(self):
1300 return len(self._d)
1301
1302 def __getitem__(self, key):
1303 return self._d[key]
1304
1305 def __hash__(self):
1306 return self._hash
1307
1308 def __repr__(self):
1309 return 'FrozenDict(%r)' % (self._d.items(),)