Improve gclient Python 3 compatibility
This enables gclient sync and gclient runhooks to run, barring hook script failures.
git cl upload also now works.
The scripts still work with Python 2.
There are no intended behaviour changes.
Bug: 942522
Change-Id: I2ac587b5f803ba7f5bb5e412337ce049f4b1a741
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1524583
Commit-Queue: Raul Tambre <raul@tambre.ee>
Reviewed-by: Dirk Pranke <dpranke@chromium.org>
diff --git a/gclient_utils.py b/gclient_utils.py
index 48c023f..ae9d721 100644
--- a/gclient_utils.py
+++ b/gclient_utils.py
@@ -4,17 +4,25 @@
"""Generic utils."""
+from __future__ import print_function
+
import codecs
import collections
import contextlib
-import cStringIO
import datetime
+import functools
+import io
import logging
import operator
import os
import pipes
import platform
-import Queue
+
+try:
+ import Queue as queue
+except ImportError: # For Py3 compatibility
+ import queue
+
import re
import stat
import subprocess
@@ -22,10 +30,19 @@
import tempfile
import threading
import time
-import urlparse
+
+try:
+ import urlparse
+except ImportError: # For Py3 compatibility
+ import urllib.parse as urlparse
import subprocess2
+if sys.version_info.major == 2:
+ from cStringIO import StringIO
+else:
+ from io import StringIO
+
RETRY_MAX = 3
RETRY_INITIAL_SLEEP = 0.5
@@ -42,6 +59,18 @@
'https://chromium.googlesource.com/chromium/reference_builds/chrome_win.git'
]
+"""To support rethrowing exceptions with tracebacks on both Py2 and 3."""
+if sys.version_info.major == 2:
+ # We have to use exec to avoid a SyntaxError in Python 3.
+ exec("def reraise(typ, value, tb=None):\n raise typ, value, tb\n")
+else:
+ def reraise(typ, value, tb=None):
+ if value is None:
+ value = typ()
+ if value.__traceback__ is not tb:
+ raise value.with_traceback(tb)
+ raise value
+
class Error(Exception):
"""gclient exception class."""
@@ -61,9 +90,9 @@
def PrintWarnings():
"""Prints any accumulated warnings."""
if _WARNINGS:
- print >> sys.stderr, '\n\nWarnings:'
+ print('\n\nWarnings:', file=sys.stderr)
for warning in _WARNINGS:
- print >> sys.stderr, warning
+ print(warning, file=sys.stderr)
def AddWarning(msg):
@@ -142,7 +171,8 @@
s = f.read()
try:
return s.decode('utf-8')
- except UnicodeDecodeError:
+ # AttributeError is for Py3 compatibility
+ except (UnicodeDecodeError, AttributeError):
return s
@@ -230,7 +260,7 @@
if exitcode == 0:
return
else:
- print >> sys.stderr, 'rd exited with code %d' % exitcode
+ print('rd exited with code %d' % exitcode, file=sys.stderr)
time.sleep(3)
raise Exception('Failed to remove path %s' % path)
@@ -268,7 +298,7 @@
count += 1
try:
os.makedirs(tree)
- except OSError, e:
+ except OSError as e:
# 17 POSIX, 183 Windows
if e.errno not in (17, 183):
raise
@@ -491,9 +521,9 @@
with GCLIENT_CHILDREN_LOCK:
if GCLIENT_CHILDREN:
- print >> sys.stderr, 'Could not kill the following subprocesses:'
+ print('Could not kill the following subprocesses:', file=sys.stderr)
for zombie in GCLIENT_CHILDREN:
- print >> sys.stderr, ' ', zombie.pid
+ print(' ', zombie.pid, file=sys.stderr)
def CheckCallAndFilter(args, stdout=None, filter_fn=None,
@@ -514,12 +544,12 @@
"""
assert print_stdout or filter_fn
stdout = stdout or sys.stdout
- output = cStringIO.StringIO()
+ output = io.BytesIO()
filter_fn = filter_fn or (lambda x: None)
sleep_interval = RETRY_INITIAL_SLEEP
run_cwd = kwargs.get('cwd', os.getcwd())
- for _ in xrange(RETRY_MAX + 1):
+ for _ in range(RETRY_MAX + 1):
kid = subprocess2.Popen(
args, bufsize=0, stdout=subprocess2.PIPE, stderr=subprocess2.STDOUT,
**kwargs)
@@ -539,16 +569,16 @@
if in_byte:
if call_filter_on_first_line:
filter_fn(None)
- in_line = ''
+ in_line = b''
while in_byte:
output.write(in_byte)
if print_stdout:
- stdout.write(in_byte)
+ stdout.write(in_byte.decode())
if in_byte not in ['\r', '\n']:
in_line += in_byte
else:
filter_fn(in_line)
- in_line = ''
+ in_line = b''
in_byte = kid.stdout.read(1)
# Flush the rest of buffered output. This is only an issue with
# stdout/stderr not ending with a \n.
@@ -561,15 +591,15 @@
GClientChildren.remove(kid)
except KeyboardInterrupt:
- print >> sys.stderr, 'Failed while running "%s"' % ' '.join(args)
+ print('Failed while running "%s"' % ' '.join(args), file=sys.stderr)
raise
if rv == 0:
return output.getvalue()
if not retry:
break
- print ("WARNING: subprocess '%s' in %s failed; will retry after a short "
- 'nap...' % (' '.join('"%s"' % x for x in args), run_cwd))
+ print("WARNING: subprocess '%s' in %s failed; will retry after a short "
+ 'nap...' % (' '.join('"%s"' % x for x in args), run_cwd))
time.sleep(sleep_interval)
sleep_interval *= 2
raise subprocess2.CalledProcessError(
@@ -602,13 +632,13 @@
def __call__(self, line):
# git uses an escape sequence to clear the line; elide it.
- esc = line.find(unichr(033))
+ esc = line.find(chr(0o33).encode())
if esc > -1:
line = line[:esc]
if self.predicate and not self.predicate(line):
return
now = time.time()
- match = self.PERCENT_RE.match(line)
+ match = self.PERCENT_RE.match(line.decode())
if match:
if match.group(1) != self.progress_prefix:
self.progress_prefix = match.group(1)
@@ -616,7 +646,7 @@
return
self.last_time = now
self.out_fh.write('[%s] ' % Elapsed())
- print >> self.out_fh, line
+ print(line, file=self.out_fh)
def FindFileUpwards(filename, path=None):
@@ -653,7 +683,7 @@
config_file = '.gclient_entries'
root = FindFileUpwards(config_file, path)
if not root:
- print "Can't find %s" % config_file
+ print("Can't find %s" % config_file)
return None
config_path = os.path.join(root, config_file)
env = {}
@@ -669,7 +699,7 @@
try:
self.lock.acquire()
except KeyboardInterrupt:
- print >> sys.stderr, 'Was deadlocked'
+ print('Was deadlocked', file=sys.stderr)
raise
return method(self, *args, **kwargs)
finally:
@@ -687,7 +717,7 @@
def __init__(self, name):
# A unique string representing this work item.
self._name = name
- self.outbuf = cStringIO.StringIO()
+ self.outbuf = StringIO()
self.start = self.finish = None
self.resources = [] # List of resources this work item requires.
@@ -724,7 +754,7 @@
# List of items currently running.
self.running = []
# Exceptions thrown if any.
- self.exceptions = Queue.Queue()
+ self.exceptions = queue.Queue()
# Progress status
self.progress = progress
if self.progress:
@@ -802,7 +832,7 @@
break
# Check for new tasks to start.
- for i in xrange(len(self.queued)):
+ for i in range(len(self.queued)):
# Verify its requirements.
if (self.ignore_requirements or
not (set(self.queued[i].requirements) - set(self.ran))):
@@ -826,28 +856,28 @@
if (now - self.last_join > datetime.timedelta(seconds=60) and
self.last_subproc_output > self.last_join):
if self.progress:
- print >> sys.stdout, ''
+ print('')
sys.stdout.flush()
elapsed = Elapsed()
- print >> sys.stdout, '[%s] Still working on:' % elapsed
+ print('[%s] Still working on:' % elapsed)
sys.stdout.flush()
for task in self.running:
- print >> sys.stdout, '[%s] %s' % (elapsed, task.item.name)
+ print('[%s] %s' % (elapsed, task.item.name))
sys.stdout.flush()
except KeyboardInterrupt:
# Help debugging by printing some information:
- print >> sys.stderr, (
+ print(
('\nAllowed parallel jobs: %d\n# queued: %d\nRan: %s\n'
- 'Running: %d') % (
- self.jobs,
- len(self.queued),
- ', '.join(self.ran),
- len(self.running)))
+ 'Running: %d') % (self.jobs, len(self.queued), ', '.join(
+ self.ran), len(self.running)),
+ file=sys.stderr)
for i in self.queued:
- print >> sys.stderr, '%s (not started): %s' % (
- i.name, ', '.join(i.requirements))
+ print(
+ '%s (not started): %s' % (i.name, ', '.join(i.requirements)),
+ file=sys.stderr)
for i in self.running:
- print >> sys.stderr, self.format_task_output(i.item, 'interrupted')
+ print(
+ self.format_task_output(i.item, 'interrupted'), file=sys.stderr)
raise
# Something happened: self.enqueue() or a thread terminated. Loop again.
finally:
@@ -856,12 +886,12 @@
assert not self.running, 'Now guaranteed to be single-threaded'
if not self.exceptions.empty():
if self.progress:
- print >> sys.stdout, ''
+ print('')
# To get back the stack location correctly, the raise a, b, c form must be
# used, passing a tuple as the first argument doesn't work.
e, task = self.exceptions.get()
- print >> sys.stderr, self.format_task_output(task.item, 'ERROR')
- raise e[0], e[1], e[2]
+ print(self.format_task_output(task.item, 'ERROR'), file=sys.stderr)
+ reraise(e[0], e[1], e[2])
elif self.progress:
self.progress.end()
@@ -877,7 +907,7 @@
self.last_join = datetime.datetime.now()
sys.stdout.flush()
if self.verbose:
- print >> sys.stdout, self.format_task_output(t.item)
+ print(self.format_task_output(t.item))
if self.progress:
self.progress.update(1, t.item.name)
if t.item.name in self.ran:
@@ -899,22 +929,24 @@
# exception.
try:
task_item.start = datetime.datetime.now()
- print >> task_item.outbuf, '[%s] Started.' % Elapsed(task_item.start)
+ print('[%s] Started.' % Elapsed(task_item.start), file=task_item.outbuf)
task_item.run(*args, **kwargs)
task_item.finish = datetime.datetime.now()
- print >> task_item.outbuf, '[%s] Finished.' % Elapsed(task_item.finish)
+ print(
+ '[%s] Finished.' % Elapsed(task_item.finish), file=task_item.outbuf)
self.ran.append(task_item.name)
if self.verbose:
if self.progress:
- print >> sys.stdout, ''
- print >> sys.stdout, self.format_task_output(task_item)
+ print('')
+ print(self.format_task_output(task_item))
if self.progress:
self.progress.update(1, ', '.join(t.item.name for t in self.running))
except KeyboardInterrupt:
- print >> sys.stderr, self.format_task_output(task_item, 'interrupted')
+ print(
+ self.format_task_output(task_item, 'interrupted'), file=sys.stderr)
raise
except Exception:
- print >> sys.stderr, self.format_task_output(task_item, 'ERROR')
+ print(self.format_task_output(task_item, 'ERROR'), file=sys.stderr)
raise
@@ -935,10 +967,11 @@
work_queue = self.kwargs['work_queue']
try:
self.item.start = datetime.datetime.now()
- print >> self.item.outbuf, '[%s] Started.' % Elapsed(self.item.start)
+ print('[%s] Started.' % Elapsed(self.item.start), file=self.item.outbuf)
self.item.run(*self.args, **self.kwargs)
self.item.finish = datetime.datetime.now()
- print >> self.item.outbuf, '[%s] Finished.' % Elapsed(self.item.finish)
+ print(
+ '[%s] Finished.' % Elapsed(self.item.finish), file=self.item.outbuf)
except KeyboardInterrupt:
logging.info('Caught KeyboardInterrupt in thread %s', self.item.name)
logging.info(str(sys.exc_info()))
@@ -989,8 +1022,8 @@
file_handle, filename = tempfile.mkstemp(text=True, prefix='cl_description')
# Make sure CRLF is handled properly by requiring none.
if '\r' in content:
- print >> sys.stderr, (
- '!! Please remove \\r from your change description !!')
+ print(
+ '!! Please remove \\r from your change description !!', file=sys.stderr)
fileobj = os.fdopen(file_handle, 'w')
# Still remove \r if present.
content = re.sub('\r?\n', '\n', content)
@@ -1143,7 +1176,7 @@
Will raise TypeError if you pass an object which is not hashable.
"""
if isinstance(obj, collections.Mapping):
- return FrozenDict((freeze(k), freeze(v)) for k, v in obj.iteritems())
+ return FrozenDict((freeze(k), freeze(v)) for k, v in obj.items())
elif isinstance(obj, (list, tuple)):
return tuple(freeze(i) for i in obj)
elif isinstance(obj, set):
@@ -1163,8 +1196,8 @@
# Calculate the hash immediately so that we know all the items are
# hashable too.
- self._hash = reduce(operator.xor,
- (hash(i) for i in enumerate(self._d.iteritems())), 0)
+ self._hash = functools.reduce(
+ operator.xor, (hash(i) for i in enumerate(self._d.items())), 0)
def __eq__(self, other):
if not isinstance(other, collections.Mapping):