Improve gclient Python 3 compatibility

This enables gclient sync and gclient runhooks to run, barring hook script failures.
git cl upload also now works.

The scripts still work with Python 2.
There are no intended behaviour changes.

Bug: 942522
Change-Id: I2ac587b5f803ba7f5bb5e412337ce049f4b1a741
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1524583
Commit-Queue: Raul Tambre <raul@tambre.ee>
Reviewed-by: Dirk Pranke <dpranke@chromium.org>
diff --git a/gclient_utils.py b/gclient_utils.py
index 48c023f..ae9d721 100644
--- a/gclient_utils.py
+++ b/gclient_utils.py
@@ -4,17 +4,25 @@
 
 """Generic utils."""
 
+from __future__ import print_function
+
 import codecs
 import collections
 import contextlib
-import cStringIO
 import datetime
+import functools
+import io
 import logging
 import operator
 import os
 import pipes
 import platform
-import Queue
+
+try:
+  import Queue as queue
+except ImportError:  # For Py3 compatibility
+  import queue
+
 import re
 import stat
 import subprocess
@@ -22,10 +30,19 @@
 import tempfile
 import threading
 import time
-import urlparse
+
+try:
+  import urlparse
+except ImportError:  # For Py3 compatibility
+  import urllib.parse as urlparse
 
 import subprocess2
 
+if sys.version_info.major == 2:
+  from cStringIO import StringIO
+else:
+  from io import StringIO
+
 
 RETRY_MAX = 3
 RETRY_INITIAL_SLEEP = 0.5
@@ -42,6 +59,18 @@
   'https://chromium.googlesource.com/chromium/reference_builds/chrome_win.git'
 ]
 
+"""To support rethrowing exceptions with tracebacks on both Py2 and 3."""
+if sys.version_info.major == 2:
+  # We have to use exec to avoid a SyntaxError in Python 3.
+  exec("def reraise(typ, value, tb=None):\n  raise typ, value, tb\n")
+else:
+  def reraise(typ, value, tb=None):
+    if value is None:
+      value = typ()
+    if value.__traceback__ is not tb:
+      raise value.with_traceback(tb)
+    raise value
+
 
 class Error(Exception):
   """gclient exception class."""
@@ -61,9 +90,9 @@
 def PrintWarnings():
   """Prints any accumulated warnings."""
   if _WARNINGS:
-    print >> sys.stderr, '\n\nWarnings:'
+    print('\n\nWarnings:', file=sys.stderr)
     for warning in _WARNINGS:
-      print >> sys.stderr, warning
+      print(warning, file=sys.stderr)
 
 
 def AddWarning(msg):
@@ -142,7 +171,8 @@
     s = f.read()
     try:
       return s.decode('utf-8')
-    except UnicodeDecodeError:
+    # AttributeError is for Py3 compatibility
+    except (UnicodeDecodeError, AttributeError):
       return s
 
 
@@ -230,7 +260,7 @@
       if exitcode == 0:
         return
       else:
-        print >> sys.stderr, 'rd exited with code %d' % exitcode
+        print('rd exited with code %d' % exitcode, file=sys.stderr)
       time.sleep(3)
     raise Exception('Failed to remove path %s' % path)
 
@@ -268,7 +298,7 @@
     count += 1
     try:
       os.makedirs(tree)
-    except OSError, e:
+    except OSError as e:
       # 17 POSIX, 183 Windows
       if e.errno not in (17, 183):
         raise
@@ -491,9 +521,9 @@
 
     with GCLIENT_CHILDREN_LOCK:
       if GCLIENT_CHILDREN:
-        print >> sys.stderr, 'Could not kill the following subprocesses:'
+        print('Could not kill the following subprocesses:', file=sys.stderr)
         for zombie in GCLIENT_CHILDREN:
-          print >> sys.stderr, '  ', zombie.pid
+          print('  ', zombie.pid, file=sys.stderr)
 
 
 def CheckCallAndFilter(args, stdout=None, filter_fn=None,
@@ -514,12 +544,12 @@
   """
   assert print_stdout or filter_fn
   stdout = stdout or sys.stdout
-  output = cStringIO.StringIO()
+  output = io.BytesIO()
   filter_fn = filter_fn or (lambda x: None)
 
   sleep_interval = RETRY_INITIAL_SLEEP
   run_cwd = kwargs.get('cwd', os.getcwd())
-  for _ in xrange(RETRY_MAX + 1):
+  for _ in range(RETRY_MAX + 1):
     kid = subprocess2.Popen(
         args, bufsize=0, stdout=subprocess2.PIPE, stderr=subprocess2.STDOUT,
         **kwargs)
@@ -539,16 +569,16 @@
       if in_byte:
         if call_filter_on_first_line:
           filter_fn(None)
-        in_line = ''
+        in_line = b''
         while in_byte:
           output.write(in_byte)
           if print_stdout:
-            stdout.write(in_byte)
+            stdout.write(in_byte.decode())
           if in_byte not in ['\r', '\n']:
             in_line += in_byte
           else:
             filter_fn(in_line)
-            in_line = ''
+            in_line = b''
           in_byte = kid.stdout.read(1)
         # Flush the rest of buffered output. This is only an issue with
         # stdout/stderr not ending with a \n.
@@ -561,15 +591,15 @@
       GClientChildren.remove(kid)
 
     except KeyboardInterrupt:
-      print >> sys.stderr, 'Failed while running "%s"' % ' '.join(args)
+      print('Failed while running "%s"' % ' '.join(args), file=sys.stderr)
       raise
 
     if rv == 0:
       return output.getvalue()
     if not retry:
       break
-    print ("WARNING: subprocess '%s' in %s failed; will retry after a short "
-           'nap...' % (' '.join('"%s"' % x for x in args), run_cwd))
+    print("WARNING: subprocess '%s' in %s failed; will retry after a short "
+          'nap...' % (' '.join('"%s"' % x for x in args), run_cwd))
     time.sleep(sleep_interval)
     sleep_interval *= 2
   raise subprocess2.CalledProcessError(
@@ -602,13 +632,13 @@
 
   def __call__(self, line):
     # git uses an escape sequence to clear the line; elide it.
-    esc = line.find(unichr(033))
+    esc = line.find(chr(0o33).encode())
     if esc > -1:
       line = line[:esc]
     if self.predicate and not self.predicate(line):
       return
     now = time.time()
-    match = self.PERCENT_RE.match(line)
+    match = self.PERCENT_RE.match(line.decode())
     if match:
       if match.group(1) != self.progress_prefix:
         self.progress_prefix = match.group(1)
@@ -616,7 +646,7 @@
         return
     self.last_time = now
     self.out_fh.write('[%s] ' % Elapsed())
-    print >> self.out_fh, line
+    print(line, file=self.out_fh)
 
 
 def FindFileUpwards(filename, path=None):
@@ -653,7 +683,7 @@
   config_file = '.gclient_entries'
   root = FindFileUpwards(config_file, path)
   if not root:
-    print "Can't find %s" % config_file
+    print("Can't find %s" % config_file)
     return None
   config_path = os.path.join(root, config_file)
   env = {}
@@ -669,7 +699,7 @@
       try:
         self.lock.acquire()
       except KeyboardInterrupt:
-        print >> sys.stderr, 'Was deadlocked'
+        print('Was deadlocked', file=sys.stderr)
         raise
       return method(self, *args, **kwargs)
     finally:
@@ -687,7 +717,7 @@
   def __init__(self, name):
     # A unique string representing this work item.
     self._name = name
-    self.outbuf = cStringIO.StringIO()
+    self.outbuf = StringIO()
     self.start = self.finish = None
     self.resources = []  # List of resources this work item requires.
 
@@ -724,7 +754,7 @@
     # List of items currently running.
     self.running = []
     # Exceptions thrown if any.
-    self.exceptions = Queue.Queue()
+    self.exceptions = queue.Queue()
     # Progress status
     self.progress = progress
     if self.progress:
@@ -802,7 +832,7 @@
             break
 
           # Check for new tasks to start.
-          for i in xrange(len(self.queued)):
+          for i in range(len(self.queued)):
             # Verify its requirements.
             if (self.ignore_requirements or
                 not (set(self.queued[i].requirements) - set(self.ran))):
@@ -826,28 +856,28 @@
           if (now - self.last_join > datetime.timedelta(seconds=60) and
               self.last_subproc_output > self.last_join):
             if self.progress:
-              print >> sys.stdout, ''
+              print('')
               sys.stdout.flush()
             elapsed = Elapsed()
-            print >> sys.stdout, '[%s] Still working on:' % elapsed
+            print('[%s] Still working on:' % elapsed)
             sys.stdout.flush()
             for task in self.running:
-              print >> sys.stdout, '[%s]   %s' % (elapsed, task.item.name)
+              print('[%s]   %s' % (elapsed, task.item.name))
               sys.stdout.flush()
         except KeyboardInterrupt:
           # Help debugging by printing some information:
-          print >> sys.stderr, (
+          print(
               ('\nAllowed parallel jobs: %d\n# queued: %d\nRan: %s\n'
-                'Running: %d') % (
-              self.jobs,
-              len(self.queued),
-              ', '.join(self.ran),
-              len(self.running)))
+               'Running: %d') % (self.jobs, len(self.queued), ', '.join(
+                   self.ran), len(self.running)),
+              file=sys.stderr)
           for i in self.queued:
-            print >> sys.stderr, '%s (not started): %s' % (
-                i.name, ', '.join(i.requirements))
+            print(
+                '%s (not started): %s' % (i.name, ', '.join(i.requirements)),
+                file=sys.stderr)
           for i in self.running:
-            print >> sys.stderr, self.format_task_output(i.item, 'interrupted')
+            print(
+                self.format_task_output(i.item, 'interrupted'), file=sys.stderr)
           raise
         # Something happened: self.enqueue() or a thread terminated. Loop again.
     finally:
@@ -856,12 +886,12 @@
     assert not self.running, 'Now guaranteed to be single-threaded'
     if not self.exceptions.empty():
       if self.progress:
-        print >> sys.stdout, ''
+        print('')
       # To get back the stack location correctly, the raise a, b, c form must be
       # used, passing a tuple as the first argument doesn't work.
       e, task = self.exceptions.get()
-      print >> sys.stderr, self.format_task_output(task.item, 'ERROR')
-      raise e[0], e[1], e[2]
+      print(self.format_task_output(task.item, 'ERROR'), file=sys.stderr)
+      reraise(e[0], e[1], e[2])
     elif self.progress:
       self.progress.end()
 
@@ -877,7 +907,7 @@
         self.last_join = datetime.datetime.now()
         sys.stdout.flush()
         if self.verbose:
-          print >> sys.stdout, self.format_task_output(t.item)
+          print(self.format_task_output(t.item))
         if self.progress:
           self.progress.update(1, t.item.name)
         if t.item.name in self.ran:
@@ -899,22 +929,24 @@
       # exception.
       try:
         task_item.start = datetime.datetime.now()
-        print >> task_item.outbuf, '[%s] Started.' % Elapsed(task_item.start)
+        print('[%s] Started.' % Elapsed(task_item.start), file=task_item.outbuf)
         task_item.run(*args, **kwargs)
         task_item.finish = datetime.datetime.now()
-        print >> task_item.outbuf, '[%s] Finished.' % Elapsed(task_item.finish)
+        print(
+            '[%s] Finished.' % Elapsed(task_item.finish), file=task_item.outbuf)
         self.ran.append(task_item.name)
         if self.verbose:
           if self.progress:
-            print >> sys.stdout, ''
-          print >> sys.stdout, self.format_task_output(task_item)
+            print('')
+          print(self.format_task_output(task_item))
         if self.progress:
           self.progress.update(1, ', '.join(t.item.name for t in self.running))
       except KeyboardInterrupt:
-        print >> sys.stderr, self.format_task_output(task_item, 'interrupted')
+        print(
+            self.format_task_output(task_item, 'interrupted'), file=sys.stderr)
         raise
       except Exception:
-        print >> sys.stderr, self.format_task_output(task_item, 'ERROR')
+        print(self.format_task_output(task_item, 'ERROR'), file=sys.stderr)
         raise
 
 
@@ -935,10 +967,11 @@
       work_queue = self.kwargs['work_queue']
       try:
         self.item.start = datetime.datetime.now()
-        print >> self.item.outbuf, '[%s] Started.' % Elapsed(self.item.start)
+        print('[%s] Started.' % Elapsed(self.item.start), file=self.item.outbuf)
         self.item.run(*self.args, **self.kwargs)
         self.item.finish = datetime.datetime.now()
-        print >> self.item.outbuf, '[%s] Finished.' % Elapsed(self.item.finish)
+        print(
+            '[%s] Finished.' % Elapsed(self.item.finish), file=self.item.outbuf)
       except KeyboardInterrupt:
         logging.info('Caught KeyboardInterrupt in thread %s', self.item.name)
         logging.info(str(sys.exc_info()))
@@ -989,8 +1022,8 @@
   file_handle, filename = tempfile.mkstemp(text=True, prefix='cl_description')
   # Make sure CRLF is handled properly by requiring none.
   if '\r' in content:
-    print >> sys.stderr, (
-        '!! Please remove \\r from your change description !!')
+    print(
+        '!! Please remove \\r from your change description !!', file=sys.stderr)
   fileobj = os.fdopen(file_handle, 'w')
   # Still remove \r if present.
   content = re.sub('\r?\n', '\n', content)
@@ -1143,7 +1176,7 @@
   Will raise TypeError if you pass an object which is not hashable.
   """
   if isinstance(obj, collections.Mapping):
-    return FrozenDict((freeze(k), freeze(v)) for k, v in obj.iteritems())
+    return FrozenDict((freeze(k), freeze(v)) for k, v in obj.items())
   elif isinstance(obj, (list, tuple)):
     return tuple(freeze(i) for i in obj)
   elif isinstance(obj, set):
@@ -1163,8 +1196,8 @@
 
     # Calculate the hash immediately so that we know all the items are
     # hashable too.
-    self._hash = reduce(operator.xor,
-                        (hash(i) for i in enumerate(self._d.iteritems())), 0)
+    self._hash = functools.reduce(
+        operator.xor, (hash(i) for i in enumerate(self._d.items())), 0)
 
   def __eq__(self, other):
     if not isinstance(other, collections.Mapping):