switch to 4 space indent
Leave the recipes/ code at 2 space to match the rest of the recipes
project in other repos.
Reformatted using:
files=( $(
git ls-tree -r --name-only HEAD | \
grep -Ev -e '^(third_party|recipes)/' | \
grep '\.py$';
git grep -l '#!/usr/bin/env.*python' | grep -v '\.py$'
) )
parallel ./yapf -i -- "${files[@]}"
~/chromiumos/chromite/contrib/reflow_overlong_comments "${files[@]}"
The files that still had strings that were too long were manually
reformatted because they were easy and only a few issues.
autoninja.py
clang_format.py
download_from_google_storage.py
fix_encoding.py
gclient_utils.py
git_cache.py
git_common.py
git_map_branches.py
git_reparent_branch.py
gn.py
my_activity.py
owners_finder.py
presubmit_canned_checks.py
reclient_helper.py
reclientreport.py
roll_dep.py
rustfmt.py
siso.py
split_cl.py
subcommand.py
subprocess2.py
swift_format.py
upload_to_google_storage.py
These files still had lines (strings) that were too long, so the pylint
warnings were suppressed with a TODO.
auth.py
gclient.py
gclient_eval.py
gclient_paths.py
gclient_scm.py
gerrit_util.py
git_cl.py
presubmit_canned_checks.py
presubmit_support.py
scm.py
Change-Id: Ia6535c4f2c48d46b589ec1e791dde6c6b2ea858f
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/4836379
Commit-Queue: Josip Sokcevic <sokcevic@chromium.org>
Auto-Submit: Mike Frysinger <vapier@chromium.org>
Reviewed-by: Josip Sokcevic <sokcevic@chromium.org>
diff --git a/metrics.py b/metrics.py
index 4641bf2..3ccbfc9 100644
--- a/metrics.py
+++ b/metrics.py
@@ -20,7 +20,6 @@
import metrics_utils
import subprocess2
-
DEPOT_TOOLS = os.path.dirname(os.path.abspath(__file__))
CONFIG_FILE = os.path.join(DEPOT_TOOLS, 'metrics.cfg')
UPLOAD_SCRIPT = os.path.join(DEPOT_TOOLS, 'upload_metrics.py')
@@ -32,294 +31,297 @@
INVALID_CONFIG_WARNING = (
'WARNING: Your metrics.cfg file was invalid or nonexistent. A new one will '
- 'be created.'
-)
+ 'be created.')
PERMISSION_DENIED_WARNING = (
'Could not write the metrics collection config:\n\t%s\n'
- 'Metrics collection will be disabled.'
-)
+ 'Metrics collection will be disabled.')
class _Config(object):
- def __init__(self):
- self._initialized = False
- self._config = {}
+ def __init__(self):
+ self._initialized = False
+ self._config = {}
- def _ensure_initialized(self):
- if self._initialized:
- return
+ def _ensure_initialized(self):
+ if self._initialized:
+ return
- # Metrics collection is disabled, so don't collect any metrics.
- if not metrics_utils.COLLECT_METRICS:
- self._config = {
- 'is-googler': False,
- 'countdown': 0,
- 'opt-in': False,
- 'version': metrics_utils.CURRENT_VERSION,
- }
- self._initialized = True
- return
+ # Metrics collection is disabled, so don't collect any metrics.
+ if not metrics_utils.COLLECT_METRICS:
+ self._config = {
+ 'is-googler': False,
+ 'countdown': 0,
+ 'opt-in': False,
+ 'version': metrics_utils.CURRENT_VERSION,
+ }
+ self._initialized = True
+ return
- # We are running on a bot. Ignore config and collect metrics.
- if metrics_utils.REPORT_BUILD:
- self._config = {
- 'is-googler': True,
- 'countdown': 0,
- 'opt-in': True,
- 'version': metrics_utils.CURRENT_VERSION,
- }
- self._initialized = True
- return
+ # We are running on a bot. Ignore config and collect metrics.
+ if metrics_utils.REPORT_BUILD:
+ self._config = {
+ 'is-googler': True,
+ 'countdown': 0,
+ 'opt-in': True,
+ 'version': metrics_utils.CURRENT_VERSION,
+ }
+ self._initialized = True
+ return
- try:
- config = json.loads(gclient_utils.FileRead(CONFIG_FILE))
- except (IOError, ValueError):
- config = {}
+ try:
+ config = json.loads(gclient_utils.FileRead(CONFIG_FILE))
+ except (IOError, ValueError):
+ config = {}
- self._config = config.copy()
+ self._config = config.copy()
- if 'is-googler' not in self._config:
- # /should-upload is only accessible from Google IPs, so we only need to
- # check if we can reach the page. An external developer would get access
- # denied.
- try:
- req = urllib.request.urlopen(metrics_utils.APP_URL + '/should-upload')
- self._config['is-googler'] = req.getcode() == 200
- except (urllib.request.URLError, urllib.request.HTTPError):
- self._config['is-googler'] = False
+ if 'is-googler' not in self._config:
+ # /should-upload is only accessible from Google IPs, so we only need
+ # to check if we can reach the page. An external developer would get
+ # access denied.
+ try:
+ req = urllib.request.urlopen(metrics_utils.APP_URL +
+ '/should-upload')
+ self._config['is-googler'] = req.getcode() == 200
+ except (urllib.request.URLError, urllib.request.HTTPError):
+ self._config['is-googler'] = False
- # Make sure the config variables we need are present, and initialize them to
- # safe values otherwise.
- self._config.setdefault('countdown', DEFAULT_COUNTDOWN)
- self._config.setdefault('opt-in', None)
- self._config.setdefault('version', metrics_utils.CURRENT_VERSION)
+ # Make sure the config variables we need are present, and initialize
+ # them to safe values otherwise.
+ self._config.setdefault('countdown', DEFAULT_COUNTDOWN)
+ self._config.setdefault('opt-in', None)
+ self._config.setdefault('version', metrics_utils.CURRENT_VERSION)
- if config != self._config:
- print(INVALID_CONFIG_WARNING, file=sys.stderr)
- self._write_config()
+ if config != self._config:
+ print(INVALID_CONFIG_WARNING, file=sys.stderr)
+ self._write_config()
- self._initialized = True
+ self._initialized = True
- def _write_config(self):
- try:
- gclient_utils.FileWrite(CONFIG_FILE, json.dumps(self._config))
- except IOError as e:
- print(PERMISSION_DENIED_WARNING % e, file=sys.stderr)
- self._config['opt-in'] = False
+ def _write_config(self):
+ try:
+ gclient_utils.FileWrite(CONFIG_FILE, json.dumps(self._config))
+ except IOError as e:
+ print(PERMISSION_DENIED_WARNING % e, file=sys.stderr)
+ self._config['opt-in'] = False
- @property
- def version(self):
- self._ensure_initialized()
- return self._config['version']
+ @property
+ def version(self):
+ self._ensure_initialized()
+ return self._config['version']
- @property
- def is_googler(self):
- self._ensure_initialized()
- return self._config['is-googler']
+ @property
+ def is_googler(self):
+ self._ensure_initialized()
+ return self._config['is-googler']
- @property
- def opted_in(self):
- self._ensure_initialized()
- return self._config['opt-in']
+ @property
+ def opted_in(self):
+ self._ensure_initialized()
+ return self._config['opt-in']
- @opted_in.setter
- def opted_in(self, value):
- self._ensure_initialized()
- self._config['opt-in'] = value
- self._config['version'] = metrics_utils.CURRENT_VERSION
- self._write_config()
+ @opted_in.setter
+ def opted_in(self, value):
+ self._ensure_initialized()
+ self._config['opt-in'] = value
+ self._config['version'] = metrics_utils.CURRENT_VERSION
+ self._write_config()
- @property
- def countdown(self):
- self._ensure_initialized()
- return self._config['countdown']
+ @property
+ def countdown(self):
+ self._ensure_initialized()
+ return self._config['countdown']
- @property
- def should_collect_metrics(self):
- # Don't report metrics if user is not a Googler.
- if not self.is_googler:
- return False
- # Don't report metrics if user has opted out.
- if self.opted_in is False:
- return False
- # Don't report metrics if countdown hasn't reached 0.
- if self.opted_in is None and self.countdown > 0:
- return False
- return True
+ @property
+ def should_collect_metrics(self):
+ # Don't report metrics if user is not a Googler.
+ if not self.is_googler:
+ return False
+ # Don't report metrics if user has opted out.
+ if self.opted_in is False:
+ return False
+ # Don't report metrics if countdown hasn't reached 0.
+ if self.opted_in is None and self.countdown > 0:
+ return False
+ return True
- def decrease_countdown(self):
- self._ensure_initialized()
- if self.countdown == 0:
- return
- self._config['countdown'] -= 1
- if self.countdown == 0:
- self._config['version'] = metrics_utils.CURRENT_VERSION
- self._write_config()
+ def decrease_countdown(self):
+ self._ensure_initialized()
+ if self.countdown == 0:
+ return
+ self._config['countdown'] -= 1
+ if self.countdown == 0:
+ self._config['version'] = metrics_utils.CURRENT_VERSION
+ self._write_config()
- def reset_config(self):
- # Only reset countdown if we're already collecting metrics.
- if self.should_collect_metrics:
- self._ensure_initialized()
- self._config['countdown'] = DEFAULT_COUNTDOWN
- self._config['opt-in'] = None
+ def reset_config(self):
+ # Only reset countdown if we're already collecting metrics.
+ if self.should_collect_metrics:
+ self._ensure_initialized()
+ self._config['countdown'] = DEFAULT_COUNTDOWN
+ self._config['opt-in'] = None
class MetricsCollector(object):
- def __init__(self):
- self._metrics_lock = threading.Lock()
- self._reported_metrics = {}
- self._config = _Config()
- self._collecting_metrics = False
- self._collect_custom_metrics = True
+ def __init__(self):
+ self._metrics_lock = threading.Lock()
+ self._reported_metrics = {}
+ self._config = _Config()
+ self._collecting_metrics = False
+ self._collect_custom_metrics = True
- @property
- def config(self):
- return self._config
+ @property
+ def config(self):
+ return self._config
- @property
- def collecting_metrics(self):
- return self._collecting_metrics
+ @property
+ def collecting_metrics(self):
+ return self._collecting_metrics
- def add(self, name, value):
- if self._collect_custom_metrics:
- with self._metrics_lock:
- self._reported_metrics[name] = value
+ def add(self, name, value):
+ if self._collect_custom_metrics:
+ with self._metrics_lock:
+ self._reported_metrics[name] = value
- def add_repeated(self, name, value):
- if self._collect_custom_metrics:
- with self._metrics_lock:
- self._reported_metrics.setdefault(name, []).append(value)
+ def add_repeated(self, name, value):
+ if self._collect_custom_metrics:
+ with self._metrics_lock:
+ self._reported_metrics.setdefault(name, []).append(value)
- @contextlib.contextmanager
- def pause_metrics_collection(self):
- collect_custom_metrics = self._collect_custom_metrics
- self._collect_custom_metrics = False
- try:
- yield
- finally:
- self._collect_custom_metrics = collect_custom_metrics
+ @contextlib.contextmanager
+ def pause_metrics_collection(self):
+ collect_custom_metrics = self._collect_custom_metrics
+ self._collect_custom_metrics = False
+ try:
+ yield
+ finally:
+ self._collect_custom_metrics = collect_custom_metrics
- def _upload_metrics_data(self):
- """Upload the metrics data to the AppEngine app."""
- p = subprocess2.Popen(['vpython3', UPLOAD_SCRIPT], stdin=subprocess2.PIPE)
- # We invoke a subprocess, and use stdin.write instead of communicate(),
- # so that we are able to return immediately, leaving the upload running in
- # the background.
- p.stdin.write(json.dumps(self._reported_metrics).encode('utf-8'))
- # ... but if we're running on a bot, wait until upload has completed.
- if metrics_utils.REPORT_BUILD:
- p.communicate()
+ def _upload_metrics_data(self):
+ """Upload the metrics data to the AppEngine app."""
+ p = subprocess2.Popen(['vpython3', UPLOAD_SCRIPT],
+ stdin=subprocess2.PIPE)
+ # We invoke a subprocess, and use stdin.write instead of communicate(),
+ # so that we are able to return immediately, leaving the upload running
+ # in the background.
+ p.stdin.write(json.dumps(self._reported_metrics).encode('utf-8'))
+ # ... but if we're running on a bot, wait until upload has completed.
+ if metrics_utils.REPORT_BUILD:
+ p.communicate()
- def _collect_metrics(self, func, command_name, *args, **kwargs):
- # If we're already collecting metrics, just execute the function.
- # e.g. git-cl split invokes git-cl upload several times to upload each
- # split CL.
- if self.collecting_metrics:
- # Don't collect metrics for this function.
- # e.g. Don't record the arguments git-cl split passes to git-cl upload.
- with self.pause_metrics_collection():
- return func(*args, **kwargs)
+ def _collect_metrics(self, func, command_name, *args, **kwargs):
+ # If we're already collecting metrics, just execute the function.
+ # e.g. git-cl split invokes git-cl upload several times to upload each
+ # split CL.
+ if self.collecting_metrics:
+ # Don't collect metrics for this function.
+ # e.g. Don't record the arguments git-cl split passes to git-cl
+ # upload.
+ with self.pause_metrics_collection():
+ return func(*args, **kwargs)
- self._collecting_metrics = True
- self.add('metrics_version', metrics_utils.CURRENT_VERSION)
- self.add('command', command_name)
- for env in DEPOT_TOOLS_ENV:
- if env in os.environ:
- self.add_repeated('env_vars', {
- 'name': env,
- 'value': os.environ.get(env)
- })
+ self._collecting_metrics = True
+ self.add('metrics_version', metrics_utils.CURRENT_VERSION)
+ self.add('command', command_name)
+ for env in DEPOT_TOOLS_ENV:
+ if env in os.environ:
+ self.add_repeated('env_vars', {
+ 'name': env,
+ 'value': os.environ.get(env)
+ })
- try:
- start = time.time()
- result = func(*args, **kwargs)
- exception = None
- # pylint: disable=bare-except
- except:
- exception = sys.exc_info()
- finally:
- self.add('execution_time', time.time() - start)
+ try:
+ start = time.time()
+ result = func(*args, **kwargs)
+ exception = None
+ # pylint: disable=bare-except
+ except:
+ exception = sys.exc_info()
+ finally:
+ self.add('execution_time', time.time() - start)
- exit_code = metrics_utils.return_code_from_exception(exception)
- self.add('exit_code', exit_code)
+ exit_code = metrics_utils.return_code_from_exception(exception)
+ self.add('exit_code', exit_code)
- # Add metrics regarding environment information.
- self.add('timestamp', int(time.time()))
- self.add('python_version', metrics_utils.get_python_version())
- self.add('host_os', gclient_utils.GetOperatingSystem())
- self.add('host_arch', detect_host_arch.HostArch())
+ # Add metrics regarding environment information.
+ self.add('timestamp', int(time.time()))
+ self.add('python_version', metrics_utils.get_python_version())
+ self.add('host_os', gclient_utils.GetOperatingSystem())
+ self.add('host_arch', detect_host_arch.HostArch())
- depot_tools_age = metrics_utils.get_repo_timestamp(DEPOT_TOOLS)
- if depot_tools_age is not None:
- self.add('depot_tools_age', int(depot_tools_age))
+ depot_tools_age = metrics_utils.get_repo_timestamp(DEPOT_TOOLS)
+ if depot_tools_age is not None:
+ self.add('depot_tools_age', int(depot_tools_age))
- git_version = metrics_utils.get_git_version()
- if git_version:
- self.add('git_version', git_version)
+ git_version = metrics_utils.get_git_version()
+ if git_version:
+ self.add('git_version', git_version)
- bot_metrics = metrics_utils.get_bot_metrics()
- if bot_metrics:
- self.add('bot_metrics', bot_metrics)
+ bot_metrics = metrics_utils.get_bot_metrics()
+ if bot_metrics:
+ self.add('bot_metrics', bot_metrics)
- self._upload_metrics_data()
- if exception:
- gclient_utils.reraise(exception[0], exception[1], exception[2])
- return result
+ self._upload_metrics_data()
+ if exception:
+ gclient_utils.reraise(exception[0], exception[1], exception[2])
+ return result
- def collect_metrics(self, command_name):
- """A decorator used to collect metrics over the life of a function.
+ def collect_metrics(self, command_name):
+ """A decorator used to collect metrics over the life of a function.
This decorator executes the function and collects metrics about the system
environment and the function performance.
"""
- def _decorator(func):
- if not self.config.should_collect_metrics:
- return func
- # Needed to preserve the __name__ and __doc__ attributes of func.
- @functools.wraps(func)
- def _inner(*args, **kwargs):
- return self._collect_metrics(func, command_name, *args, **kwargs)
- return _inner
- return _decorator
+ def _decorator(func):
+ if not self.config.should_collect_metrics:
+ return func
+ # Needed to preserve the __name__ and __doc__ attributes of func.
+ @functools.wraps(func)
+ def _inner(*args, **kwargs):
+ return self._collect_metrics(func, command_name, *args,
+ **kwargs)
- @contextlib.contextmanager
- def print_notice_and_exit(self):
- """A context manager used to print the notice and terminate execution.
+ return _inner
+
+ return _decorator
+
+ @contextlib.contextmanager
+ def print_notice_and_exit(self):
+ """A context manager used to print the notice and terminate execution.
This decorator executes the function and prints the monitoring notice if
necessary. If an exception is raised, we will catch it, and print it before
printing the metrics collection notice.
This will call sys.exit() with an appropriate exit code to ensure the notice
is the last thing printed."""
- # Needed to preserve the __name__ and __doc__ attributes of func.
- try:
- yield
- exception = None
- # pylint: disable=bare-except
- except:
- exception = sys.exc_info()
+ # Needed to preserve the __name__ and __doc__ attributes of func.
+ try:
+ yield
+ exception = None
+ # pylint: disable=bare-except
+ except:
+ exception = sys.exc_info()
- # Print the exception before the metrics notice, so that the notice is
- # clearly visible even if gclient fails.
- if exception:
- if isinstance(exception[1], KeyboardInterrupt):
- sys.stderr.write('Interrupted\n')
- elif not isinstance(exception[1], SystemExit):
- traceback.print_exception(*exception)
+ # Print the exception before the metrics notice, so that the notice is
+ # clearly visible even if gclient fails.
+ if exception:
+ if isinstance(exception[1], KeyboardInterrupt):
+ sys.stderr.write('Interrupted\n')
+ elif not isinstance(exception[1], SystemExit):
+ traceback.print_exception(*exception)
- # Check if the version has changed
- if (self.config.is_googler
- and self.config.opted_in is not False
- and self.config.version != metrics_utils.CURRENT_VERSION):
- metrics_utils.print_version_change(self.config.version)
- self.config.reset_config()
+ # Check if the version has changed
+ if (self.config.is_googler and self.config.opted_in is not False
+ and self.config.version != metrics_utils.CURRENT_VERSION):
+ metrics_utils.print_version_change(self.config.version)
+ self.config.reset_config()
- # Print the notice
- if self.config.is_googler and self.config.opted_in is None:
- metrics_utils.print_notice(self.config.countdown)
- self.config.decrease_countdown()
+ # Print the notice
+ if self.config.is_googler and self.config.opted_in is None:
+ metrics_utils.print_notice(self.config.countdown)
+ self.config.decrease_countdown()
- sys.exit(metrics_utils.return_code_from_exception(exception))
+ sys.exit(metrics_utils.return_code_from_exception(exception))
collector = MetricsCollector()