Edward Lemur | 32e3d1e | 2018-07-12 00:54:05 +0000 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # Copyright (c) 2018 The Chromium Authors. All rights reserved. |
| 3 | # Use of this source code is governed by a BSD-style license that can be |
| 4 | # found in the LICENSE file. |
| 5 | |
| 6 | import functools |
| 7 | import json |
| 8 | import os |
| 9 | import subprocess |
| 10 | import sys |
| 11 | import tempfile |
| 12 | import threading |
| 13 | import time |
| 14 | import traceback |
| 15 | import urllib2 |
| 16 | |
| 17 | import detect_host_arch |
| 18 | import gclient_utils |
| 19 | import metrics_utils |
| 20 | |
| 21 | |
| 22 | DEPOT_TOOLS = os.path.dirname(os.path.abspath(__file__)) |
| 23 | CONFIG_FILE = os.path.join(DEPOT_TOOLS, 'metrics.cfg') |
| 24 | UPLOAD_SCRIPT = os.path.join(DEPOT_TOOLS, 'upload_metrics.py') |
| 25 | |
Edward Lemur | 32e3d1e | 2018-07-12 00:54:05 +0000 | [diff] [blame] | 26 | DISABLE_METRICS_COLLECTION = os.environ.get('DEPOT_TOOLS_METRICS') == '0' |
| 27 | DEFAULT_COUNTDOWN = 10 |
| 28 | |
| 29 | INVALID_CONFIG_WARNING = ( |
| 30 | 'WARNING: Your metrics.cfg file was invalid or nonexistent. A new one has ' |
| 31 | 'been created' |
| 32 | ) |
| 33 | |
| 34 | |
| 35 | class _Config(object): |
| 36 | def __init__(self): |
| 37 | self._initialized = False |
| 38 | self._config = {} |
| 39 | |
| 40 | def _ensure_initialized(self): |
| 41 | if self._initialized: |
| 42 | return |
| 43 | |
| 44 | try: |
| 45 | config = json.loads(gclient_utils.FileRead(CONFIG_FILE)) |
| 46 | except (IOError, ValueError): |
| 47 | config = {} |
| 48 | |
| 49 | self._config = config.copy() |
| 50 | |
| 51 | if 'is-googler' not in self._config: |
| 52 | # /should-upload is only accessible from Google IPs, so we only need to |
| 53 | # check if we can reach the page. An external developer would get access |
| 54 | # denied. |
| 55 | try: |
Edward Lemur | 5ba1e9c | 2018-07-23 18:19:02 +0000 | [diff] [blame^] | 56 | req = urllib2.urlopen(metrics_utils.APP_URL + '/should-upload') |
Edward Lemur | 32e3d1e | 2018-07-12 00:54:05 +0000 | [diff] [blame] | 57 | self._config['is-googler'] = req.getcode() == 200 |
| 58 | except (urllib2.URLError, urllib2.HTTPError): |
| 59 | self._config['is-googler'] = False |
| 60 | |
| 61 | # Make sure the config variables we need are present, and initialize them to |
| 62 | # safe values otherwise. |
| 63 | self._config.setdefault('countdown', DEFAULT_COUNTDOWN) |
| 64 | self._config.setdefault('opt-in', None) |
| 65 | |
| 66 | if config != self._config: |
| 67 | print INVALID_CONFIG_WARNING |
| 68 | self._write_config() |
| 69 | |
| 70 | self._initialized = True |
| 71 | |
| 72 | def _write_config(self): |
| 73 | gclient_utils.FileWrite(CONFIG_FILE, json.dumps(self._config)) |
| 74 | |
| 75 | @property |
| 76 | def is_googler(self): |
| 77 | self._ensure_initialized() |
| 78 | return self._config['is-googler'] |
| 79 | |
| 80 | @property |
| 81 | def opted_in(self): |
| 82 | self._ensure_initialized() |
| 83 | return self._config['opt-in'] |
| 84 | |
| 85 | @opted_in.setter |
| 86 | def opted_in(self, value): |
| 87 | self._ensure_initialized() |
| 88 | self._config['opt-in'] = value |
| 89 | self._write_config() |
| 90 | |
| 91 | @property |
| 92 | def countdown(self): |
| 93 | self._ensure_initialized() |
| 94 | return self._config['countdown'] |
| 95 | |
| 96 | def decrease_countdown(self): |
| 97 | self._ensure_initialized() |
| 98 | if self.countdown == 0: |
| 99 | return |
| 100 | self._config['countdown'] -= 1 |
| 101 | self._write_config() |
| 102 | |
| 103 | |
| 104 | class MetricsCollector(object): |
| 105 | def __init__(self): |
| 106 | self._metrics_lock = threading.Lock() |
| 107 | self._reported_metrics = {} |
| 108 | self._config = _Config() |
Edward Lemur | 3298e7b | 2018-07-17 18:21:27 +0000 | [diff] [blame] | 109 | self._collecting_metrics = False |
Edward Lemur | 32e3d1e | 2018-07-12 00:54:05 +0000 | [diff] [blame] | 110 | |
| 111 | @property |
| 112 | def config(self): |
| 113 | return self._config |
| 114 | |
Edward Lemur | 3298e7b | 2018-07-17 18:21:27 +0000 | [diff] [blame] | 115 | @property |
| 116 | def collecting_metrics(self): |
| 117 | return self._collecting_metrics |
| 118 | |
Edward Lemur | 32e3d1e | 2018-07-12 00:54:05 +0000 | [diff] [blame] | 119 | def add(self, name, value): |
Edward Lemur | 3298e7b | 2018-07-17 18:21:27 +0000 | [diff] [blame] | 120 | if not self.collecting_metrics: |
| 121 | return |
Edward Lemur | 32e3d1e | 2018-07-12 00:54:05 +0000 | [diff] [blame] | 122 | with self._metrics_lock: |
| 123 | self._reported_metrics[name] = value |
| 124 | |
| 125 | def _upload_metrics_data(self): |
| 126 | """Upload the metrics data to the AppEngine app.""" |
| 127 | # We invoke a subprocess, and use stdin.write instead of communicate(), |
| 128 | # so that we are able to return immediately, leaving the upload running in |
| 129 | # the background. |
| 130 | p = subprocess.Popen([sys.executable, UPLOAD_SCRIPT], stdin=subprocess.PIPE) |
| 131 | p.stdin.write(json.dumps(self._reported_metrics)) |
| 132 | |
| 133 | def _collect_metrics(self, func, command_name, *args, **kwargs): |
Edward Lemur | 7fa0f19 | 2018-07-17 21:33:37 +0000 | [diff] [blame] | 134 | # If the user hasn't opted in or out, and the countdown is not yet 0, just |
| 135 | # display the notice. |
| 136 | if self.config.opted_in == None and self.config.countdown > 0: |
| 137 | metrics_utils.print_notice(self.config.countdown) |
| 138 | self.config.decrease_countdown() |
| 139 | func(*args, **kwargs) |
| 140 | return |
| 141 | |
| 142 | self._collecting_metrics = True |
Edward Lemur | 32e3d1e | 2018-07-12 00:54:05 +0000 | [diff] [blame] | 143 | self.add('command', command_name) |
| 144 | try: |
| 145 | start = time.time() |
| 146 | func(*args, **kwargs) |
| 147 | exception = None |
| 148 | # pylint: disable=bare-except |
| 149 | except: |
| 150 | exception = sys.exc_info() |
| 151 | finally: |
| 152 | self.add('execution_time', time.time() - start) |
| 153 | |
| 154 | # Print the exception before the metrics notice, so that the notice is |
| 155 | # clearly visible even if gclient fails. |
| 156 | if exception and not isinstance(exception[1], SystemExit): |
| 157 | traceback.print_exception(*exception) |
| 158 | |
| 159 | exit_code = metrics_utils.return_code_from_exception(exception) |
| 160 | self.add('exit_code', exit_code) |
| 161 | |
| 162 | # Print the metrics notice only if the user has not explicitly opted in |
| 163 | # or out. |
| 164 | if self.config.opted_in is None: |
| 165 | metrics_utils.print_notice(self.config.countdown) |
| 166 | |
| 167 | # Add metrics regarding environment information. |
| 168 | self.add('timestamp', metrics_utils.seconds_to_weeks(time.time())) |
| 169 | self.add('python_version', metrics_utils.get_python_version()) |
| 170 | self.add('host_os', gclient_utils.GetMacWinOrLinux()) |
| 171 | self.add('host_arch', detect_host_arch.HostArch()) |
| 172 | self.add('depot_tools_age', metrics_utils.get_repo_timestamp(DEPOT_TOOLS)) |
| 173 | |
| 174 | self._upload_metrics_data() |
| 175 | sys.exit(exit_code) |
| 176 | |
| 177 | def collect_metrics(self, command_name): |
| 178 | """A decorator used to collect metrics over the life of a function. |
| 179 | |
| 180 | This decorator executes the function and collects metrics about the system |
| 181 | environment and the function performance. It also catches all the Exceptions |
| 182 | and invokes sys.exit once the function is done executing. |
| 183 | """ |
| 184 | def _decorator(func): |
| 185 | # Do this first so we don't have to read, and possibly create a config |
| 186 | # file. |
| 187 | if DISABLE_METRICS_COLLECTION: |
| 188 | return func |
| 189 | # If the user has opted out or the user is not a googler, then there is no |
| 190 | # need to do anything. |
| 191 | if self.config.opted_in == False or not self.config.is_googler: |
| 192 | return func |
Edward Lemur | 32e3d1e | 2018-07-12 00:54:05 +0000 | [diff] [blame] | 193 | # Otherwise, collect the metrics. |
| 194 | # Needed to preserve the __name__ and __doc__ attributes of func. |
| 195 | @functools.wraps(func) |
| 196 | def _inner(*args, **kwargs): |
| 197 | self._collect_metrics(func, command_name, *args, **kwargs) |
Edward Lemur | 32e3d1e | 2018-07-12 00:54:05 +0000 | [diff] [blame] | 198 | return _inner |
| 199 | return _decorator |
| 200 | |
| 201 | |
| 202 | collector = MetricsCollector() |