Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 1 | # Copyright 2016 The Chromium OS Authors. All rights reserved. |
| 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 5 | """System metrics.""" |
| 6 | |
Allen Li | 13bdf0c | 2017-03-02 15:18:16 -0800 | [diff] [blame^] | 7 | from __future__ import absolute_import |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 8 | from __future__ import print_function |
Allen Li | 13bdf0c | 2017-03-02 15:18:16 -0800 | [diff] [blame^] | 9 | from __future__ import unicode_literals |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 10 | |
| 11 | import errno |
| 12 | import os |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 13 | import time |
| 14 | |
| 15 | import psutil |
| 16 | |
| 17 | from chromite.lib import cros_logging as logging |
| 18 | from infra_libs import ts_mon |
| 19 | |
Allen Li | 79317bb | 2016-12-16 18:25:07 -0800 | [diff] [blame] | 20 | logger = logging.getLogger(__name__) |
| 21 | |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 22 | |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 23 | _cpu_count_metric = ts_mon.GaugeMetric( |
| 24 | 'dev/cpu/count', |
| 25 | description='Number of CPU cores.') |
| 26 | _cpu_time_metric = ts_mon.FloatMetric( |
| 27 | 'dev/cpu/time', |
| 28 | description='percentage of time spent by the CPU ' |
| 29 | 'in different states.') |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 30 | |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 31 | _disk_free_metric = ts_mon.GaugeMetric( |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 32 | 'dev/disk/free', |
| 33 | description='Available bytes on disk partition.', |
| 34 | units=ts_mon.MetricsDataUnits.BYTES) |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 35 | _disk_total_metric = ts_mon.GaugeMetric( |
| 36 | 'dev/disk/total', |
| 37 | description='Total bytes on disk partition.', |
| 38 | units=ts_mon.MetricsDataUnits.BYTES) |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 39 | |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 40 | _inodes_free_metric = ts_mon.GaugeMetric( |
| 41 | 'dev/inodes/free', |
| 42 | description='Number of available inodes on ' |
| 43 | 'disk partition (unix only).') |
| 44 | _inodes_total_metric = ts_mon.GaugeMetric( |
| 45 | 'dev/inodes/total', |
| 46 | description='Number of possible inodes on ' |
| 47 | 'disk partition (unix only)') |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 48 | |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 49 | _mem_free_metric = ts_mon.GaugeMetric( |
| 50 | 'dev/mem/free', |
| 51 | description='Amount of memory available to a ' |
| 52 | 'process (in Bytes). Buffers are considered ' |
| 53 | 'free memory.', |
| 54 | units=ts_mon.MetricsDataUnits.BYTES) |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 55 | |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 56 | _mem_total_metric = ts_mon.GaugeMetric( |
| 57 | 'dev/mem/total', |
| 58 | description='Total physical memory in Bytes.', |
| 59 | units=ts_mon.MetricsDataUnits.BYTES) |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 60 | |
Allen Li | 325c076 | 2017-03-02 15:00:19 -0800 | [diff] [blame] | 61 | _BOOT_TIME = psutil.boot_time() |
Allen Li | c987fc9 | 2017-03-02 14:54:51 -0800 | [diff] [blame] | 62 | |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 63 | _disk_read_metric = ts_mon.CounterMetric( |
Allen Li | 325c076 | 2017-03-02 15:00:19 -0800 | [diff] [blame] | 64 | 'dev/disk/read', start_time=_BOOT_TIME, |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 65 | description='Number of Bytes read on disk.', |
| 66 | units=ts_mon.MetricsDataUnits.BYTES) |
| 67 | _disk_write_metric = ts_mon.CounterMetric( |
Allen Li | 325c076 | 2017-03-02 15:00:19 -0800 | [diff] [blame] | 68 | 'dev/disk/write', start_time=_BOOT_TIME, |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 69 | description='Number of Bytes written on disk.', |
| 70 | units=ts_mon.MetricsDataUnits.BYTES) |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 71 | |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 72 | _uptime_metric = ts_mon.GaugeMetric( |
| 73 | 'dev/uptime', |
| 74 | description='Machine uptime, in seconds.', |
| 75 | units=ts_mon.MetricsDataUnits.SECONDS) |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 76 | |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 77 | _proc_count_metric = ts_mon.GaugeMetric( |
| 78 | 'dev/proc/count', |
| 79 | description='Number of processes currently running.') |
Allen Li | efe7adf | 2016-10-27 11:36:04 -0700 | [diff] [blame] | 80 | _autoserv_proc_count_metric = ts_mon.GaugeMetric( |
| 81 | 'dev/proc/autoserv_count', |
| 82 | description='Number of autoserv processes currently running.') |
Allen Li | 0937a52 | 2016-11-23 13:34:48 -0800 | [diff] [blame] | 83 | _sysmon_proc_count_metric = ts_mon.GaugeMetric( |
| 84 | 'dev/proc/sysmon_count', |
| 85 | description='Number of sysmon processes currently running.') |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 86 | _load_average_metric = ts_mon.FloatMetric( |
| 87 | 'dev/proc/load_average', |
| 88 | description='Number of processes currently ' |
| 89 | 'in the system run queue.') |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 90 | |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 91 | # ts_mon pipeline uses backend clocks when assigning timestamps to metric |
| 92 | # points. By comparing point timestamp to the point value (i.e. time by |
| 93 | # machine's local clock), we can potentially detect some anomalies (clock |
| 94 | # drift, unusually high metrics pipeline delay, completely wrong clocks, etc). |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 95 | # |
| 96 | # It is important to gather this metric right before the flush. |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 97 | _unix_time_metric = ts_mon.GaugeMetric( |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 98 | 'dev/unix_time', |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 99 | description='Number of milliseconds since epoch' |
| 100 | ' based on local machine clock.') |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 101 | |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 102 | _os_name_metric = ts_mon.StringMetric( |
| 103 | 'proc/os/name', |
| 104 | description='OS name on the machine') |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 105 | |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 106 | _os_version_metric = ts_mon.StringMetric( |
| 107 | 'proc/os/version', |
| 108 | description='OS version on the machine') |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 109 | |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 110 | _os_arch_metric = ts_mon.StringMetric( |
| 111 | 'proc/os/arch', |
| 112 | description='OS architecture on this machine') |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 113 | |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 114 | _python_arch_metric = ts_mon.StringMetric( |
| 115 | 'proc/python/arch', |
| 116 | description='python userland ' |
| 117 | 'architecture on this machine') |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 118 | |
| 119 | |
Allen Li | 45ae839 | 2017-03-02 14:19:35 -0800 | [diff] [blame] | 120 | def collect_uptime(): |
Allen Li | 325c076 | 2017-03-02 15:00:19 -0800 | [diff] [blame] | 121 | _uptime_metric.set(int(time.time() - _BOOT_TIME)) |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 122 | |
| 123 | |
Allen Li | 45ae839 | 2017-03-02 14:19:35 -0800 | [diff] [blame] | 124 | def collect_cpu_info(): |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 125 | _cpu_count_metric.set(psutil.cpu_count()) |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 126 | |
| 127 | times = psutil.cpu_times_percent() |
| 128 | for mode in ('user', 'system', 'idle'): |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 129 | _cpu_time_metric.set(getattr(times, mode), {'mode': mode}) |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 130 | |
| 131 | |
Allen Li | 45ae839 | 2017-03-02 14:19:35 -0800 | [diff] [blame] | 132 | def collect_disk_info(mountpoints=None): |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 133 | if mountpoints is None: |
| 134 | mountpoints = [disk.mountpoint for disk in psutil.disk_partitions()] |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 135 | for mountpoint in mountpoints: |
Allen Li | 45ae839 | 2017-03-02 14:19:35 -0800 | [diff] [blame] | 136 | _collect_disk_info_single(mountpoint) |
| 137 | _collect_fs_inode_info(mountpoint) |
| 138 | _collect_disk_io_info() |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 139 | |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 140 | |
Allen Li | 45ae839 | 2017-03-02 14:19:35 -0800 | [diff] [blame] | 141 | def _collect_disk_info_single(mountpoint): |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 142 | fields = {'path': mountpoint} |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 143 | |
| 144 | try: |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 145 | usage = psutil.disk_usage(mountpoint) |
| 146 | except OSError as ex: |
| 147 | if ex.errno == errno.ENOENT: |
| 148 | # This happens on Windows when querying a removable drive that |
| 149 | # doesn't have any media inserted right now. |
| 150 | pass |
| 151 | else: |
| 152 | raise |
| 153 | else: |
| 154 | _disk_free_metric.set(usage.free, fields=fields) |
| 155 | _disk_total_metric.set(usage.total, fields=fields) |
| 156 | |
| 157 | # inode counts are only available on Unix. |
| 158 | if os.name == 'posix': |
Allen Li | 45ae839 | 2017-03-02 14:19:35 -0800 | [diff] [blame] | 159 | _collect_fs_inode_info(mountpoint) |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 160 | |
| 161 | |
Allen Li | 45ae839 | 2017-03-02 14:19:35 -0800 | [diff] [blame] | 162 | def _collect_fs_inode_info(mountpoint): |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 163 | fields = {'path': mountpoint} |
| 164 | stats = os.statvfs(mountpoint) |
| 165 | _inodes_free_metric.set(stats.f_favail, fields=fields) |
| 166 | _inodes_total_metric.set(stats.f_files, fields=fields) |
| 167 | |
| 168 | |
Allen Li | 45ae839 | 2017-03-02 14:19:35 -0800 | [diff] [blame] | 169 | def _collect_disk_io_info(): |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 170 | try: |
| 171 | disk_counters = psutil.disk_io_counters(perdisk=True).iteritems() |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 172 | except RuntimeError as ex: |
| 173 | if "couldn't find any physical disk" in str(ex): |
| 174 | # Disk performance counters aren't enabled on Windows. |
| 175 | pass |
| 176 | else: |
| 177 | raise |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 178 | else: |
| 179 | for disk, counters in disk_counters: |
| 180 | fields = {'disk': disk} |
| 181 | _disk_read_metric.set(counters.read_bytes, fields=fields) |
| 182 | _disk_write_metric.set(counters.write_bytes, fields=fields) |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 183 | |
| 184 | |
Allen Li | 45ae839 | 2017-03-02 14:19:35 -0800 | [diff] [blame] | 185 | def collect_mem_info(): |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 186 | # We don't report mem.used because (due to virtual memory) it is not |
| 187 | # useful. |
| 188 | mem = psutil.virtual_memory() |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 189 | _mem_free_metric.set(mem.available) |
| 190 | _mem_total_metric.set(mem.total) |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 191 | |
| 192 | |
Allen Li | 45ae839 | 2017-03-02 14:19:35 -0800 | [diff] [blame] | 193 | def collect_proc_info(): |
Allen Li | efe7adf | 2016-10-27 11:36:04 -0700 | [diff] [blame] | 194 | autoserv_count = 0 |
Allen Li | 0937a52 | 2016-11-23 13:34:48 -0800 | [diff] [blame] | 195 | sysmon_count = 0 |
Allen Li | efe7adf | 2016-10-27 11:36:04 -0700 | [diff] [blame] | 196 | total = 0 |
| 197 | for proc in psutil.process_iter(): |
Allen Li | 80dae19 | 2016-11-01 11:58:10 -0700 | [diff] [blame] | 198 | if _is_parent_autoserv(proc): |
Allen Li | efe7adf | 2016-10-27 11:36:04 -0700 | [diff] [blame] | 199 | autoserv_count += 1 |
Allen Li | 0937a52 | 2016-11-23 13:34:48 -0800 | [diff] [blame] | 200 | elif _is_sysmon(proc): |
| 201 | sysmon_count += 1 |
Allen Li | efe7adf | 2016-10-27 11:36:04 -0700 | [diff] [blame] | 202 | total += 1 |
Allen Li | 79317bb | 2016-12-16 18:25:07 -0800 | [diff] [blame] | 203 | logger.debug('autoserv_count: %s', autoserv_count) |
| 204 | logger.debug('sysmon_count: %s', sysmon_count) |
Allen Li | efe7adf | 2016-10-27 11:36:04 -0700 | [diff] [blame] | 205 | _autoserv_proc_count_metric.set(autoserv_count) |
Allen Li | 0937a52 | 2016-11-23 13:34:48 -0800 | [diff] [blame] | 206 | _sysmon_proc_count_metric.set(sysmon_count) |
Allen Li | efe7adf | 2016-10-27 11:36:04 -0700 | [diff] [blame] | 207 | _proc_count_metric.set(total) |
| 208 | |
| 209 | |
Allen Li | 80dae19 | 2016-11-01 11:58:10 -0700 | [diff] [blame] | 210 | def _is_parent_autoserv(proc): |
| 211 | """Return whether proc is a parent (not forked) autoserv process.""" |
| 212 | return _is_autoserv(proc) and not _is_autoserv(proc.parent()) |
| 213 | |
| 214 | |
| 215 | def _is_autoserv(proc): |
| 216 | """Return whether proc is an autoserv process.""" |
| 217 | # This relies on the autoserv script being run directly. The script should |
| 218 | # be named autoserv exactly and start with a shebang that is /usr/bin/python, |
| 219 | # NOT /bin/env |
| 220 | return proc.name() == 'autoserv' |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 221 | |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 222 | |
Allen Li | 0937a52 | 2016-11-23 13:34:48 -0800 | [diff] [blame] | 223 | def _is_sysmon(proc): |
| 224 | """Return whether proc is a sysmon process.""" |
Allen Li | dfdfbda | 2016-12-16 17:41:16 -0800 | [diff] [blame] | 225 | return proc.cmdline()[:3] == ['python', '-m', 'chromite.scripts.sysmon'] |
Allen Li | 0937a52 | 2016-11-23 13:34:48 -0800 | [diff] [blame] | 226 | |
| 227 | |
Allen Li | 45ae839 | 2017-03-02 14:19:35 -0800 | [diff] [blame] | 228 | def collect_load_avg(): |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 229 | try: |
| 230 | avg1, avg5, avg15 = os.getloadavg() |
| 231 | except OSError: |
| 232 | pass |
| 233 | else: |
| 234 | _load_average_metric.set(avg1, fields={'minutes': 1}) |
| 235 | _load_average_metric.set(avg5, fields={'minutes': 5}) |
| 236 | _load_average_metric.set(avg15, fields={'minutes': 15}) |
Allen Li | ec5beb3 | 2016-09-08 15:31:41 -0700 | [diff] [blame] | 237 | |
| 238 | |
Allen Li | 45ae839 | 2017-03-02 14:19:35 -0800 | [diff] [blame] | 239 | def collect_unix_time(): |
Allen Li | a6b0225 | 2016-10-26 14:40:51 -0700 | [diff] [blame] | 240 | _unix_time_metric.set(int(time.time() * 1000)) |