Zhizhou Yang | 81d651f | 2020-02-10 16:51:20 -0800 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
Tiancong Wang | d034813 | 2019-03-07 11:22:48 -0800 | [diff] [blame] | 2 | # -*- coding: utf-8 -*- |
Ting-Yuan Huang | e581987 | 2016-12-15 14:22:26 -0800 | [diff] [blame] | 3 | # |
| 4 | # Copyright 2016 The Chromium OS Authors. All rights reserved. |
| 5 | # Use of this source code is governed by a BSD-style license that can be |
| 6 | # found in the LICENSE file. |
Denis Nikitin | 265c296 | 2019-08-01 19:52:49 -0700 | [diff] [blame] | 7 | |
Yunlian Jiang | 14cf596 | 2015-12-11 15:50:14 -0800 | [diff] [blame] | 8 | """Script for running nightly compiler tests on ChromeOS. |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 9 | |
| 10 | This script launches a buildbot to build ChromeOS with the latest compiler on |
| 11 | a particular board; then it finds and downloads the trybot image and the |
| 12 | corresponding official image, and runs crosperf performance tests comparing |
| 13 | the two. It then generates a report, emails it to the c-compiler-chrome, as |
| 14 | well as copying the images into the seven-day reports directory. |
| 15 | """ |
| 16 | |
| 17 | # Script to test different toolchains against ChromeOS benchmarks. |
Yunlian Jiang | 14cf596 | 2015-12-11 15:50:14 -0800 | [diff] [blame] | 18 | |
| 19 | from __future__ import print_function |
| 20 | |
Caroline Tice | eddb063 | 2016-04-14 09:19:02 -0700 | [diff] [blame] | 21 | import argparse |
cmtice | ce5ffa4 | 2015-02-12 15:18:43 -0800 | [diff] [blame] | 22 | import datetime |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 23 | import os |
Luis Lozano | c75fd05 | 2016-02-19 17:37:01 -0800 | [diff] [blame] | 24 | import re |
zhizhouy | 9258b05 | 2020-04-15 17:33:46 -0700 | [diff] [blame] | 25 | import shutil |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 26 | import sys |
| 27 | import time |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 28 | |
Caroline Tice | a8af9a7 | 2016-07-20 12:52:59 -0700 | [diff] [blame] | 29 | from cros_utils import command_executer |
| 30 | from cros_utils import logger |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 31 | |
Caroline Tice | a8af9a7 | 2016-07-20 12:52:59 -0700 | [diff] [blame] | 32 | from cros_utils import buildbot_utils |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 33 | |
Manoj Gupta | c411035 | 2016-12-28 13:47:12 -0800 | [diff] [blame] | 34 | # CL that uses LLVM-Next to build the images (includes chrome). |
Manoj Gupta | 66682c7 | 2017-05-24 12:19:57 -0700 | [diff] [blame] | 35 | USE_LLVM_NEXT_PATCH = '513590' |
Manoj Gupta | c411035 | 2016-12-28 13:47:12 -0800 | [diff] [blame] | 36 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 37 | CROSTC_ROOT = '/usr/local/google/crostc' |
Luis Lozano | be756e0 | 2020-02-07 10:54:08 -0800 | [diff] [blame] | 38 | NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly-tests') |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 39 | ROLE_ACCOUNT = 'mobiletc-prebuild' |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 40 | TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__)) |
zhizhouy | 9258b05 | 2020-04-15 17:33:46 -0700 | [diff] [blame] | 41 | TMP_TOOLCHAIN_TEST = '/tmp/toolchain-tests' |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 42 | MAIL_PROGRAM = '~/var/bin/mail-sheriff' |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 43 | PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, 'pending_archives') |
Luis Lozano | b02958e | 2020-02-10 02:12:18 -0800 | [diff] [blame] | 44 | NIGHTLY_TESTS_RESULTS = os.path.join(CROSTC_ROOT, 'nightly_test_reports') |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 45 | |
Ting-Yuan Huang | e581987 | 2016-12-15 14:22:26 -0800 | [diff] [blame] | 46 | IMAGE_DIR = '{board}-{image_type}' |
| 47 | IMAGE_VERSION_STR = r'{chrome_version}-{tip}\.{branch}\.{branch_branch}' |
| 48 | IMAGE_FS = IMAGE_DIR + '/' + IMAGE_VERSION_STR |
Ting-Yuan Huang | 112d562 | 2018-03-26 13:49:42 -0700 | [diff] [blame] | 49 | TRYBOT_IMAGE_FS = IMAGE_FS + '-{build_id}' |
Manoj Gupta | aee96b7 | 2016-10-24 13:43:28 -0700 | [diff] [blame] | 50 | IMAGE_RE_GROUPS = { |
| 51 | 'board': r'(?P<board>\S+)', |
| 52 | 'image_type': r'(?P<image_type>\S+)', |
| 53 | 'chrome_version': r'(?P<chrome_version>R\d+)', |
| 54 | 'tip': r'(?P<tip>\d+)', |
| 55 | 'branch': r'(?P<branch>\d+)', |
| 56 | 'branch_branch': r'(?P<branch_branch>\d+)', |
| 57 | 'build_id': r'(?P<build_id>b\d+)' |
| 58 | } |
Luis Lozano | c75fd05 | 2016-02-19 17:37:01 -0800 | [diff] [blame] | 59 | TRYBOT_IMAGE_RE = TRYBOT_IMAGE_FS.format(**IMAGE_RE_GROUPS) |
| 60 | |
zhizhouy | 9258b05 | 2020-04-15 17:33:46 -0700 | [diff] [blame] | 61 | RECIPE_IMAGE_FS = IMAGE_FS + '-{build_id}-{buildbucket_id}' |
| 62 | RECIPE_IMAGE_RE_GROUPS = { |
| 63 | 'board': r'(?P<board>\S+)', |
| 64 | 'image_type': r'(?P<image_type>\S+)', |
| 65 | 'chrome_version': r'(?P<chrome_version>R\d+)', |
| 66 | 'tip': r'(?P<tip>\d+)', |
| 67 | 'branch': r'(?P<branch>\d+)', |
| 68 | 'branch_branch': r'(?P<branch_branch>\d+)', |
| 69 | 'build_id': r'(?P<build_id>\d+)', |
| 70 | 'buildbucket_id': r'(?P<buildbucket_id>\d+)' |
| 71 | } |
| 72 | RECIPE_IMAGE_RE = RECIPE_IMAGE_FS.format(**RECIPE_IMAGE_RE_GROUPS) |
| 73 | |
Tiancong Wang | 0323466 | 2019-10-30 10:16:38 -0700 | [diff] [blame] | 74 | TELEMETRY_AQUARIUM_UNSUPPORTED = ['bob', 'elm', 'veyron_minnie'] |
| 75 | |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 76 | |
Yunlian Jiang | 14cf596 | 2015-12-11 15:50:14 -0800 | [diff] [blame] | 77 | class ToolchainComparator(object): |
| 78 | """Class for doing the nightly tests work.""" |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 79 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 80 | def __init__(self, |
| 81 | board, |
| 82 | remotes, |
| 83 | chromeos_root, |
| 84 | weekday, |
| 85 | patches, |
zhizhouy | 9258b05 | 2020-04-15 17:33:46 -0700 | [diff] [blame] | 86 | recipe=False, |
| 87 | test=False, |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 88 | noschedv2=False): |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 89 | self._board = board |
| 90 | self._remotes = remotes |
| 91 | self._chromeos_root = chromeos_root |
| 92 | self._base_dir = os.getcwd() |
| 93 | self._ce = command_executer.GetCommandExecuter() |
| 94 | self._l = logger.GetLogger() |
Yunlian Jiang | 73580dd | 2017-11-21 04:48:40 -0800 | [diff] [blame] | 95 | self._build = '%s-release-tryjob' % board |
Manoj Gupta | d575b8a | 2017-03-08 10:51:28 -0800 | [diff] [blame] | 96 | self._patches = patches.split(',') if patches else [] |
Yunlian Jiang | 3c6e467 | 2015-08-24 15:58:22 -0700 | [diff] [blame] | 97 | self._patches_string = '_'.join(str(p) for p in self._patches) |
zhizhouy | 9258b05 | 2020-04-15 17:33:46 -0700 | [diff] [blame] | 98 | self._recipe = recipe |
| 99 | self._test = test |
Han Shen | 4349429 | 2015-09-14 10:26:40 -0700 | [diff] [blame] | 100 | self._noschedv2 = noschedv2 |
Yunlian Jiang | 3c6e467 | 2015-08-24 15:58:22 -0700 | [diff] [blame] | 101 | |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 102 | if not weekday: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 103 | self._weekday = time.strftime('%a') |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 104 | else: |
| 105 | self._weekday = weekday |
zhizhouy | d8fcbf5 | 2020-04-06 16:47:20 -0700 | [diff] [blame] | 106 | self._date = datetime.date.today().strftime('%Y/%m/%d') |
| 107 | timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H:%M:%S') |
Manoj Gupta | aee96b7 | 2016-10-24 13:43:28 -0700 | [diff] [blame] | 108 | self._reports_dir = os.path.join( |
zhizhouy | 9258b05 | 2020-04-15 17:33:46 -0700 | [diff] [blame] | 109 | TMP_TOOLCHAIN_TEST if self._test else NIGHTLY_TESTS_RESULTS, |
Caroline Tice | 9c4003a | 2017-11-07 16:37:33 -0800 | [diff] [blame] | 110 | '%s.%s' % (timestamp, board), |
| 111 | ) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 112 | |
Luis Lozano | c75fd05 | 2016-02-19 17:37:01 -0800 | [diff] [blame] | 113 | def _GetVanillaImageName(self, trybot_image): |
Ting-Yuan Huang | e581987 | 2016-12-15 14:22:26 -0800 | [diff] [blame] | 114 | """Given a trybot artifact name, get latest vanilla image name. |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 115 | |
Luis Lozano | 783954f | 2015-12-21 18:06:29 -0800 | [diff] [blame] | 116 | Args: |
| 117 | trybot_image: artifact name such as |
Caroline Tice | 219e3b7 | 2018-12-18 15:54:49 -0800 | [diff] [blame] | 118 | 'daisy-release-tryjob/R40-6394.0.0-b1389' |
zhizhouy | 9258b05 | 2020-04-15 17:33:46 -0700 | [diff] [blame] | 119 | for recipe images, name is in this format: |
| 120 | 'lulu-llvm-next-nightly/R84-13037.0.0-31011-8883172717979984032/' |
Luis Lozano | 783954f | 2015-12-21 18:06:29 -0800 | [diff] [blame] | 121 | |
| 122 | Returns: |
Ting-Yuan Huang | e581987 | 2016-12-15 14:22:26 -0800 | [diff] [blame] | 123 | Latest official image name, e.g. 'daisy-release/R57-9089.0.0'. |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 124 | """ |
Yunlian Jiang | 2e0ad05 | 2017-11-22 14:06:45 -0800 | [diff] [blame] | 125 | # We need to filter out -tryjob in the trybot_image. |
zhizhouy | 9258b05 | 2020-04-15 17:33:46 -0700 | [diff] [blame] | 126 | if self._recipe: |
| 127 | trybot = re.sub('-llvm-next-nightly', '-release', trybot_image) |
| 128 | mo = re.search(RECIPE_IMAGE_RE, trybot) |
| 129 | else: |
| 130 | trybot = re.sub('-tryjob', '', trybot_image) |
| 131 | mo = re.search(TRYBOT_IMAGE_RE, trybot) |
Luis Lozano | c75fd05 | 2016-02-19 17:37:01 -0800 | [diff] [blame] | 132 | assert mo |
Ting-Yuan Huang | e581987 | 2016-12-15 14:22:26 -0800 | [diff] [blame] | 133 | dirname = IMAGE_DIR.replace('\\', '').format(**mo.groupdict()) |
Ting-Yuan Huang | 99d32c4 | 2017-04-24 20:34:43 -0700 | [diff] [blame] | 134 | return buildbot_utils.GetLatestImage(self._chromeos_root, dirname) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 135 | |
zhizhouy | 9258b05 | 2020-04-15 17:33:46 -0700 | [diff] [blame] | 136 | def _TestImages(self, trybot_image, vanilla_image): |
Yunlian Jiang | 14cf596 | 2015-12-11 15:50:14 -0800 | [diff] [blame] | 137 | """Create crosperf experiment file. |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 138 | |
Luis Lozano | 783954f | 2015-12-21 18:06:29 -0800 | [diff] [blame] | 139 | Given the names of the trybot, vanilla and non-AFDO images, create the |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 140 | appropriate crosperf experiment file and launch crosperf on it. |
| 141 | """ |
zhizhouy | 9258b05 | 2020-04-15 17:33:46 -0700 | [diff] [blame] | 142 | if self._test: |
| 143 | experiment_file_dir = TMP_TOOLCHAIN_TEST |
| 144 | else: |
| 145 | experiment_file_dir = os.path.join(NIGHTLY_TESTS_DIR, self._weekday) |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 146 | experiment_file_name = '%s_toolchain_experiment.txt' % self._board |
Yunlian Jiang | 2f56356 | 2015-08-28 13:54:04 -0700 | [diff] [blame] | 147 | |
Manoj Gupta | d575b8a | 2017-03-08 10:51:28 -0800 | [diff] [blame] | 148 | compiler_string = 'llvm' |
Manoj Gupta | c411035 | 2016-12-28 13:47:12 -0800 | [diff] [blame] | 149 | if USE_LLVM_NEXT_PATCH in self._patches_string: |
| 150 | experiment_file_name = '%s_llvm_next_experiment.txt' % self._board |
| 151 | compiler_string = 'llvm_next' |
Yunlian Jiang | 2f56356 | 2015-08-28 13:54:04 -0700 | [diff] [blame] | 152 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 153 | experiment_file = os.path.join(experiment_file_dir, experiment_file_name) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 154 | experiment_header = """ |
| 155 | board: %s |
| 156 | remote: %s |
Luis Lozano | e1efeb8 | 2015-06-16 16:35:44 -0700 | [diff] [blame] | 157 | retries: 1 |
Denis Nikitin | d7cded2 | 2019-09-12 13:38:50 -0700 | [diff] [blame] | 158 | cooldown_temp: 40 |
| 159 | cooldown_time: 10 |
Denis Nikitin | 858e145 | 2019-09-16 15:30:05 -0700 | [diff] [blame] | 160 | cpu_freq_pct: 95 |
Denis Nikitin | 30a061f | 2019-09-27 09:04:31 -0700 | [diff] [blame] | 161 | top_interval: 1 |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 162 | """ % (self._board, self._remotes) |
| 163 | experiment_tests = """ |
Luis Lozano | 1489d64 | 2015-12-08 10:08:19 -0800 | [diff] [blame] | 164 | benchmark: all_toolchain_perf { |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 165 | suite: telemetry_Crosperf |
Tiancong Wang | d034813 | 2019-03-07 11:22:48 -0800 | [diff] [blame] | 166 | iterations: 5 |
Manoj Gupta | 5a51638 | 2017-08-23 12:27:54 -0700 | [diff] [blame] | 167 | run_local: False |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 168 | } |
Caroline Tice | e82513b | 2016-10-27 12:45:15 -0700 | [diff] [blame] | 169 | |
Caroline Tice | 219e3b7 | 2018-12-18 15:54:49 -0800 | [diff] [blame] | 170 | benchmark: loading.desktop { |
Caroline Tice | e82513b | 2016-10-27 12:45:15 -0700 | [diff] [blame] | 171 | suite: telemetry_Crosperf |
Caroline Tice | 219e3b7 | 2018-12-18 15:54:49 -0800 | [diff] [blame] | 172 | test_args: --story-tag-filter=typical |
| 173 | iterations: 3 |
Caroline Tice | e82513b | 2016-10-27 12:45:15 -0700 | [diff] [blame] | 174 | run_local: False |
| 175 | retries: 0 |
| 176 | } |
Tiancong Wang | 0323466 | 2019-10-30 10:16:38 -0700 | [diff] [blame] | 177 | """ |
| 178 | telemetry_aquarium_tests = """ |
Tiancong Wang | d8bf281 | 2019-08-30 11:34:05 -0700 | [diff] [blame] | 179 | benchmark: rendering.desktop { |
| 180 | run_local: False |
| 181 | suite: telemetry_Crosperf |
| 182 | test_args: --story-filter=aquarium$ |
| 183 | iterations: 5 |
| 184 | } |
| 185 | |
| 186 | benchmark: rendering.desktop { |
| 187 | run_local: False |
| 188 | suite: telemetry_Crosperf |
| 189 | test_args: --story-filter=aquarium_20k$ |
| 190 | iterations: 3 |
| 191 | } |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 192 | """ |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 193 | |
Zhizhou Yang | 81d651f | 2020-02-10 16:51:20 -0800 | [diff] [blame] | 194 | with open(experiment_file, 'w', encoding='utf-8') as f: |
Yunlian Jiang | 14cf596 | 2015-12-11 15:50:14 -0800 | [diff] [blame] | 195 | f.write(experiment_header) |
| 196 | f.write(experiment_tests) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 197 | |
Tiancong Wang | 011c571 | 2019-11-01 10:33:34 -0700 | [diff] [blame] | 198 | if self._board not in TELEMETRY_AQUARIUM_UNSUPPORTED: |
Tiancong Wang | 0323466 | 2019-10-30 10:16:38 -0700 | [diff] [blame] | 199 | f.write(telemetry_aquarium_tests) |
| 200 | |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 201 | # Now add vanilla to test file. |
Yunlian Jiang | fceaba3 | 2018-06-06 09:08:44 -0700 | [diff] [blame] | 202 | official_image = """ |
Zhizhou Yang | cbc6a9d | 2020-02-13 15:51:36 -0800 | [diff] [blame] | 203 | vanilla_image { |
| 204 | chromeos_root: %s |
| 205 | build: %s |
| 206 | compiler: llvm |
| 207 | } |
| 208 | """ % (self._chromeos_root, vanilla_image) |
Yunlian Jiang | 14cf596 | 2015-12-11 15:50:14 -0800 | [diff] [blame] | 209 | f.write(official_image) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 210 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 211 | label_string = '%s_trybot_image' % compiler_string |
Caroline Tice | 80eab98 | 2015-11-04 14:03:14 -0800 | [diff] [blame] | 212 | |
Manoj Gupta | 3594db8 | 2017-01-31 11:48:57 -0800 | [diff] [blame] | 213 | # Reuse autotest files from vanilla image for trybot images |
| 214 | autotest_files = os.path.join('/tmp', vanilla_image, 'autotest_files') |
Yunlian Jiang | fceaba3 | 2018-06-06 09:08:44 -0700 | [diff] [blame] | 215 | experiment_image = """ |
Zhizhou Yang | cbc6a9d | 2020-02-13 15:51:36 -0800 | [diff] [blame] | 216 | %s { |
| 217 | chromeos_root: %s |
| 218 | build: %s |
| 219 | autotest_path: %s |
| 220 | compiler: %s |
| 221 | } |
| 222 | """ % (label_string, self._chromeos_root, trybot_image, autotest_files, |
| 223 | compiler_string) |
Yunlian Jiang | 14cf596 | 2015-12-11 15:50:14 -0800 | [diff] [blame] | 224 | f.write(experiment_image) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 225 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 226 | crosperf = os.path.join(TOOLCHAIN_DIR, 'crosperf', 'crosperf') |
Han Shen | 4349429 | 2015-09-14 10:26:40 -0700 | [diff] [blame] | 227 | noschedv2_opts = '--noschedv2' if self._noschedv2 else '' |
zhizhouy | 9258b05 | 2020-04-15 17:33:46 -0700 | [diff] [blame] | 228 | command = ('{crosperf} --no_email={no_email} --results_dir={r_dir} ' |
zhizhouy | eb6e55f | 2020-03-24 12:09:20 -0700 | [diff] [blame] | 229 | '--intel_pstate=no_hwp --logging_level=verbose ' |
Ting-Yuan Huang | b1afe3f | 2018-08-16 21:03:50 +0000 | [diff] [blame] | 230 | '--json_report=True {noschedv2_opts} {exp_file}').format( |
Yunlian Jiang | 14cf596 | 2015-12-11 15:50:14 -0800 | [diff] [blame] | 231 | crosperf=crosperf, |
zhizhouy | 9258b05 | 2020-04-15 17:33:46 -0700 | [diff] [blame] | 232 | no_email=not self._test, |
Yunlian Jiang | 14cf596 | 2015-12-11 15:50:14 -0800 | [diff] [blame] | 233 | r_dir=self._reports_dir, |
| 234 | noschedv2_opts=noschedv2_opts, |
| 235 | exp_file=experiment_file) |
cmtice | aa700b0 | 2015-06-12 13:26:47 -0700 | [diff] [blame] | 236 | |
zhizhouy | d8fcbf5 | 2020-04-06 16:47:20 -0700 | [diff] [blame] | 237 | return self._ce.RunCommand(command) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 238 | |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 239 | def _SendEmail(self): |
| 240 | """Find email message generated by crosperf and send it.""" |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 241 | filename = os.path.join(self._reports_dir, 'msg_body.html') |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 242 | if (os.path.exists(filename) and |
| 243 | os.path.exists(os.path.expanduser(MAIL_PROGRAM))): |
Manoj Gupta | d575b8a | 2017-03-08 10:51:28 -0800 | [diff] [blame] | 244 | email_title = 'buildbot llvm test results' |
Manoj Gupta | c411035 | 2016-12-28 13:47:12 -0800 | [diff] [blame] | 245 | if USE_LLVM_NEXT_PATCH in self._patches_string: |
| 246 | email_title = 'buildbot llvm_next test results' |
zhizhouy | d8fcbf5 | 2020-04-06 16:47:20 -0700 | [diff] [blame] | 247 | command = ('cat %s | %s -s "%s, %s %s" -team -html' % |
| 248 | (filename, MAIL_PROGRAM, email_title, self._board, self._date)) |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 249 | self._ce.RunCommand(command) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 250 | |
zhizhouy | d8fcbf5 | 2020-04-06 16:47:20 -0700 | [diff] [blame] | 251 | def _CopyJson(self): |
| 252 | # Copy json report to pending archives directory. |
| 253 | command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR) |
| 254 | ret = self._ce.RunCommand(command) |
| 255 | # Failing to access json report means that crosperf terminated or all tests |
| 256 | # failed, raise an error. |
| 257 | if ret != 0: |
| 258 | raise RuntimeError( |
| 259 | 'Crosperf failed to run tests, cannot copy json report!') |
| 260 | |
Ting-Yuan Huang | 6a9a98a | 2018-03-07 17:35:13 -0800 | [diff] [blame] | 261 | def DoAll(self): |
Yunlian Jiang | 14cf596 | 2015-12-11 15:50:14 -0800 | [diff] [blame] | 262 | """Main function inside ToolchainComparator class. |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 263 | |
| 264 | Launch trybot, get image names, create crosperf experiment file, run |
| 265 | crosperf, and copy images into seven-day report directories. |
| 266 | """ |
zhizhouy | 9258b05 | 2020-04-15 17:33:46 -0700 | [diff] [blame] | 267 | if self._recipe: |
| 268 | print('Using recipe buckets to get latest image.') |
Bob Haarman | 42e215f | 2020-05-01 14:15:28 -0700 | [diff] [blame^] | 269 | # crbug.com/1077313: Some boards are not consistently |
| 270 | # spelled, having underscores in some places and dashes in others. |
| 271 | # The image directories consistenly use dashes, so convert underscores |
| 272 | # to dashes to work around this. |
zhizhouy | 9258b05 | 2020-04-15 17:33:46 -0700 | [diff] [blame] | 273 | trybot_image = buildbot_utils.GetLatestRecipeImage( |
Bob Haarman | 42e215f | 2020-05-01 14:15:28 -0700 | [diff] [blame^] | 274 | self._chromeos_root, '%s-llvm-next-nightly' % self._board.replace('_', '-')) |
zhizhouy | 9258b05 | 2020-04-15 17:33:46 -0700 | [diff] [blame] | 275 | else: |
| 276 | # Launch tryjob and wait to get image location. |
| 277 | buildbucket_id, trybot_image = buildbot_utils.GetTrybotImage( |
| 278 | self._chromeos_root, |
| 279 | self._build, |
| 280 | self._patches, |
| 281 | tryjob_flags=['--notests'], |
| 282 | build_toolchain=True) |
| 283 | print('trybot_url: \ |
| 284 | http://cros-goldeneye/chromeos/healthmonitoring/buildDetails?buildbucketId=%s' |
| 285 | % buildbucket_id) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 286 | |
Tiancong Wang | 0323466 | 2019-10-30 10:16:38 -0700 | [diff] [blame] | 287 | if not trybot_image: |
Caroline Tice | efb7990 | 2018-04-18 11:23:01 -0700 | [diff] [blame] | 288 | self._l.LogError('Unable to find trybot_image!') |
Yunlian Jiang | cdd1907 | 2017-09-29 10:15:56 -0700 | [diff] [blame] | 289 | return 2 |
Luis Lozano | 783954f | 2015-12-21 18:06:29 -0800 | [diff] [blame] | 290 | |
Luis Lozano | c75fd05 | 2016-02-19 17:37:01 -0800 | [diff] [blame] | 291 | vanilla_image = self._GetVanillaImageName(trybot_image) |
Luis Lozano | 783954f | 2015-12-21 18:06:29 -0800 | [diff] [blame] | 292 | |
| 293 | print('trybot_image: %s' % trybot_image) |
| 294 | print('vanilla_image: %s' % vanilla_image) |
Luis Lozano | c75fd05 | 2016-02-19 17:37:01 -0800 | [diff] [blame] | 295 | |
zhizhouy | 9258b05 | 2020-04-15 17:33:46 -0700 | [diff] [blame] | 296 | ret = self._TestImages(trybot_image, vanilla_image) |
zhizhouy | d8fcbf5 | 2020-04-06 16:47:20 -0700 | [diff] [blame] | 297 | # Always try to send report email as crosperf will generate report when |
| 298 | # tests partially succeeded. |
zhizhouy | 9258b05 | 2020-04-15 17:33:46 -0700 | [diff] [blame] | 299 | if not self._test: |
| 300 | self._SendEmail() |
| 301 | self._CopyJson() |
zhizhouy | d8fcbf5 | 2020-04-06 16:47:20 -0700 | [diff] [blame] | 302 | # Non-zero ret here means crosperf tests partially failed, raise error here |
| 303 | # so that toolchain summary report can catch it. |
| 304 | if ret != 0: |
| 305 | raise RuntimeError('Crosperf tests partially failed!') |
| 306 | |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 307 | return 0 |
| 308 | |
| 309 | |
| 310 | def Main(argv): |
| 311 | """The main function.""" |
| 312 | |
| 313 | # Common initializations |
| 314 | command_executer.InitCommandExecuter() |
Caroline Tice | eddb063 | 2016-04-14 09:19:02 -0700 | [diff] [blame] | 315 | parser = argparse.ArgumentParser() |
Manoj Gupta | aee96b7 | 2016-10-24 13:43:28 -0700 | [diff] [blame] | 316 | parser.add_argument( |
| 317 | '--remote', dest='remote', help='Remote machines to run tests on.') |
| 318 | parser.add_argument( |
| 319 | '--board', dest='board', default='x86-zgb', help='The target board.') |
| 320 | parser.add_argument( |
| 321 | '--chromeos_root', |
| 322 | dest='chromeos_root', |
| 323 | help='The chromeos root from which to run tests.') |
| 324 | parser.add_argument( |
| 325 | '--weekday', |
| 326 | default='', |
| 327 | dest='weekday', |
| 328 | help='The day of the week for which to run tests.') |
| 329 | parser.add_argument( |
| 330 | '--patch', |
| 331 | dest='patches', |
| 332 | help='The patches to use for the testing, ' |
| 333 | "seprate the patch numbers with ',' " |
| 334 | 'for more than one patches.') |
| 335 | parser.add_argument( |
| 336 | '--noschedv2', |
| 337 | dest='noschedv2', |
| 338 | action='store_true', |
| 339 | default=False, |
| 340 | help='Pass --noschedv2 to crosperf.') |
zhizhouy | 9258b05 | 2020-04-15 17:33:46 -0700 | [diff] [blame] | 341 | parser.add_argument( |
| 342 | '--recipe', |
| 343 | dest='recipe', |
| 344 | default=True, |
| 345 | help='Use images generated from recipe rather than' |
| 346 | 'launching tryjob to get images.') |
| 347 | parser.add_argument( |
| 348 | '--test', |
| 349 | dest='test', |
| 350 | default=False, |
| 351 | help='Test this script on local desktop, ' |
| 352 | 'disabling mobiletc checking and email sending.' |
| 353 | 'Artifacts stored in /tmp/toolchain-tests') |
Han Shen | 3641312 | 2015-08-28 11:05:40 -0700 | [diff] [blame] | 354 | |
Caroline Tice | eddb063 | 2016-04-14 09:19:02 -0700 | [diff] [blame] | 355 | options = parser.parse_args(argv[1:]) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 356 | if not options.board: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 357 | print('Please give a board.') |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 358 | return 1 |
| 359 | if not options.remote: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 360 | print('Please give at least one remote machine.') |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 361 | return 1 |
| 362 | if not options.chromeos_root: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 363 | print('Please specify the ChromeOS root directory.') |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 364 | return 1 |
zhizhouy | 9258b05 | 2020-04-15 17:33:46 -0700 | [diff] [blame] | 365 | if options.test: |
| 366 | print('Cleaning local test directory for this script.') |
| 367 | if os.path.exists(TMP_TOOLCHAIN_TEST): |
| 368 | shutil.rmtree(TMP_TOOLCHAIN_TEST) |
| 369 | os.mkdir(TMP_TOOLCHAIN_TEST) |
Yunlian Jiang | e52838c | 2015-08-20 14:32:37 -0700 | [diff] [blame] | 370 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 371 | fc = ToolchainComparator(options.board, options.remote, options.chromeos_root, |
zhizhouy | 9258b05 | 2020-04-15 17:33:46 -0700 | [diff] [blame] | 372 | options.weekday, options.patches, options.recipe, |
| 373 | options.test, options.noschedv2) |
Ting-Yuan Huang | 6a9a98a | 2018-03-07 17:35:13 -0800 | [diff] [blame] | 374 | return fc.DoAll() |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 375 | |
| 376 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 377 | if __name__ == '__main__': |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 378 | retval = Main(sys.argv) |
| 379 | sys.exit(retval) |