blob: c2d88733e4208d9b0ebed9d6da943e2fe329ab9d [file] [log] [blame]
Yunlian Jiange84ea3d2016-12-12 11:07:40 -08001#!/usr/bin/env python2
Tiancong Wangd0348132019-03-07 11:22:48 -08002# -*- coding: utf-8 -*-
Ting-Yuan Huange5819872016-12-15 14:22:26 -08003#
4# Copyright 2016 The Chromium OS Authors. All rights reserved.
5# Use of this source code is governed by a BSD-style license that can be
6# found in the LICENSE file.
Yunlian Jiang14cf5962015-12-11 15:50:14 -08007"""Script for running nightly compiler tests on ChromeOS.
cmtice46093e52014-12-09 14:59:16 -08008
9This script launches a buildbot to build ChromeOS with the latest compiler on
10a particular board; then it finds and downloads the trybot image and the
11corresponding official image, and runs crosperf performance tests comparing
12the two. It then generates a report, emails it to the c-compiler-chrome, as
13well as copying the images into the seven-day reports directory.
14"""
15
16# Script to test different toolchains against ChromeOS benchmarks.
Yunlian Jiang14cf5962015-12-11 15:50:14 -080017
18from __future__ import print_function
19
Caroline Ticeeddb0632016-04-14 09:19:02 -070020import argparse
cmticece5ffa42015-02-12 15:18:43 -080021import datetime
cmtice46093e52014-12-09 14:59:16 -080022import os
Luis Lozanoc75fd052016-02-19 17:37:01 -080023import re
cmtice46093e52014-12-09 14:59:16 -080024import sys
25import time
cmtice46093e52014-12-09 14:59:16 -080026
Caroline Ticea8af9a72016-07-20 12:52:59 -070027from cros_utils import command_executer
28from cros_utils import logger
cmtice46093e52014-12-09 14:59:16 -080029
Caroline Ticea8af9a72016-07-20 12:52:59 -070030from cros_utils import buildbot_utils
cmtice46093e52014-12-09 14:59:16 -080031
Manoj Guptac4110352016-12-28 13:47:12 -080032# CL that uses LLVM-Next to build the images (includes chrome).
Manoj Gupta66682c72017-05-24 12:19:57 -070033USE_LLVM_NEXT_PATCH = '513590'
Manoj Guptac4110352016-12-28 13:47:12 -080034
Luis Lozanof2a3ef42015-12-15 13:49:30 -080035CROSTC_ROOT = '/usr/local/google/crostc'
36ROLE_ACCOUNT = 'mobiletc-prebuild'
cmtice46093e52014-12-09 14:59:16 -080037TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
Luis Lozanof2a3ef42015-12-15 13:49:30 -080038MAIL_PROGRAM = '~/var/bin/mail-sheriff'
Luis Lozanof2a3ef42015-12-15 13:49:30 -080039PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, 'pending_archives')
40NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly_test_reports')
41
Ting-Yuan Huange5819872016-12-15 14:22:26 -080042IMAGE_DIR = '{board}-{image_type}'
43IMAGE_VERSION_STR = r'{chrome_version}-{tip}\.{branch}\.{branch_branch}'
44IMAGE_FS = IMAGE_DIR + '/' + IMAGE_VERSION_STR
Ting-Yuan Huang112d5622018-03-26 13:49:42 -070045TRYBOT_IMAGE_FS = IMAGE_FS + '-{build_id}'
Luis Lozanoc75fd052016-02-19 17:37:01 -080046PFQ_IMAGE_FS = IMAGE_FS + '-rc1'
Manoj Guptaaee96b72016-10-24 13:43:28 -070047IMAGE_RE_GROUPS = {
48 'board': r'(?P<board>\S+)',
49 'image_type': r'(?P<image_type>\S+)',
50 'chrome_version': r'(?P<chrome_version>R\d+)',
51 'tip': r'(?P<tip>\d+)',
52 'branch': r'(?P<branch>\d+)',
53 'branch_branch': r'(?P<branch_branch>\d+)',
54 'build_id': r'(?P<build_id>b\d+)'
55}
Luis Lozanoc75fd052016-02-19 17:37:01 -080056TRYBOT_IMAGE_RE = TRYBOT_IMAGE_FS.format(**IMAGE_RE_GROUPS)
57
cmtice46093e52014-12-09 14:59:16 -080058
Yunlian Jiang14cf5962015-12-11 15:50:14 -080059class ToolchainComparator(object):
60 """Class for doing the nightly tests work."""
cmtice46093e52014-12-09 14:59:16 -080061
Luis Lozanof2a3ef42015-12-15 13:49:30 -080062 def __init__(self,
63 board,
64 remotes,
65 chromeos_root,
66 weekday,
67 patches,
68 noschedv2=False):
cmtice46093e52014-12-09 14:59:16 -080069 self._board = board
70 self._remotes = remotes
71 self._chromeos_root = chromeos_root
72 self._base_dir = os.getcwd()
73 self._ce = command_executer.GetCommandExecuter()
74 self._l = logger.GetLogger()
Yunlian Jiang73580dd2017-11-21 04:48:40 -080075 self._build = '%s-release-tryjob' % board
Manoj Guptad575b8a2017-03-08 10:51:28 -080076 self._patches = patches.split(',') if patches else []
Yunlian Jiang3c6e4672015-08-24 15:58:22 -070077 self._patches_string = '_'.join(str(p) for p in self._patches)
Han Shen43494292015-09-14 10:26:40 -070078 self._noschedv2 = noschedv2
Yunlian Jiang3c6e4672015-08-24 15:58:22 -070079
cmtice46093e52014-12-09 14:59:16 -080080 if not weekday:
Luis Lozanof2a3ef42015-12-15 13:49:30 -080081 self._weekday = time.strftime('%a')
cmtice46093e52014-12-09 14:59:16 -080082 else:
83 self._weekday = weekday
cmtice7f3190b2015-05-22 14:14:51 -070084 timestamp = datetime.datetime.strftime(datetime.datetime.now(),
Luis Lozanof2a3ef42015-12-15 13:49:30 -080085 '%Y-%m-%d_%H:%M:%S')
Manoj Guptaaee96b72016-10-24 13:43:28 -070086 self._reports_dir = os.path.join(
87 NIGHTLY_TESTS_DIR,
Caroline Tice9c4003a2017-11-07 16:37:33 -080088 '%s.%s' % (timestamp, board),
89 )
cmtice46093e52014-12-09 14:59:16 -080090
Luis Lozanoc75fd052016-02-19 17:37:01 -080091 def _GetVanillaImageName(self, trybot_image):
Ting-Yuan Huange5819872016-12-15 14:22:26 -080092 """Given a trybot artifact name, get latest vanilla image name.
cmtice46093e52014-12-09 14:59:16 -080093
Luis Lozano783954f2015-12-21 18:06:29 -080094 Args:
95 trybot_image: artifact name such as
Caroline Tice219e3b72018-12-18 15:54:49 -080096 'daisy-release-tryjob/R40-6394.0.0-b1389'
Luis Lozano783954f2015-12-21 18:06:29 -080097
98 Returns:
Ting-Yuan Huange5819872016-12-15 14:22:26 -080099 Latest official image name, e.g. 'daisy-release/R57-9089.0.0'.
cmtice46093e52014-12-09 14:59:16 -0800100 """
Yunlian Jiang2e0ad052017-11-22 14:06:45 -0800101 # We need to filter out -tryjob in the trybot_image.
102 trybot = re.sub('-tryjob', '', trybot_image)
103 mo = re.search(TRYBOT_IMAGE_RE, trybot)
Luis Lozanoc75fd052016-02-19 17:37:01 -0800104 assert mo
Ting-Yuan Huange5819872016-12-15 14:22:26 -0800105 dirname = IMAGE_DIR.replace('\\', '').format(**mo.groupdict())
Ting-Yuan Huang99d32c42017-04-24 20:34:43 -0700106 return buildbot_utils.GetLatestImage(self._chromeos_root, dirname)
cmtice46093e52014-12-09 14:59:16 -0800107
Luis Lozanoc75fd052016-02-19 17:37:01 -0800108 def _GetNonAFDOImageName(self, trybot_image):
109 """Given a trybot artifact name, get corresponding non-AFDO image name.
110
111 We get the non-AFDO image from the PFQ builders. This image
112 is not generated for all the boards and, the closest PFQ image
113 was the one build for the previous ChromeOS version (the chrome
114 used in the current version is the one validated in the previous
115 version).
116 The previous ChromeOS does not always exist either. So, we try
117 a couple of versions before.
Luis Lozano783954f2015-12-21 18:06:29 -0800118
119 Args:
120 trybot_image: artifact name such as
Caroline Tice219e3b72018-12-18 15:54:49 -0800121 'daisy-release-tryjob/R40-6394.0.0-b1389'
Luis Lozano783954f2015-12-21 18:06:29 -0800122
123 Returns:
124 Corresponding chrome PFQ image name, e.g.
Luis Lozanoc75fd052016-02-19 17:37:01 -0800125 'daisy-chrome-pfq/R40-6393.0.0-rc1'.
Luis Lozano783954f2015-12-21 18:06:29 -0800126 """
Yunlian Jiang2e0ad052017-11-22 14:06:45 -0800127 trybot = re.sub('-tryjob', '', trybot_image)
128 mo = re.search(TRYBOT_IMAGE_RE, trybot)
Luis Lozanoc75fd052016-02-19 17:37:01 -0800129 assert mo
130 image_dict = mo.groupdict()
131 image_dict['image_type'] = 'chrome-pfq'
132 for _ in xrange(2):
133 image_dict['tip'] = str(int(image_dict['tip']) - 1)
134 nonafdo_image = PFQ_IMAGE_FS.replace('\\', '').format(**image_dict)
135 if buildbot_utils.DoesImageExist(self._chromeos_root, nonafdo_image):
136 return nonafdo_image
137 return ''
Luis Lozano783954f2015-12-21 18:06:29 -0800138
Luis Lozano783954f2015-12-21 18:06:29 -0800139 def _TestImages(self, trybot_image, vanilla_image, nonafdo_image):
Yunlian Jiang14cf5962015-12-11 15:50:14 -0800140 """Create crosperf experiment file.
cmtice46093e52014-12-09 14:59:16 -0800141
Luis Lozano783954f2015-12-21 18:06:29 -0800142 Given the names of the trybot, vanilla and non-AFDO images, create the
cmtice46093e52014-12-09 14:59:16 -0800143 appropriate crosperf experiment file and launch crosperf on it.
144 """
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800145 experiment_file_dir = os.path.join(self._chromeos_root, '..', self._weekday)
146 experiment_file_name = '%s_toolchain_experiment.txt' % self._board
Yunlian Jiang2f563562015-08-28 13:54:04 -0700147
Manoj Guptad575b8a2017-03-08 10:51:28 -0800148 compiler_string = 'llvm'
Manoj Guptac4110352016-12-28 13:47:12 -0800149 if USE_LLVM_NEXT_PATCH in self._patches_string:
150 experiment_file_name = '%s_llvm_next_experiment.txt' % self._board
151 compiler_string = 'llvm_next'
Yunlian Jiang2f563562015-08-28 13:54:04 -0700152
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800153 experiment_file = os.path.join(experiment_file_dir, experiment_file_name)
cmtice46093e52014-12-09 14:59:16 -0800154 experiment_header = """
155 board: %s
156 remote: %s
Luis Lozanoe1efeb82015-06-16 16:35:44 -0700157 retries: 1
cmtice46093e52014-12-09 14:59:16 -0800158 """ % (self._board, self._remotes)
159 experiment_tests = """
Luis Lozano1489d642015-12-08 10:08:19 -0800160 benchmark: all_toolchain_perf {
cmtice46093e52014-12-09 14:59:16 -0800161 suite: telemetry_Crosperf
Tiancong Wangd0348132019-03-07 11:22:48 -0800162 iterations: 5
Manoj Gupta5a516382017-08-23 12:27:54 -0700163 run_local: False
cmtice46093e52014-12-09 14:59:16 -0800164 }
Caroline Ticee82513b2016-10-27 12:45:15 -0700165
Caroline Tice219e3b72018-12-18 15:54:49 -0800166 benchmark: loading.desktop {
Caroline Ticee82513b2016-10-27 12:45:15 -0700167 suite: telemetry_Crosperf
Caroline Tice219e3b72018-12-18 15:54:49 -0800168 test_args: --story-tag-filter=typical
169 iterations: 3
Caroline Ticee82513b2016-10-27 12:45:15 -0700170 run_local: False
171 retries: 0
172 }
cmtice46093e52014-12-09 14:59:16 -0800173 """
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800174
175 with open(experiment_file, 'w') as f:
Yunlian Jiang14cf5962015-12-11 15:50:14 -0800176 f.write(experiment_header)
177 f.write(experiment_tests)
cmtice46093e52014-12-09 14:59:16 -0800178
179 # Now add vanilla to test file.
Yunlian Jiangfceaba32018-06-06 09:08:44 -0700180 official_image = """
cmtice46093e52014-12-09 14:59:16 -0800181 vanilla_image {
182 chromeos_root: %s
183 build: %s
Yunlian Jiang8c18be12017-03-20 16:52:33 -0700184 compiler: llvm
cmtice46093e52014-12-09 14:59:16 -0800185 }
186 """ % (self._chromeos_root, vanilla_image)
Yunlian Jiang14cf5962015-12-11 15:50:14 -0800187 f.write(official_image)
cmtice46093e52014-12-09 14:59:16 -0800188
Luis Lozano783954f2015-12-21 18:06:29 -0800189 # Now add non-AFDO image to test file.
Luis Lozano439f2b72016-01-08 11:56:02 -0800190 if nonafdo_image:
Yunlian Jiangfceaba32018-06-06 09:08:44 -0700191 official_nonafdo_image = """
Luis Lozano783954f2015-12-21 18:06:29 -0800192 nonafdo_image {
193 chromeos_root: %s
194 build: %s
Yunlian Jiang8c18be12017-03-20 16:52:33 -0700195 compiler: llvm
Luis Lozano783954f2015-12-21 18:06:29 -0800196 }
197 """ % (self._chromeos_root, nonafdo_image)
Luis Lozano439f2b72016-01-08 11:56:02 -0800198 f.write(official_nonafdo_image)
Luis Lozano783954f2015-12-21 18:06:29 -0800199
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800200 label_string = '%s_trybot_image' % compiler_string
Caroline Tice80eab982015-11-04 14:03:14 -0800201
Manoj Gupta3594db82017-01-31 11:48:57 -0800202 # Reuse autotest files from vanilla image for trybot images
203 autotest_files = os.path.join('/tmp', vanilla_image, 'autotest_files')
Yunlian Jiangfceaba32018-06-06 09:08:44 -0700204 experiment_image = """
Caroline Tice80eab982015-11-04 14:03:14 -0800205 %s {
cmtice46093e52014-12-09 14:59:16 -0800206 chromeos_root: %s
207 build: %s
Manoj Gupta3594db82017-01-31 11:48:57 -0800208 autotest_path: %s
Caroline Ticeddde5052015-09-23 09:43:35 -0700209 compiler: %s
cmtice46093e52014-12-09 14:59:16 -0800210 }
Caroline Tice80eab982015-11-04 14:03:14 -0800211 """ % (label_string, self._chromeos_root, trybot_image,
Manoj Gupta3594db82017-01-31 11:48:57 -0800212 autotest_files, compiler_string)
Yunlian Jiang14cf5962015-12-11 15:50:14 -0800213 f.write(experiment_image)
cmtice46093e52014-12-09 14:59:16 -0800214
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800215 crosperf = os.path.join(TOOLCHAIN_DIR, 'crosperf', 'crosperf')
Han Shen43494292015-09-14 10:26:40 -0700216 noschedv2_opts = '--noschedv2' if self._noschedv2 else ''
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800217 command = ('{crosperf} --no_email=True --results_dir={r_dir} '
Ting-Yuan Huangb1afe3f2018-08-16 21:03:50 +0000218 '--json_report=True {noschedv2_opts} {exp_file}').format(
Yunlian Jiang14cf5962015-12-11 15:50:14 -0800219 crosperf=crosperf,
220 r_dir=self._reports_dir,
221 noschedv2_opts=noschedv2_opts,
222 exp_file=experiment_file)
cmticeaa700b02015-06-12 13:26:47 -0700223
cmtice46093e52014-12-09 14:59:16 -0800224 ret = self._ce.RunCommand(command)
cmtice7f3190b2015-05-22 14:14:51 -0700225 if ret != 0:
Manoj Guptaaee96b72016-10-24 13:43:28 -0700226 raise RuntimeError('Crosperf execution error!')
Caroline Ticeebbc3da2015-09-03 10:27:20 -0700227 else:
228 # Copy json report to pending archives directory.
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800229 command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR)
Caroline Ticeebbc3da2015-09-03 10:27:20 -0700230 ret = self._ce.RunCommand(command)
cmtice7f3190b2015-05-22 14:14:51 -0700231 return
cmtice46093e52014-12-09 14:59:16 -0800232
cmtice7f3190b2015-05-22 14:14:51 -0700233 def _SendEmail(self):
234 """Find email message generated by crosperf and send it."""
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800235 filename = os.path.join(self._reports_dir, 'msg_body.html')
cmtice7f3190b2015-05-22 14:14:51 -0700236 if (os.path.exists(filename) and
237 os.path.exists(os.path.expanduser(MAIL_PROGRAM))):
Manoj Guptad575b8a2017-03-08 10:51:28 -0800238 email_title = 'buildbot llvm test results'
Manoj Guptac4110352016-12-28 13:47:12 -0800239 if USE_LLVM_NEXT_PATCH in self._patches_string:
240 email_title = 'buildbot llvm_next test results'
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800241 command = ('cat %s | %s -s "%s, %s" -team -html' %
242 (filename, MAIL_PROGRAM, email_title, self._board))
cmtice7f3190b2015-05-22 14:14:51 -0700243 self._ce.RunCommand(command)
cmtice46093e52014-12-09 14:59:16 -0800244
Ting-Yuan Huang6a9a98a2018-03-07 17:35:13 -0800245 def DoAll(self):
Yunlian Jiang14cf5962015-12-11 15:50:14 -0800246 """Main function inside ToolchainComparator class.
cmtice46093e52014-12-09 14:59:16 -0800247
248 Launch trybot, get image names, create crosperf experiment file, run
249 crosperf, and copy images into seven-day report directories.
250 """
Ting-Yuan Huang6a9a98a2018-03-07 17:35:13 -0800251 buildbucket_id, trybot_image = buildbot_utils.GetTrybotImage(
252 self._chromeos_root,
253 self._build,
254 self._patches,
Jian Cai9d576402019-06-17 20:49:35 +0000255 tryjob_flags=['--notests'],
Ting-Yuan Huang6a9a98a2018-03-07 17:35:13 -0800256 build_toolchain=True)
cmtice46093e52014-12-09 14:59:16 -0800257
Yunlian Jiange84ea3d2016-12-12 11:07:40 -0800258 print('trybot_url: \
Ting-Yuan Huang6a9a98a2018-03-07 17:35:13 -0800259 http://cros-goldeneye/chromeos/healthmonitoring/buildDetails?buildbucketId=%s'
260 % buildbucket_id)
cmticed54f9802015-02-05 11:04:11 -0800261 if len(trybot_image) == 0:
Caroline Ticeefb79902018-04-18 11:23:01 -0700262 self._l.LogError('Unable to find trybot_image!')
Yunlian Jiangcdd19072017-09-29 10:15:56 -0700263 return 2
Luis Lozano783954f2015-12-21 18:06:29 -0800264
Luis Lozanoc75fd052016-02-19 17:37:01 -0800265 vanilla_image = self._GetVanillaImageName(trybot_image)
266 nonafdo_image = self._GetNonAFDOImageName(trybot_image)
Luis Lozano783954f2015-12-21 18:06:29 -0800267
268 print('trybot_image: %s' % trybot_image)
269 print('vanilla_image: %s' % vanilla_image)
270 print('nonafdo_image: %s' % nonafdo_image)
Luis Lozanoc75fd052016-02-19 17:37:01 -0800271
Luis Lozano783954f2015-12-21 18:06:29 -0800272 self._TestImages(trybot_image, vanilla_image, nonafdo_image)
cmtice7f3190b2015-05-22 14:14:51 -0700273 self._SendEmail()
cmtice46093e52014-12-09 14:59:16 -0800274 return 0
275
276
277def Main(argv):
278 """The main function."""
279
280 # Common initializations
281 command_executer.InitCommandExecuter()
Caroline Ticeeddb0632016-04-14 09:19:02 -0700282 parser = argparse.ArgumentParser()
Manoj Guptaaee96b72016-10-24 13:43:28 -0700283 parser.add_argument(
284 '--remote', dest='remote', help='Remote machines to run tests on.')
285 parser.add_argument(
286 '--board', dest='board', default='x86-zgb', help='The target board.')
287 parser.add_argument(
288 '--chromeos_root',
289 dest='chromeos_root',
290 help='The chromeos root from which to run tests.')
291 parser.add_argument(
292 '--weekday',
293 default='',
294 dest='weekday',
295 help='The day of the week for which to run tests.')
296 parser.add_argument(
297 '--patch',
298 dest='patches',
299 help='The patches to use for the testing, '
300 "seprate the patch numbers with ',' "
301 'for more than one patches.')
302 parser.add_argument(
303 '--noschedv2',
304 dest='noschedv2',
305 action='store_true',
306 default=False,
307 help='Pass --noschedv2 to crosperf.')
Han Shen36413122015-08-28 11:05:40 -0700308
Caroline Ticeeddb0632016-04-14 09:19:02 -0700309 options = parser.parse_args(argv[1:])
cmtice46093e52014-12-09 14:59:16 -0800310 if not options.board:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800311 print('Please give a board.')
cmtice46093e52014-12-09 14:59:16 -0800312 return 1
313 if not options.remote:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800314 print('Please give at least one remote machine.')
cmtice46093e52014-12-09 14:59:16 -0800315 return 1
316 if not options.chromeos_root:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800317 print('Please specify the ChromeOS root directory.')
cmtice46093e52014-12-09 14:59:16 -0800318 return 1
Yunlian Jiange52838c2015-08-20 14:32:37 -0700319
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800320 fc = ToolchainComparator(options.board, options.remote, options.chromeos_root,
Manoj Guptad575b8a2017-03-08 10:51:28 -0800321 options.weekday, options.patches, options.noschedv2)
Ting-Yuan Huang6a9a98a2018-03-07 17:35:13 -0800322 return fc.DoAll()
cmtice46093e52014-12-09 14:59:16 -0800323
324
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800325if __name__ == '__main__':
cmtice46093e52014-12-09 14:59:16 -0800326 retval = Main(sys.argv)
327 sys.exit(retval)