blob: feb11e1631bec523aecfd74bcbb976b7e9ee0d91 [file] [log] [blame]
Yunlian Jiang14cf5962015-12-11 15:50:14 -08001#!/usr/bin/python2
2"""Script for running nightly compiler tests on ChromeOS.
cmtice46093e52014-12-09 14:59:16 -08003
4This script launches a buildbot to build ChromeOS with the latest compiler on
5a particular board; then it finds and downloads the trybot image and the
6corresponding official image, and runs crosperf performance tests comparing
7the two. It then generates a report, emails it to the c-compiler-chrome, as
8well as copying the images into the seven-day reports directory.
9"""
10
11# Script to test different toolchains against ChromeOS benchmarks.
Yunlian Jiang14cf5962015-12-11 15:50:14 -080012
13from __future__ import print_function
14
cmticece5ffa42015-02-12 15:18:43 -080015import datetime
cmtice46093e52014-12-09 14:59:16 -080016import optparse
17import os
18import sys
19import time
cmtice46093e52014-12-09 14:59:16 -080020
21from utils import command_executer
22from utils import logger
23
24from utils import buildbot_utils
25
26# CL that updated GCC ebuilds to use 'next_gcc'.
Luis Lozanof2a3ef42015-12-15 13:49:30 -080027USE_NEXT_GCC_PATCH = '230260'
Yunlian Jiang3c6e4672015-08-24 15:58:22 -070028
Yunlian Jiang2f563562015-08-28 13:54:04 -070029# CL that uses LLVM to build the peppy image.
Luis Lozanof2a3ef42015-12-15 13:49:30 -080030USE_LLVM_PATCH = '295217'
Yunlian Jiang2f563562015-08-28 13:54:04 -070031
Yunlian Jiang3c6e4672015-08-24 15:58:22 -070032# The boards on which we run weekly reports
Luis Lozanof2a3ef42015-12-15 13:49:30 -080033WEEKLY_REPORT_BOARDS = ['lumpy']
cmtice46093e52014-12-09 14:59:16 -080034
Luis Lozanof2a3ef42015-12-15 13:49:30 -080035CROSTC_ROOT = '/usr/local/google/crostc'
36ROLE_ACCOUNT = 'mobiletc-prebuild'
cmtice46093e52014-12-09 14:59:16 -080037TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
Luis Lozanof2a3ef42015-12-15 13:49:30 -080038MAIL_PROGRAM = '~/var/bin/mail-sheriff'
39WEEKLY_REPORTS_ROOT = os.path.join(CROSTC_ROOT, 'weekly_test_data')
40PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, 'pending_archives')
41NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly_test_reports')
42
cmtice46093e52014-12-09 14:59:16 -080043
Yunlian Jiang14cf5962015-12-11 15:50:14 -080044class ToolchainComparator(object):
45 """Class for doing the nightly tests work."""
cmtice46093e52014-12-09 14:59:16 -080046
Luis Lozanof2a3ef42015-12-15 13:49:30 -080047 def __init__(self,
48 board,
49 remotes,
50 chromeos_root,
51 weekday,
52 patches,
53 noschedv2=False):
cmtice46093e52014-12-09 14:59:16 -080054 self._board = board
55 self._remotes = remotes
56 self._chromeos_root = chromeos_root
57 self._base_dir = os.getcwd()
58 self._ce = command_executer.GetCommandExecuter()
59 self._l = logger.GetLogger()
Luis Lozanof2a3ef42015-12-15 13:49:30 -080060 self._build = '%s-release' % board
Yunlian Jiang3c6e4672015-08-24 15:58:22 -070061 self._patches = patches.split(',')
62 self._patches_string = '_'.join(str(p) for p in self._patches)
Han Shen43494292015-09-14 10:26:40 -070063 self._noschedv2 = noschedv2
Yunlian Jiang3c6e4672015-08-24 15:58:22 -070064
cmtice46093e52014-12-09 14:59:16 -080065 if not weekday:
Luis Lozanof2a3ef42015-12-15 13:49:30 -080066 self._weekday = time.strftime('%a')
cmtice46093e52014-12-09 14:59:16 -080067 else:
68 self._weekday = weekday
cmtice7f3190b2015-05-22 14:14:51 -070069 timestamp = datetime.datetime.strftime(datetime.datetime.now(),
Luis Lozanof2a3ef42015-12-15 13:49:30 -080070 '%Y-%m-%d_%H:%M:%S')
Caroline Ticeebbc3da2015-09-03 10:27:20 -070071 self._reports_dir = os.path.join(NIGHTLY_TESTS_DIR,
Luis Lozanof2a3ef42015-12-15 13:49:30 -080072 '%s.%s' % (timestamp, board),)
cmtice46093e52014-12-09 14:59:16 -080073
74 def _ParseVanillaImage(self, trybot_image):
Yunlian Jiang14cf5962015-12-11 15:50:14 -080075 """Parse a trybot artifact name to get corresponding vanilla image.
cmtice46093e52014-12-09 14:59:16 -080076
77 This function takes an artifact name, such as
78 'trybot-daisy-release/R40-6394.0.0-b1389', and returns the
79 corresponding official build name, e.g. 'daisy-release/R40-6394.0.0'.
80 """
81 start_pos = trybot_image.find(self._build)
Luis Lozanof2a3ef42015-12-15 13:49:30 -080082 end_pos = trybot_image.rfind('-b')
cmtice46093e52014-12-09 14:59:16 -080083 vanilla_image = trybot_image[start_pos:end_pos]
84 return vanilla_image
85
cmtice46093e52014-12-09 14:59:16 -080086 def _FinishSetup(self):
Yunlian Jiang14cf5962015-12-11 15:50:14 -080087 """Make sure testing_rsa file is properly set up."""
cmtice46093e52014-12-09 14:59:16 -080088 # Fix protections on ssh key
Luis Lozanof2a3ef42015-12-15 13:49:30 -080089 command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target'
90 '/chrome-src-internal/src/third_party/chromite/ssh_keys'
91 '/testing_rsa')
cmtice46093e52014-12-09 14:59:16 -080092 ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command)
cmtice7f3190b2015-05-22 14:14:51 -070093 if ret_val != 0:
Luis Lozanof2a3ef42015-12-15 13:49:30 -080094 raise RuntimeError('chmod for testing_rsa failed')
cmtice46093e52014-12-09 14:59:16 -080095
96 def _TestImages(self, trybot_image, vanilla_image):
Yunlian Jiang14cf5962015-12-11 15:50:14 -080097 """Create crosperf experiment file.
cmtice46093e52014-12-09 14:59:16 -080098
99 Given the names of the trybot and vanilla images, create the
100 appropriate crosperf experiment file and launch crosperf on it.
101 """
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800102 experiment_file_dir = os.path.join(self._chromeos_root, '..', self._weekday)
103 experiment_file_name = '%s_toolchain_experiment.txt' % self._board
Yunlian Jiang2f563562015-08-28 13:54:04 -0700104
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800105 compiler_string = 'gcc'
Yunlian Jiang14cf5962015-12-11 15:50:14 -0800106 if USE_LLVM_PATCH in self._patches_string:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800107 experiment_file_name = '%s_llvm_experiment.txt' % self._board
108 compiler_string = 'llvm'
Yunlian Jiang2f563562015-08-28 13:54:04 -0700109
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800110 experiment_file = os.path.join(experiment_file_dir, experiment_file_name)
cmtice46093e52014-12-09 14:59:16 -0800111 experiment_header = """
112 board: %s
113 remote: %s
Luis Lozanoe1efeb82015-06-16 16:35:44 -0700114 retries: 1
cmtice46093e52014-12-09 14:59:16 -0800115 """ % (self._board, self._remotes)
116 experiment_tests = """
Luis Lozano1489d642015-12-08 10:08:19 -0800117 benchmark: all_toolchain_perf {
cmtice46093e52014-12-09 14:59:16 -0800118 suite: telemetry_Crosperf
119 iterations: 3
120 }
121 """
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800122
123 with open(experiment_file, 'w') as f:
Yunlian Jiang14cf5962015-12-11 15:50:14 -0800124 f.write(experiment_header)
125 f.write(experiment_tests)
cmtice46093e52014-12-09 14:59:16 -0800126
127 # Now add vanilla to test file.
128 official_image = """
129 vanilla_image {
130 chromeos_root: %s
131 build: %s
Caroline Ticeddde5052015-09-23 09:43:35 -0700132 compiler: gcc
cmtice46093e52014-12-09 14:59:16 -0800133 }
134 """ % (self._chromeos_root, vanilla_image)
Yunlian Jiang14cf5962015-12-11 15:50:14 -0800135 f.write(official_image)
cmtice46093e52014-12-09 14:59:16 -0800136
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800137 label_string = '%s_trybot_image' % compiler_string
Caroline Tice80eab982015-11-04 14:03:14 -0800138 if USE_NEXT_GCC_PATCH in self._patches:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800139 label_string = 'gcc_next_trybot_image'
Caroline Tice80eab982015-11-04 14:03:14 -0800140
cmtice46093e52014-12-09 14:59:16 -0800141 experiment_image = """
Caroline Tice80eab982015-11-04 14:03:14 -0800142 %s {
cmtice46093e52014-12-09 14:59:16 -0800143 chromeos_root: %s
144 build: %s
Caroline Ticeddde5052015-09-23 09:43:35 -0700145 compiler: %s
cmtice46093e52014-12-09 14:59:16 -0800146 }
Caroline Tice80eab982015-11-04 14:03:14 -0800147 """ % (label_string, self._chromeos_root, trybot_image,
148 compiler_string)
Yunlian Jiang14cf5962015-12-11 15:50:14 -0800149 f.write(experiment_image)
cmtice46093e52014-12-09 14:59:16 -0800150
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800151 crosperf = os.path.join(TOOLCHAIN_DIR, 'crosperf', 'crosperf')
Han Shen43494292015-09-14 10:26:40 -0700152 noschedv2_opts = '--noschedv2' if self._noschedv2 else ''
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800153 command = ('{crosperf} --no_email=True --results_dir={r_dir} '
154 '--json_report=True {noschedv2_opts} {exp_file}').format(
Yunlian Jiang14cf5962015-12-11 15:50:14 -0800155 crosperf=crosperf,
156 r_dir=self._reports_dir,
157 noschedv2_opts=noschedv2_opts,
158 exp_file=experiment_file)
cmticeaa700b02015-06-12 13:26:47 -0700159
cmtice46093e52014-12-09 14:59:16 -0800160 ret = self._ce.RunCommand(command)
cmtice7f3190b2015-05-22 14:14:51 -0700161 if ret != 0:
162 raise RuntimeError("Couldn't run crosperf!")
Caroline Ticeebbc3da2015-09-03 10:27:20 -0700163 else:
164 # Copy json report to pending archives directory.
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800165 command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR)
Caroline Ticeebbc3da2015-09-03 10:27:20 -0700166 ret = self._ce.RunCommand(command)
cmtice7f3190b2015-05-22 14:14:51 -0700167 return
cmtice46093e52014-12-09 14:59:16 -0800168
169 def _CopyWeeklyReportFiles(self, trybot_image, vanilla_image):
Yunlian Jiang14cf5962015-12-11 15:50:14 -0800170 """Put files in place for running seven-day reports.
cmtice46093e52014-12-09 14:59:16 -0800171
172 Create tar files of the custom and official images and copy them
173 to the weekly reports directory, so they exist when the weekly report
174 gets generated. IMPORTANT NOTE: This function must run *after*
175 crosperf has been run; otherwise the vanilla images will not be there.
176 """
177
178 dry_run = False
Yunlian Jiang14cf5962015-12-11 15:50:14 -0800179 if os.getlogin() != ROLE_ACCOUNT:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800180 self._l.LogOutput('Running this from non-role account; not copying '
181 'tar files for weekly reports.')
cmtice46093e52014-12-09 14:59:16 -0800182 dry_run = True
183
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800184 images_path = os.path.join(
185 os.path.realpath(self._chromeos_root), 'chroot/tmp')
cmtice46093e52014-12-09 14:59:16 -0800186
187 data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board)
Yunlian Jiang14cf5962015-12-11 15:50:14 -0800188 dest_dir = os.path.join(data_dir, self._weekday)
cmtice46093e52014-12-09 14:59:16 -0800189 if not os.path.exists(dest_dir):
190 os.makedirs(dest_dir)
191
192 # Make sure dest_dir is empty (clean out last week's data).
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800193 cmd = 'cd %s; rm -Rf %s_*_image*' % (dest_dir, self._weekday)
cmtice46093e52014-12-09 14:59:16 -0800194 if dry_run:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800195 print('CMD: %s' % cmd)
cmtice46093e52014-12-09 14:59:16 -0800196 else:
197 self._ce.RunCommand(cmd)
198
199 # Now create new tar files and copy them over.
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800200 labels = ['test', 'vanilla']
cmtice46093e52014-12-09 14:59:16 -0800201 for label_name in labels:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800202 if label_name == 'test':
cmtice46093e52014-12-09 14:59:16 -0800203 test_path = trybot_image
204 else:
205 test_path = vanilla_image
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800206 tar_file_name = '%s_%s_image.tar' % (self._weekday, label_name)
207 cmd = ('cd %s; tar -cvf %s %s/chromiumos_test_image.bin; '
208 'cp %s %s/.') % (images_path, tar_file_name, test_path,
209 tar_file_name, dest_dir)
cmtice46093e52014-12-09 14:59:16 -0800210 if dry_run:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800211 print('CMD: %s' % cmd)
cmtice46093e52014-12-09 14:59:16 -0800212 tar_ret = 0
213 else:
214 tar_ret = self._ce.RunCommand(cmd)
215 if tar_ret:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800216 self._l.LogOutput('Error while creating/copying test tar file(%s).' %
217 tar_file_name)
cmtice46093e52014-12-09 14:59:16 -0800218
cmtice7f3190b2015-05-22 14:14:51 -0700219 def _SendEmail(self):
220 """Find email message generated by crosperf and send it."""
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800221 filename = os.path.join(self._reports_dir, 'msg_body.html')
cmtice7f3190b2015-05-22 14:14:51 -0700222 if (os.path.exists(filename) and
223 os.path.exists(os.path.expanduser(MAIL_PROGRAM))):
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800224 email_title = 'buildbot test results'
Yunlian Jiang2f563562015-08-28 13:54:04 -0700225 if self._patches_string == USE_LLVM_PATCH:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800226 email_title = 'buildbot llvm test results'
227 command = ('cat %s | %s -s "%s, %s" -team -html' %
228 (filename, MAIL_PROGRAM, email_title, self._board))
cmtice7f3190b2015-05-22 14:14:51 -0700229 self._ce.RunCommand(command)
cmtice46093e52014-12-09 14:59:16 -0800230
231 def DoAll(self):
Yunlian Jiang14cf5962015-12-11 15:50:14 -0800232 """Main function inside ToolchainComparator class.
cmtice46093e52014-12-09 14:59:16 -0800233
234 Launch trybot, get image names, create crosperf experiment file, run
235 crosperf, and copy images into seven-day report directories.
236 """
cmticece5ffa42015-02-12 15:18:43 -0800237 date_str = datetime.date.today()
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800238 description = 'master_%s_%s_%s' % (self._patches_string, self._build,
Han Shenfe054f12015-02-18 15:00:13 -0800239 date_str)
cmtice46093e52014-12-09 14:59:16 -0800240 trybot_image = buildbot_utils.GetTrybotImage(self._chromeos_root,
241 self._build,
Yunlian Jiang3c6e4672015-08-24 15:58:22 -0700242 self._patches,
Luis Lozano8a68b2d2015-04-23 14:37:09 -0700243 description,
244 build_toolchain=True)
cmtice46093e52014-12-09 14:59:16 -0800245
246 vanilla_image = self._ParseVanillaImage(trybot_image)
247
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800248 print('trybot_image: %s' % trybot_image)
249 print('vanilla_image: %s' % vanilla_image)
cmticed54f9802015-02-05 11:04:11 -0800250 if len(trybot_image) == 0:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800251 self._l.LogError('Unable to find trybot_image for %s!' % description)
Luis Lozano7f20acb2015-11-04 17:15:08 -0800252 return 1
cmticed54f9802015-02-05 11:04:11 -0800253 if len(vanilla_image) == 0:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800254 self._l.LogError('Unable to find vanilla image for %s!' % description)
Luis Lozano7f20acb2015-11-04 17:15:08 -0800255 return 1
cmtice46093e52014-12-09 14:59:16 -0800256 if os.getlogin() == ROLE_ACCOUNT:
257 self._FinishSetup()
258
cmtice7f3190b2015-05-22 14:14:51 -0700259 self._TestImages(trybot_image, vanilla_image)
260 self._SendEmail()
Yunlian Jiang3c6e4672015-08-24 15:58:22 -0700261 if (self._patches_string == USE_NEXT_GCC_PATCH and
262 self._board in WEEKLY_REPORT_BOARDS):
Luis Lozano7f20acb2015-11-04 17:15:08 -0800263 # Only try to copy the image files if the test runs ran successfully.
264 self._CopyWeeklyReportFiles(trybot_image, vanilla_image)
cmtice46093e52014-12-09 14:59:16 -0800265 return 0
266
267
268def Main(argv):
269 """The main function."""
270
271 # Common initializations
272 command_executer.InitCommandExecuter()
273 parser = optparse.OptionParser()
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800274 parser.add_option('--remote',
275 dest='remote',
276 help='Remote machines to run tests on.')
277 parser.add_option('--board',
278 dest='board',
279 default='x86-zgb',
280 help='The target board.')
281 parser.add_option('--chromeos_root',
282 dest='chromeos_root',
283 help='The chromeos root from which to run tests.')
284 parser.add_option('--weekday',
285 default='',
286 dest='weekday',
287 help='The day of the week for which to run tests.')
288 parser.add_option('--patch',
289 dest='patches',
290 help='The patches to use for the testing, '
Yunlian Jiange52838c2015-08-20 14:32:37 -0700291 "seprate the patch numbers with ',' "
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800292 'for more than one patches.')
293 parser.add_option('--noschedv2',
294 dest='noschedv2',
295 action='store_true',
Han Shen36413122015-08-28 11:05:40 -0700296 default=False,
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800297 help='Pass --noschedv2 to crosperf.')
Han Shen36413122015-08-28 11:05:40 -0700298
cmtice46093e52014-12-09 14:59:16 -0800299 options, _ = parser.parse_args(argv)
300 if not options.board:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800301 print('Please give a board.')
cmtice46093e52014-12-09 14:59:16 -0800302 return 1
303 if not options.remote:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800304 print('Please give at least one remote machine.')
cmtice46093e52014-12-09 14:59:16 -0800305 return 1
306 if not options.chromeos_root:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800307 print('Please specify the ChromeOS root directory.')
cmtice46093e52014-12-09 14:59:16 -0800308 return 1
Yunlian Jiang76259e62015-08-21 08:44:31 -0700309 if options.patches:
Yunlian Jiang3c6e4672015-08-24 15:58:22 -0700310 patches = options.patches
311 else:
312 patches = USE_NEXT_GCC_PATCH
Yunlian Jiange52838c2015-08-20 14:32:37 -0700313
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800314 fc = ToolchainComparator(options.board, options.remote, options.chromeos_root,
315 options.weekday, patches, options.noschedv2)
cmtice46093e52014-12-09 14:59:16 -0800316 return fc.DoAll()
317
318
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800319if __name__ == '__main__':
cmtice46093e52014-12-09 14:59:16 -0800320 retval = Main(sys.argv)
321 sys.exit(retval)