blob: c83f9d837ce4b29d1585cd9f3454fad67c2d0d64 [file] [log] [blame]
cmtice46093e52014-12-09 14:59:16 -08001#!/usr/bin/python
2"""
3Script for running nightly compiler tests on ChromeOS.
4
5This script launches a buildbot to build ChromeOS with the latest compiler on
6a particular board; then it finds and downloads the trybot image and the
7corresponding official image, and runs crosperf performance tests comparing
8the two. It then generates a report, emails it to the c-compiler-chrome, as
9well as copying the images into the seven-day reports directory.
10"""
11
12# Script to test different toolchains against ChromeOS benchmarks.
13import optparse
14import os
15import sys
16import time
17import urllib2
18
19from utils import command_executer
20from utils import logger
21
22from utils import buildbot_utils
23
24# CL that updated GCC ebuilds to use 'next_gcc'.
25USE_NEXT_GCC_PATCH ="230260"
26
27WEEKLY_REPORTS_ROOT = "/usr/local/google/crostc/weekly_test_data"
28ROLE_ACCOUNT = "mobiletc-prebuild"
29TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
30
31
32class ToolchainComparator():
33 """
34 Class for doing the nightly tests work.
35 """
36
37 def __init__(self, board, remotes, chromeos_root, weekday):
38 self._board = board
39 self._remotes = remotes
40 self._chromeos_root = chromeos_root
41 self._base_dir = os.getcwd()
42 self._ce = command_executer.GetCommandExecuter()
43 self._l = logger.GetLogger()
44 self._build = "%s-release" % board
45 if not weekday:
46 self._weekday = time.strftime("%a")
47 else:
48 self._weekday = weekday
49
50 def _ParseVanillaImage(self, trybot_image):
51 """
52 Parse a trybot artifact name to get corresponding vanilla image.
53
54 This function takes an artifact name, such as
55 'trybot-daisy-release/R40-6394.0.0-b1389', and returns the
56 corresponding official build name, e.g. 'daisy-release/R40-6394.0.0'.
57 """
58 start_pos = trybot_image.find(self._build)
59 end_pos = trybot_image.rfind("-b")
60 vanilla_image = trybot_image[start_pos:end_pos]
61 return vanilla_image
62
63
64 def _FinishSetup(self):
65 """
66 Make sure testing_rsa file is properly set up.
67 """
68 # Fix protections on ssh key
69 command = ("chmod 600 /var/cache/chromeos-cache/distfiles/target"
70 "/chrome-src-internal/src/third_party/chromite/ssh_keys"
71 "/testing_rsa")
72 ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command)
73 if ret_val:
74 raise Exception("chmod for testing_rsa failed")
75
76 def _TestImages(self, trybot_image, vanilla_image):
77 """
78 Create crosperf experiment file.
79
80 Given the names of the trybot and vanilla images, create the
81 appropriate crosperf experiment file and launch crosperf on it.
82 """
83 experiment_file_dir = os.path.join (self._chromeos_root, "..",
84 self._weekday)
85 experiment_file_name = "%s_toolchain_experiment.txt" % self._board
86 experiment_file = os.path.join (experiment_file_dir,
87 experiment_file_name)
88 experiment_header = """
89 board: %s
90 remote: %s
91 """ % (self._board, self._remotes)
92 experiment_tests = """
93 benchmark: all_perfv2 {
94 suite: telemetry_Crosperf
95 iterations: 3
96 }
97 """
98 with open(experiment_file, "w") as f:
99 print >> f, experiment_header
100 print >> f, experiment_tests
101
102 # Now add vanilla to test file.
103 official_image = """
104 vanilla_image {
105 chromeos_root: %s
106 build: %s
107 }
108 """ % (self._chromeos_root, vanilla_image)
109 print >> f, official_image
110
111 experiment_image = """
112 test_image {
113 chromeos_root: %s
114 build: %s
115 }
116 """ % (self._chromeos_root, trybot_image)
117 print >> f, experiment_image
118
119 crosperf = os.path.join(TOOLCHAIN_DIR,
120 "crosperf",
121 "crosperf")
122 command = "%s --email=c-compiler-chrome %s" % (crosperf, experiment_file)
123 ret = self._ce.RunCommand(command)
124 if ret:
125 raise Exception("Couldn't run crosperf!")
126 return ret
127
128
129 def _CopyWeeklyReportFiles(self, trybot_image, vanilla_image):
130 """
131 Put files in place for running seven-day reports.
132
133 Create tar files of the custom and official images and copy them
134 to the weekly reports directory, so they exist when the weekly report
135 gets generated. IMPORTANT NOTE: This function must run *after*
136 crosperf has been run; otherwise the vanilla images will not be there.
137 """
138
139 dry_run = False
140 if (os.getlogin() != ROLE_ACCOUNT):
141 self._l.LogOutput("Running this from non-role account; not copying "
142 "tar files for weekly reports.")
143 dry_run = True
144
145 images_path = os.path.join(os.path.realpath(self._chromeos_root),
146 "chroot/tmp")
147
148 data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board)
149 dest_dir = os.path.join (data_dir, self._weekday)
150 if not os.path.exists(dest_dir):
151 os.makedirs(dest_dir)
152
153 # Make sure dest_dir is empty (clean out last week's data).
154 cmd = "cd %s; rm -Rf %s_*_image*" % (dest_dir, self._weekday)
155 if dry_run:
156 print "CMD: %s" % cmd
157 else:
158 self._ce.RunCommand(cmd)
159
160 # Now create new tar files and copy them over.
161 labels = [ "test", "vanilla" ]
162 for label_name in labels:
163 if label_name == "test":
164 test_path = trybot_image
165 else:
166 test_path = vanilla_image
167 tar_file_name = "%s_%s_image.tar" % (self._weekday, label_name)
168 cmd = "cd %s; tar -cvf %s %s/*; cp %s %s/." % (images_path,
169 tar_file_name,
170 test_path,
171 tar_file_name,
172 dest_dir)
173 if dry_run:
174 print "CMD: %s" % cmd
175 tar_ret = 0
176 else:
177 tar_ret = self._ce.RunCommand(cmd)
178 if tar_ret:
179 self._l.LogOutput("Error while creating/copying test tar file(%s)."
180 % tar_file_name)
181
182
183 def DoAll(self):
184 """
185 Main function inside ToolchainComparator class.
186
187 Launch trybot, get image names, create crosperf experiment file, run
188 crosperf, and copy images into seven-day report directories.
189 """
190
191 description = "master_%s_%s" % (USE_NEXT_GCC_PATCH, self._build)
192 trybot_image = buildbot_utils.GetTrybotImage(self._chromeos_root,
193 self._build,
194 [ USE_NEXT_GCC_PATCH ],
195 description)
196
197 vanilla_image = self._ParseVanillaImage(trybot_image)
198
199 print ("trybot_image: %s" % trybot_image)
200 print ("vanilla_image: %s" % vanilla_image)
cmticed54f9802015-02-05 11:04:11 -0800201 if len(trybot_image) == 0:
202 self._l.LogError("Unable to find trybot_image for %s!" % description)
203 return 1
204 if len(vanilla_image) == 0:
205 self._l.LogError("Unable to find vanilla image for %s!" % description)
206 return 1
cmtice46093e52014-12-09 14:59:16 -0800207 if os.getlogin() == ROLE_ACCOUNT:
208 self._FinishSetup()
209
210 if not self._TestImages(trybot_image, vanilla_image):
211 # Only try to copy the image files if the test runs ran successfully.
212 self._CopyWeeklyReportFiles(trybot_image, vanilla_image)
213 return 0
214
215
216def Main(argv):
217 """The main function."""
218
219 # Common initializations
220 command_executer.InitCommandExecuter()
221 parser = optparse.OptionParser()
222 parser.add_option("--remote",
223 dest="remote",
224 help="Remote machines to run tests on.")
225 parser.add_option("--board",
226 dest="board",
227 default="x86-zgb",
228 help="The target board.")
229 parser.add_option("--chromeos_root",
230 dest="chromeos_root",
231 help="The chromeos root from which to run tests.")
232 parser.add_option("--weekday", default="",
233 dest="weekday",
234 help="The day of the week for which to run tests.")
235 options, _ = parser.parse_args(argv)
236 if not options.board:
237 print "Please give a board."
238 return 1
239 if not options.remote:
240 print "Please give at least one remote machine."
241 return 1
242 if not options.chromeos_root:
243 print "Please specify the ChromeOS root directory."
244 return 1
245
246 fc = ToolchainComparator(options.board, options.remote,
247 options.chromeos_root, options.weekday)
248 return fc.DoAll()
249
250
251if __name__ == "__main__":
252 retval = Main(sys.argv)
253 sys.exit(retval)