blob: 0135c0dbcd0d471bb0781c27b7857a7b2bb8b261 [file] [log] [blame]
asharif96f59692013-02-16 03:13:36 +00001#!/usr/bin/python
2
3# Script to test different toolchains against ChromeOS benchmarks.
4import optparse
5import os
6import sys
7import build_chromeos
8import setup_chromeos
cmtice94bc4702014-05-29 16:29:04 -07009import time
asharif96f59692013-02-16 03:13:36 +000010from utils import command_executer
11from utils import misc
12from utils import logger
13
14
cmtice94bc4702014-05-29 16:29:04 -070015WEEKLY_REPORTS_ROOT="/usr/local/google/crostc/weekly_test_data"
16
asharif96f59692013-02-16 03:13:36 +000017class GCCConfig(object):
18 def __init__(self, githash):
19 self.githash = githash
20
21
22class ToolchainConfig:
23 def __init__(self, gcc_config=None, binutils_config=None):
24 self.gcc_config = gcc_config
25
26
27class ChromeOSCheckout(object):
28 def __init__(self, board, chromeos_root):
29 self._board = board
30 self._chromeos_root = chromeos_root
31 self._ce = command_executer.GetCommandExecuter()
32 self._l = logger.GetLogger()
33
asharif3e38de02013-02-19 19:34:59 +000034 def _DeleteChroot(self):
asharif87414272013-02-19 19:58:45 +000035 command = "cd %s; cros_sdk --delete" % self._chromeos_root
asharif3e38de02013-02-19 19:34:59 +000036 return self._ce.RunCommand(command)
37
asharif67973582013-02-19 20:19:40 +000038 def _DeleteCcahe(self):
39 # crosbug.com/34956
40 command = "sudo rm -rf %s" % os.path.join(self._chromeos_root, ".cache")
41 return self._ce.RunCommand(command)
42
asharif96f59692013-02-16 03:13:36 +000043 def _BuildAndImage(self, label=""):
44 if (not label or
45 not misc.DoesLabelExist(self._chromeos_root, self._board, label)):
46 build_chromeos_args = [build_chromeos.__file__,
47 "--chromeos_root=%s" % self._chromeos_root,
48 "--board=%s" % self._board,
49 "--rebuild"]
asharife6b72fe2013-02-19 19:58:18 +000050 if self._public:
51 build_chromeos_args.append("--env=USE=-chrome_internal")
Yunlian Jiangd145a582013-08-19 13:59:34 -070052 if label == "vanilla":
53 build_chromeos_args.append("--vanilla_image")
asharif96f59692013-02-16 03:13:36 +000054 ret = build_chromeos.Main(build_chromeos_args)
55 if ret:
56 raise Exception("Couldn't build ChromeOS!")
57 if label:
58 misc.LabelLatestImage(self._chromeos_root, self._board, label)
59 return label
60
61 def _SetupBoard(self, env_dict):
62 env_string = misc.GetEnvStringFromDict(env_dict)
63 command = ("%s %s" %
64 (env_string,
65 misc.GetSetupBoardCommand(self._board,
66 usepkg=False)))
67 ret = self._ce.ChrootRunCommand(self._chromeos_root,
68 command)
69 assert ret == 0, "Could not setup board with new toolchain."
70
71 def _UnInstallToolchain(self):
72 command = ("sudo CLEAN_DELAY=0 emerge -C cross-%s/gcc" %
73 misc.GetCtargetFromBoard(self._board,
74 self._chromeos_root))
75 ret = self._ce.ChrootRunCommand(self._chromeos_root,
76 command)
77 if ret:
78 raise Exception("Couldn't uninstall the toolchain!")
79
80 def _CheckoutChromeOS(self):
81 # TODO(asharif): Setup a fixed ChromeOS version (quarterly snapshot).
82 if not os.path.exists(self._chromeos_root):
83 setup_chromeos_args = [setup_chromeos.__file__,
Caroline Ticec35909e2013-10-02 17:13:13 -070084 "--dir=%s" % self._chromeos_root]
asharife6b72fe2013-02-19 19:58:18 +000085 if self._public:
86 setup_chromeos_args.append("--public")
asharif5fe40e22013-02-19 19:58:50 +000087 ret = setup_chromeos.Main(setup_chromeos_args)
88 if ret:
89 raise Exception("Couldn't run setup_chromeos!")
asharif96f59692013-02-16 03:13:36 +000090
Caroline Ticec35909e2013-10-02 17:13:13 -070091
asharif96f59692013-02-16 03:13:36 +000092 def _BuildToolchain(self, config):
93 self._UnInstallToolchain()
94 self._SetupBoard({"USE": "git_gcc",
95 "GCC_GITHASH": config.gcc_config.githash,
96 "EMERGE_DEFAULT_OPTS": "--exclude=gcc"})
97
98
99class ToolchainComparator(ChromeOSCheckout):
asharif58a8c9f2013-02-19 20:42:43 +0000100 def __init__(self, board, remotes, configs, clean, public, force_mismatch):
asharif96f59692013-02-16 03:13:36 +0000101 self._board = board
102 self._remotes = remotes
103 self._chromeos_root = "chromeos"
104 self._configs = configs
asharif3e38de02013-02-19 19:34:59 +0000105 self._clean = clean
asharife6b72fe2013-02-19 19:58:18 +0000106 self._public = public
asharif58a8c9f2013-02-19 20:42:43 +0000107 self._force_mismatch = force_mismatch
asharif96f59692013-02-16 03:13:36 +0000108 self._ce = command_executer.GetCommandExecuter()
109 self._l = logger.GetLogger()
110 ChromeOSCheckout.__init__(self, board, self._chromeos_root)
111
cmticea6255d02014-01-10 10:27:22 -0800112
113 def _FinishSetup(self):
114 # Get correct .boto file
115 current_dir = os.getcwd()
116 src = "/home/mobiletc-prebuild/.boto"
117 dest = os.path.join(current_dir, self._chromeos_root,
118 "src/private-overlays/chromeos-overlay/"
119 "googlestorage_account.boto")
120 # Copy the file to the correct place
121 copy_cmd = "cp %s %s" % (src, dest)
122 retval = self._ce.RunCommand(copy_cmd)
123 if retval:
124 raise Exception("Couldn't copy .boto file for google storage.")
125
126 # Fix protections on ssh key
127 command = ("chmod 600 /var/cache/chromeos-cache/distfiles/target"
128 "/chrome-src-internal/src/third_party/chromite/ssh_keys"
129 "/testing_rsa")
130 retval = self._ce.ChrootRunCommand(self._chromeos_root, command)
131 if retval:
132 raise Exception("chmod for testing_rsa failed")
133
asharif96f59692013-02-16 03:13:36 +0000134 def _TestLabels(self, labels):
135 experiment_file = "toolchain_experiment.txt"
asharif58a8c9f2013-02-19 20:42:43 +0000136 image_args = ""
137 if self._force_mismatch:
asharif0b5d5c82013-02-19 20:42:56 +0000138 image_args = "--force-mismatch"
asharif96f59692013-02-16 03:13:36 +0000139 experiment_header = """
140 board: %s
141 remote: %s
142 """ % (self._board, self._remotes)
143 experiment_tests = """
cmtice04403882013-11-04 16:38:37 -0500144 benchmark: all_perfv2 {
145 suite: telemetry_Crosperf
cmtice6de7f8f2014-03-14 14:08:21 -0700146 iterations: 3
asharif96f59692013-02-16 03:13:36 +0000147 }
148 """
149 with open(experiment_file, "w") as f:
150 print >>f, experiment_header
151 print >>f, experiment_tests
152 for label in labels:
153 # TODO(asharif): Fix crosperf so it accepts labels with symbols
154 crosperf_label = label
155 crosperf_label = crosperf_label.replace("-", "minus")
156 crosperf_label = crosperf_label.replace("+", "plus")
asharife4a5a8f2013-02-19 20:19:33 +0000157 crosperf_label = crosperf_label.replace(".", "")
asharif96f59692013-02-16 03:13:36 +0000158 experiment_image = """
159 %s {
160 chromeos_image: %s
asharif58a8c9f2013-02-19 20:42:43 +0000161 image_args: %s
asharif96f59692013-02-16 03:13:36 +0000162 }
163 """ % (crosperf_label,
164 os.path.join(misc.GetImageDir(self._chromeos_root, self._board),
165 label,
asharif58a8c9f2013-02-19 20:42:43 +0000166 "chromiumos_test_image.bin"),
167 image_args)
asharif96f59692013-02-16 03:13:36 +0000168 print >>f, experiment_image
cmtice94bc4702014-05-29 16:29:04 -0700169 images_path = os.path.join(os.path.realpath(self._chromeos_root),
170 "src/build/images", self._board)
171 weekday = time.strftime("%a")
172 for l in labels:
173 test_path = os.path.join(images_path, l)
174 if os.path.exists(test_path):
175 data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board)
176 if l == "vanilla":
177 label_name = l
178 else:
179 label_name = "test"
180 tar_file_name = "%s_%s_image.tar" % (weekday, label_name)
181 dest_dir = os.path.join (data_dir, weekday)
182 if not os.path.exists(dest_dir):
183 os.makedirs(dest_dir)
184 cmd="cd %s; tar -cvf %s %s/*; cp %s %s/." % (images_path,tar_file_name,
185 l, tar_file_name,
186 dest_dir)
187 tar_ret = self._ce.RunCommand(cmd)
188 if tar_ret:
189 self._l.LogOutput("Error while creating/copying test tar file (%s)."
190 % tar_file_name)
asharif96f59692013-02-16 03:13:36 +0000191 crosperf = os.path.join(os.path.dirname(__file__),
192 "crosperf",
193 "crosperf")
asharifb2cd6342013-02-19 19:42:57 +0000194 command = "%s --email=c-compiler-chrome %s" % (crosperf, experiment_file)
asharif96f59692013-02-16 03:13:36 +0000195 ret = self._ce.RunCommand(command)
196 if ret:
197 raise Exception("Couldn't run crosperf!")
198
199 def DoAll(self):
200 self._CheckoutChromeOS()
asharif96f59692013-02-16 03:13:36 +0000201 labels = []
202 vanilla_label = self._BuildAndImage("vanilla")
203 labels.append(vanilla_label)
204 for config in self._configs:
205 label = misc.GetFilenameFromString(config.gcc_config.githash)
206 if (not misc.DoesLabelExist(self._chromeos_root,
207 self._board,
208 label)):
209 self._BuildToolchain(config)
210 label = self._BuildAndImage(label)
211 labels.append(label)
cmticea6255d02014-01-10 10:27:22 -0800212 self._FinishSetup()
asharif96f59692013-02-16 03:13:36 +0000213 self._TestLabels(labels)
asharif3e38de02013-02-19 19:34:59 +0000214 if self._clean:
215 ret = self._DeleteChroot()
216 if ret: return ret
asharif67973582013-02-19 20:19:40 +0000217 ret = self._DeleteCcahe()
218 if ret: return ret
asharif96f59692013-02-16 03:13:36 +0000219 return 0
220
221
222def Main(argv):
223 """The main function."""
224 # Common initializations
225### command_executer.InitCommandExecuter(True)
226 command_executer.InitCommandExecuter()
227 parser = optparse.OptionParser()
228 parser.add_option("--remote",
229 dest="remote",
230 help="Remote machines to run tests on.")
231 parser.add_option("--board",
232 dest="board",
233 default="x86-zgb",
234 help="The target board.")
235 parser.add_option("--githashes",
236 dest="githashes",
237 default="master",
238 help="The gcc githashes to test.")
asharif3e38de02013-02-19 19:34:59 +0000239 parser.add_option("--clean",
240 dest="clean",
241 default=False,
242 action="store_true",
243 help="Clean the chroot after testing.")
asharife6b72fe2013-02-19 19:58:18 +0000244 parser.add_option("--public",
245 dest="public",
246 default=False,
247 action="store_true",
248 help="Use the public checkout/build.")
asharif58a8c9f2013-02-19 20:42:43 +0000249 parser.add_option("--force-mismatch",
250 dest="force_mismatch",
251 default="",
252 help="Force the image regardless of board mismatch")
asharif96f59692013-02-16 03:13:36 +0000253 options, _ = parser.parse_args(argv)
254 if not options.board:
255 print "Please give a board."
256 return 1
257 if not options.remote:
258 print "Please give at least one remote machine."
259 return 1
260 toolchain_configs = []
261 for githash in options.githashes.split(","):
262 gcc_config = GCCConfig(githash=githash)
263 toolchain_config = ToolchainConfig(gcc_config=gcc_config)
264 toolchain_configs.append(toolchain_config)
asharif3e38de02013-02-19 19:34:59 +0000265 fc = ToolchainComparator(options.board, options.remote, toolchain_configs,
asharif58a8c9f2013-02-19 20:42:43 +0000266 options.clean, options.public,
267 options.force_mismatch)
asharif96f59692013-02-16 03:13:36 +0000268 return fc.DoAll()
269
270
271if __name__ == "__main__":
272 retval = Main(sys.argv)
273 sys.exit(retval)