blob: 54d83f286fb1749ec07c8c58e85d3f263b3e8d22 [file] [log] [blame]
Caroline Tice88272d42016-01-13 09:48:29 -08001#!/usr/bin/python2
Han Shen819f8622014-04-08 16:57:38 -07002"""Script to bootstrap the chroot using new toolchain.
3
4This script allows you to build/install a customized version of gcc/binutils,
5either by specifying branch or a local directory.
6
7This script must be executed outside chroot.
8
9Below is some typical usage -
10
Caroline Tice88272d42016-01-13 09:48:29 -080011## Build gcc located at /local/gcc/dir and do a bootstrap using the new
12## compiler for the chromeos root. The script tries to find a valid chromeos
13## tree all the way up from your current working directory.
Han Shen819f8622014-04-08 16:57:38 -070014./build_tool.py --gcc_dir=/loca/gcc/dir --bootstrap
15
Caroline Tice88272d42016-01-13 09:48:29 -080016## Build binutils, using remote branch "mobile_toolchain_v17" and do a
17## bootstrap using the new binutils for the chromeos root. The script tries to
18## find a valid chromeos tree all the way up from your current working
19## directory.
Han Shen819f8622014-04-08 16:57:38 -070020./build_tool.py --binutils_branch=cros/mobile_toolchain_v17 \
21 --chromeos_root=/chromeos/dir --bootstrap
22
23## Same as above except only do it for board daisy - no bootstrapping involved.
24./build_tool.py --binutils_branch=cros/mobile_toolchain_v16 \
25 --chromeos_root=/chromeos/dir --board=daisy
26"""
Han Shen91445322013-03-20 13:43:31 -070027
Caroline Tice88272d42016-01-13 09:48:29 -080028from __future__ import print_function
29
Han Shen91445322013-03-20 13:43:31 -070030__author__ = 'shenhan@google.com (Han Shen)'
31
Caroline Tice88272d42016-01-13 09:48:29 -080032import argparse
Han Shen91445322013-03-20 13:43:31 -070033import os
34import re
Han Shen91445322013-03-20 13:43:31 -070035import sys
36
Han Shen819f8622014-04-08 16:57:38 -070037import repo_to_repo
Caroline Tice88272d42016-01-13 09:48:29 -080038from cros_utils import command_executer
39from cros_utils import logger
40from cros_utils import misc
Han Shen91445322013-03-20 13:43:31 -070041
Han Shen819f8622014-04-08 16:57:38 -070042REPO_PATH_PATTERN = 'src/third_party/{0}'
43TEMP_BRANCH_NAME = 'internal_testing_branch_no_use'
44CHROMIUMOS_OVERLAY_PATH = 'src/third_party/chromiumos-overlay'
45EBUILD_PATH_PATTERN = 'src/third_party/chromiumos-overlay/sys-devel/{0}'
46
Han Shen91445322013-03-20 13:43:31 -070047
48class Bootstrapper(object):
Caroline Tice88272d42016-01-13 09:48:29 -080049 """Class that handles bootstrap process."""
Han Shen819f8622014-04-08 16:57:38 -070050
Luis Lozanof2a3ef42015-12-15 13:49:30 -080051 def __init__(self,
52 chromeos_root,
53 gcc_branch=None,
54 gcc_dir=None,
55 binutils_branch=None,
56 binutils_dir=None,
57 board=None,
58 disable_2nd_bootstrap=False,
Han Shen03d30982014-06-12 11:22:29 -070059 setup_tool_ebuild_file_only=False):
Han Shen91445322013-03-20 13:43:31 -070060 self._chromeos_root = chromeos_root
Han Shen819f8622014-04-08 16:57:38 -070061
62 self._gcc_branch = gcc_branch
63 self._gcc_branch_tree = None
Han Shen91445322013-03-20 13:43:31 -070064 self._gcc_dir = gcc_dir
Han Shen91445322013-03-20 13:43:31 -070065 self._gcc_ebuild_file = None
66 self._gcc_ebuild_file_name = None
Han Shen819f8622014-04-08 16:57:38 -070067
68 self._binutils_branch = binutils_branch
69 self._binutils_branch_tree = None
70 self._binutils_dir = binutils_dir
71 self._binutils_ebuild_file = None
72 self._binutils_ebuild_file_name = None
73
74 self._setup_tool_ebuild_file_only = setup_tool_ebuild_file_only
75
76 self._ce = command_executer.GetCommandExecuter()
77 self._logger = logger.GetLogger()
78 self._board = board
Han Shen03d30982014-06-12 11:22:29 -070079 self._disable_2nd_bootstrap = disable_2nd_bootstrap
Han Shen819f8622014-04-08 16:57:38 -070080
81 def IsTreeSame(self, t1, t2):
82 diff = 'diff -qr -x .git -x .svn "{0}" "{1}"'.format(t1, t2)
83 if self._ce.RunCommand(diff, print_to_console=False) == 0:
84 self._logger.LogOutput('"{0}" and "{1}" are the same."'.format(t1, t2))
85 return True
86 self._logger.LogWarning('"{0}" and "{1}" are different."'.format(t1, t2))
87 return False
Han Shen91445322013-03-20 13:43:31 -070088
89 def SubmitToLocalBranch(self):
Han Shen819f8622014-04-08 16:57:38 -070090 """Copy source code to the chromium source tree and submit it locally."""
91 if self._gcc_dir:
Luis Lozanof2a3ef42015-12-15 13:49:30 -080092 if not self.SubmitToolToLocalBranch(tool_name='gcc',
93 tool_dir=self._gcc_dir):
Han Shen819f8622014-04-08 16:57:38 -070094 return False
95 self._gcc_branch = TEMP_BRANCH_NAME
Han Shen91445322013-03-20 13:43:31 -070096
Han Shen819f8622014-04-08 16:57:38 -070097 if self._binutils_dir:
Luis Lozanof2a3ef42015-12-15 13:49:30 -080098 if not self.SubmitToolToLocalBranch(tool_name='binutils',
99 tool_dir=self._binutils_dir):
Han Shen819f8622014-04-08 16:57:38 -0700100 return False
101 self._binutils_branch = TEMP_BRANCH_NAME
102
103 return True
104
105 def SubmitToolToLocalBranch(self, tool_name, tool_dir):
106 """Copy the source code to local chromium source tree.
107
108 Args:
109 tool_name: either 'gcc' or 'binutils'
110 tool_dir: the tool source dir to be used
Caroline Tice88272d42016-01-13 09:48:29 -0800111
Han Shen819f8622014-04-08 16:57:38 -0700112 Returns:
113 True if all succeeded False otherwise.
114 """
115
116 # The next few steps creates an internal branch to sync with the tool dir
Han Shen91445322013-03-20 13:43:31 -0700117 # user provided.
Han Shen819f8622014-04-08 16:57:38 -0700118 chrome_tool_dir = self.GetChromeOsToolDir(tool_name)
Han Shen91445322013-03-20 13:43:31 -0700119
120 # 0. Test to see if git tree is free of local changes.
Han Shen819f8622014-04-08 16:57:38 -0700121 if not misc.IsGitTreeClean(chrome_tool_dir):
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800122 self._logger.LogError('Git repository "{0}" not clean, aborted.'.format(
123 chrome_tool_dir))
Han Shen91445322013-03-20 13:43:31 -0700124 return False
125
126 # 1. Checkout/create a (new) branch for testing.
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800127 command = 'cd "{0}" && git checkout -B {1}'.format(chrome_tool_dir,
128 TEMP_BRANCH_NAME)
Han Shen91445322013-03-20 13:43:31 -0700129 ret = self._ce.RunCommand(command)
130 if ret:
131 self._logger.LogError('Failed to create a temp branch for test, aborted.')
132 return False
133
Han Shen819f8622014-04-08 16:57:38 -0700134 if self.IsTreeSame(tool_dir, chrome_tool_dir):
135 self._logger.LogOutput(
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800136 '"{0}" and "{1}" are the same, sync skipped.'.format(tool_dir,
137 chrome_tool_dir))
Han Shen819f8622014-04-08 16:57:38 -0700138 return True
Han Shen91445322013-03-20 13:43:31 -0700139
Han Shen819f8622014-04-08 16:57:38 -0700140 # 2. Sync sources from user provided tool dir to chromiumos tool git.
141 local_tool_repo = repo_to_repo.FileRepo(tool_dir)
142 chrome_tool_repo = repo_to_repo.GitRepo(chrome_tool_dir, TEMP_BRANCH_NAME)
Caroline Tice88272d42016-01-13 09:48:29 -0800143 chrome_tool_repo.SetRoot(chrome_tool_dir)
Han Shen03d30982014-06-12 11:22:29 -0700144 # Delete all stuff except '.git' before start mapping.
145 self._ce.RunCommand(
146 'cd {0} && find . -maxdepth 1 -not -name ".git" -not -name "." '
147 r'\( -type f -exec rm {{}} \; -o '
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800148 r' -type d -exec rm -fr {{}} \; \)'.format(chrome_tool_dir))
Han Shen819f8622014-04-08 16:57:38 -0700149 local_tool_repo.MapSources(chrome_tool_repo.GetRoot())
150
151 # 3. Ensure after sync tree is the same.
152 if self.IsTreeSame(tool_dir, chrome_tool_dir):
Han Shen91445322013-03-20 13:43:31 -0700153 self._logger.LogOutput('Sync successfully done.')
Han Shen819f8622014-04-08 16:57:38 -0700154 else:
155 self._logger.LogError('Sync not successful, aborted.')
156 return False
Han Shen91445322013-03-20 13:43:31 -0700157
158 # 4. Commit all changes.
Han Shen03d30982014-06-12 11:22:29 -0700159 # 4.1 Try to get some information about the tool dir we are using.
160 cmd = 'cd {0} && git log -1 --pretty=oneline'.format(tool_dir)
161 tool_dir_extra_info = None
Luis Lozano036c9232015-12-10 10:47:01 -0800162 ret, tool_dir_extra_info, _ = self._ce.RunCommandWOutput(
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800163 cmd,
164 print_to_console=False)
Han Shen03d30982014-06-12 11:22:29 -0700165 commit_message = 'Synced with tool source tree at - "{0}".'.format(tool_dir)
166 if not ret:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800167 commit_message += '\nGit log for {0}:\n{1}'.format(tool_dir,
168 tool_dir_extra_info)
Han Shen03d30982014-06-12 11:22:29 -0700169
170 if chrome_tool_repo.CommitLocally(commit_message):
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800171 self._logger.LogError(
172 'Commit to local branch "{0}" failed, aborted.'.format(
173 TEMP_BRANCH_NAME))
Han Shen91445322013-03-20 13:43:31 -0700174 return False
175 return True
176
177 def CheckoutBranch(self):
Han Shen819f8622014-04-08 16:57:38 -0700178 """Checkout working branch for the tools.
Han Shen91445322013-03-20 13:43:31 -0700179
Han Shen819f8622014-04-08 16:57:38 -0700180 Returns:
181 True: if operation succeeds.
182 """
Han Shen91445322013-03-20 13:43:31 -0700183
Han Shen819f8622014-04-08 16:57:38 -0700184 if self._gcc_branch:
185 rv = self.CheckoutToolBranch('gcc', self._gcc_branch)
186 if rv:
187 self._gcc_branch_tree = rv
188 else:
189 return False
190
191 if self._binutils_branch:
192 rv = self.CheckoutToolBranch('binutils', self._binutils_branch)
193 if rv:
194 self._binutils_branch_tree = rv
195 else:
196 return False
197
Han Shen91445322013-03-20 13:43:31 -0700198 return True
199
Han Shen819f8622014-04-08 16:57:38 -0700200 def CheckoutToolBranch(self, tool_name, tool_branch):
201 """Checkout the tool branch for a certain tool.
202
203 Args:
204 tool_name: either 'gcc' or 'binutils'
205 tool_branch: tool branch to use
Caroline Tice88272d42016-01-13 09:48:29 -0800206
Han Shen819f8622014-04-08 16:57:38 -0700207 Returns:
208 True: if operation succeeds. Otherwise False.
Han Shen91445322013-03-20 13:43:31 -0700209 """
Han Shen819f8622014-04-08 16:57:38 -0700210
211 chrome_tool_dir = self.GetChromeOsToolDir(tool_name)
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800212 command = 'cd "{0}" && git checkout {1}'.format(chrome_tool_dir,
213 tool_branch)
Han Shen819f8622014-04-08 16:57:38 -0700214 if not self._ce.RunCommand(command, print_to_console=True):
215 # Get 'TREE' value of this commit
216 command = ('cd "{0}" && git cat-file -p {1} '
217 '| grep -E "^tree [a-f0-9]+$" '
218 '| cut -d" " -f2').format(chrome_tool_dir, tool_branch)
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800219 ret, stdout, _ = self._ce.RunCommandWOutput(command,
220 print_to_console=False)
Han Shen819f8622014-04-08 16:57:38 -0700221 # Pipe operation always has a zero return value. So need to check if
222 # stdout is valid.
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800223 if not ret and stdout and re.match('[0-9a-h]{40}', stdout.strip(),
224 re.IGNORECASE):
Han Shen819f8622014-04-08 16:57:38 -0700225 tool_branch_tree = stdout.strip()
226 self._logger.LogOutput('Find tree for {0} branch "{1}" - "{2}"'.format(
227 tool_name, tool_branch, tool_branch_tree))
228 return tool_branch_tree
229 self._logger.LogError(('Failed to checkout "{0}" or failed to '
230 'get tree value, aborted.').format(tool_branch))
231 return None
232
233 def FindEbuildFile(self):
234 """Find the ebuild files for the tools.
235
236 Returns:
237 True: if operation succeeds.
238 """
239
240 if self._gcc_branch:
241 (rv, ef, efn) = self.FindToolEbuildFile('gcc')
242 if rv:
243 self._gcc_ebuild_file = ef
244 self._gcc_ebuild_file_name = efn
245 else:
246 return False
247
248 if self._binutils_branch:
249 (rv, ef, efn) = self.FindToolEbuildFile('binutils')
250 if rv:
251 self._binutils_ebuild_file = ef
252 self._binutils_ebuild_file_name = efn
253 else:
254 return False
255
256 return True
257
258 def FindToolEbuildFile(self, tool_name):
259 """Find ebuild file for a specific tool.
260
261 Args:
262 tool_name: either "gcc" or "binutils".
Caroline Tice88272d42016-01-13 09:48:29 -0800263
Han Shen819f8622014-04-08 16:57:38 -0700264 Returns:
265 A triplet that consisits of whether operation succeeds or not,
266 tool ebuild file full path and tool ebuild file name.
267 """
268
269 # To get the active gcc ebuild file, we need a workable chroot first.
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800270 if not os.path.exists(os.path.join(
271 self._chromeos_root, 'chroot')) and self._ce.RunCommand(
Han Shen819f8622014-04-08 16:57:38 -0700272 'cd "{0}" && cros_sdk --create'.format(self._chromeos_root)):
273 self._logger.LogError(('Failed to install a initial chroot, aborted.\n'
274 'If previous bootstrap failed, do a '
275 '"cros_sdk --delete" to remove '
276 'in-complete chroot.'))
277 return (False, None, None)
278
Luis Lozano036c9232015-12-10 10:47:01 -0800279 rv, stdout, _ = self._ce.ChrootRunCommandWOutput(
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800280 self._chromeos_root,
281 'equery w sys-devel/{0}'.format(tool_name),
Luis Lozano036c9232015-12-10 10:47:01 -0800282 print_to_console=True)
Han Shen819f8622014-04-08 16:57:38 -0700283 if rv:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800284 self._logger.LogError(('Failed to execute inside chroot '
285 '"equery w sys-devel/{0}", aborted.').format(
286 tool_name))
Han Shen819f8622014-04-08 16:57:38 -0700287 return (False, None, None)
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800288 m = re.match(r'^.*/({0}/(.*\.ebuild))$'.format(EBUILD_PATH_PATTERN.format(
289 tool_name)), stdout)
Han Shen819f8622014-04-08 16:57:38 -0700290 if not m:
291 self._logger.LogError(
292 ('Failed to find {0} ebuild file, aborted. '
293 'If previous bootstrap failed, do a "cros_sdk --delete" to remove '
294 'in-complete chroot.').format(tool_name))
295 return (False, None, None)
296 tool_ebuild_file = os.path.join(self._chromeos_root, m.group(1))
297 tool_ebuild_file_name = m.group(2)
298
299 return (True, tool_ebuild_file, tool_ebuild_file_name)
300
301 def InplaceModifyEbuildFile(self):
302 """Modify the ebuild file.
303
304 Returns:
305 True if operation succeeds.
306 """
307
308 # Note we shall not use remote branch name (eg. "cros/gcc.gnu.org/...") in
309 # CROS_WORKON_COMMIT, we have to use GITHASH. So we call GitGetCommitHash on
310 # tool_branch.
311 if self._gcc_branch:
312 tool_branch_githash = misc.GitGetCommitHash(
313 self.GetChromeOsToolDir('gcc'), self._gcc_branch)
314 if not tool_branch_githash:
315 return False
316 if not self.InplaceModifyToolEbuildFile(
317 tool_branch_githash, self._gcc_branch_tree, self._gcc_ebuild_file):
318 return False
319
320 if self._binutils_branch:
321 tool_branch_githash = misc.GitGetCommitHash(
322 self.GetChromeOsToolDir('binutils'), self._binutils_branch)
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800323 if not self.InplaceModifyToolEbuildFile(tool_branch_githash,
324 self._binutils_branch_tree,
325 self._binutils_ebuild_file):
Han Shen819f8622014-04-08 16:57:38 -0700326 return False
327 return True
328
329 @staticmethod
330 def ResetToolEbuildFile(chromeos_root, tool_name):
331 """Reset tool ebuild file to clean state.
332
333 Args:
334 chromeos_root: chromeos source tree
335 tool_name: either "gcc" or "binutils"
Caroline Tice88272d42016-01-13 09:48:29 -0800336
Han Shen819f8622014-04-08 16:57:38 -0700337 Returns:
338 True if operation succeds.
339 """
340 rv = misc.GetGitChangesAsList(
341 os.path.join(chromeos_root, CHROMIUMOS_OVERLAY_PATH),
342 path=('sys-devel/{0}/{0}-*.ebuild'.format(tool_name)),
343 staged=False)
344 if rv:
345 cmd = 'cd {0} && git checkout --'.format(os.path.join(
346 chromeos_root, CHROMIUMOS_OVERLAY_PATH))
347 for g in rv:
348 cmd += ' ' + g
349 rv = command_executer.GetCommandExecuter().RunCommand(cmd)
350 if rv:
351 logger.GetLogger().LogWarning(
352 'Failed to reset the ebuild file. Please refer to log above.')
353 return False
354 else:
355 logger.GetLogger().LogWarning(
356 'Note - did not find any modified {0} ebuild file.'.format(tool_name))
357 # Fall through
358 return True
359
360 def GetChromeOsToolDir(self, tool_name):
361 """Return the chromeos git dir for a specific tool.
362
363 Args:
364 tool_name: either 'gcc' or 'binutils'.
Caroline Tice88272d42016-01-13 09:48:29 -0800365
Han Shen819f8622014-04-08 16:57:38 -0700366 Returns:
367 Absolute git path for the tool.
368 """
369
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800370 return os.path.join(self._chromeos_root,
371 REPO_PATH_PATTERN.format(tool_name))
Han Shen819f8622014-04-08 16:57:38 -0700372
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800373 def InplaceModifyToolEbuildFile(self, tool_branch_githash, tool_branch_tree,
374 tool_ebuild_file):
Han Shen819f8622014-04-08 16:57:38 -0700375 """Using sed to fill properly values into the ebuild file.
376
377 Args:
378 tool_branch_githash: githash for tool_branch
379 tool_branch_tree: treeish for the tool branch
380 tool_ebuild_file: tool ebuild file
Caroline Tice88272d42016-01-13 09:48:29 -0800381
Han Shen819f8622014-04-08 16:57:38 -0700382 Returns:
383 True: if operation succeeded.
384 """
385
386 command = ('sed -i '
387 '-e \'/^CROS_WORKON_COMMIT=".*"/i'
388 ' # The following line is modified by script.\' '
389 '-e \'s!^CROS_WORKON_COMMIT=".*"$!CROS_WORKON_COMMIT="{0}"!\' '
390 '-e \'/^CROS_WORKON_TREE=".*"/i'
391 ' # The following line is modified by script.\' '
392 '-e \'s!^CROS_WORKON_TREE=".*"$!CROS_WORKON_TREE="{1}"!\' '
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800393 '{2}').format(tool_branch_githash, tool_branch_tree,
Han Shen819f8622014-04-08 16:57:38 -0700394 tool_ebuild_file)
Han Shen91445322013-03-20 13:43:31 -0700395 rv = self._ce.RunCommand(command)
396 if rv:
397 self._logger.LogError(
Han Shen819f8622014-04-08 16:57:38 -0700398 'Failed to modify commit and tree value for "{0}"", aborted.'.format(
399 tool_ebuild_file))
400 return False
401
402 # Warn that the ebuild file has been modified.
403 self._logger.LogWarning(
404 ('Ebuild file "{0}" is modified, to revert the file - \n'
405 'bootstrap_compiler.py --chromeos_root={1} '
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800406 '--reset_tool_ebuild_file').format(tool_ebuild_file,
407 self._chromeos_root))
Han Shen819f8622014-04-08 16:57:38 -0700408 return True
409
410 def DoBuildForBoard(self):
411 """Build tool for a specific board.
412
413 Returns:
414 True if operation succeeds.
415 """
416
417 if self._gcc_branch:
418 if not self.DoBuildToolForBoard('gcc'):
419 return False
420 if self._binutils_branch:
421 if not self.DoBuildToolForBoard('binutils'):
422 return False
423 return True
424
425 def DoBuildToolForBoard(self, tool_name):
426 """Build a specific tool for a specific board.
427
428 Args:
429 tool_name: either "gcc" or "binutils"
Caroline Tice88272d42016-01-13 09:48:29 -0800430
Han Shen819f8622014-04-08 16:57:38 -0700431 Returns:
432 True if operation succeeds.
433 """
434
Han Shenfe3001c2014-04-28 16:36:28 -0700435 boards_to_build = self._board.split(',')
Han Shen819f8622014-04-08 16:57:38 -0700436
Han Shenfe3001c2014-04-28 16:36:28 -0700437 failed = []
438 for board in boards_to_build:
439 if board == 'host':
440 command = 'sudo emerge sys-devel/{0}'.format(tool_name)
441 else:
442 target = misc.GetCtargetFromBoard(board, self._chromeos_root)
443 if not target:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800444 self._logger.LogError('Unsupported board "{0}", skip.'.format(board))
Han Shenfe3001c2014-04-28 16:36:28 -0700445 failed.append(board)
446 continue
447 command = 'sudo emerge cross-{0}/{1}'.format(target, tool_name)
448
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800449 rv = self._ce.ChrootRunCommand(self._chromeos_root,
450 command,
Luis Lozano036c9232015-12-10 10:47:01 -0800451 print_to_console=True)
Han Shenfe3001c2014-04-28 16:36:28 -0700452 if rv:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800453 self._logger.LogError('Build {0} failed for {1}, aborted.'.format(
454 tool_name, board))
Han Shenfe3001c2014-04-28 16:36:28 -0700455 failed.append(board)
456 else:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800457 self._logger.LogOutput('Successfully built {0} for board {1}.'.format(
458 tool_name, board))
Han Shenfe3001c2014-04-28 16:36:28 -0700459
Han Shen8dfdabd2014-05-08 10:15:09 -0700460 if failed:
Han Shen819f8622014-04-08 16:57:38 -0700461 self._logger.LogError(
Han Shenfe3001c2014-04-28 16:36:28 -0700462 'Failed to build {0} for the following board(s): "{1}"'.format(
463 tool_name, ' '.join(failed)))
Han Shen91445322013-03-20 13:43:31 -0700464 return False
Han Shenfe3001c2014-04-28 16:36:28 -0700465 # All boards build successfully
Han Shen91445322013-03-20 13:43:31 -0700466 return True
467
468 def DoBootstrapping(self):
Han Shen819f8622014-04-08 16:57:38 -0700469 """Do bootstrapping the chroot.
470
Han Shen03d30982014-06-12 11:22:29 -0700471 This step firstly downloads a prestine sdk, then use this sdk to build the
472 new sdk, finally use the new sdk to build every host package.
473
Han Shen819f8622014-04-08 16:57:38 -0700474 Returns:
475 True if operation succeeds.
476 """
477
Han Shen91445322013-03-20 13:43:31 -0700478 logfile = os.path.join(self._chromeos_root, 'bootstrap.log')
Han Shen819f8622014-04-08 16:57:38 -0700479 command = 'cd "{0}" && cros_sdk --delete --bootstrap |& tee "{1}"'.format(
480 self._chromeos_root, logfile)
Luis Lozano036c9232015-12-10 10:47:01 -0800481 rv = self._ce.RunCommand(command, print_to_console=True)
Han Shen91445322013-03-20 13:43:31 -0700482 if rv:
483 self._logger.LogError('Bootstrapping failed, log file - "{0}"\n'.format(
484 logfile))
485 return False
486
487 self._logger.LogOutput('Bootstrap succeeded.')
488 return True
489
Han Shen03d30982014-06-12 11:22:29 -0700490 def BuildAndInstallAmd64Host(self):
491 """Build amd64-host (host) packages.
492
493 Build all host packages in the newly-bootstrapped 'chroot' using *NEW*
494 toolchain.
495
496 So actually we perform 2 builds of all host packages -
497 1. build new toolchain using old toolchain and build all host packages
498 using the newly built toolchain
499 2. build the new toolchain again but using new toolchain built in step 1,
500 and build all host packages using the newly built toolchain
501
502 Returns:
503 True if operation succeeds.
504 """
505
506 cmd = ('cd {0} && cros_sdk -- -- ./setup_board --board=amd64-host '
507 '--accept_licenses=@CHROMEOS --skip_chroot_upgrade --nousepkg '
508 '--reuse_pkgs_from_local_boards').format(self._chromeos_root)
Luis Lozano036c9232015-12-10 10:47:01 -0800509 rv = self._ce.RunCommand(cmd, print_to_console=True)
Han Shen03d30982014-06-12 11:22:29 -0700510 if rv:
511 self._logger.LogError('Build amd64-host failed.')
512 return False
513
514 # Package amd64-host into 'built-sdk.tar.xz'.
515 sdk_package = os.path.join(self._chromeos_root, 'built-sdk.tar.xz')
516 cmd = ('cd {0}/chroot/build/amd64-host && sudo XZ_OPT="-e9" '
517 'tar --exclude="usr/lib/debug/*" --exclude="packages/*" '
518 '--exclude="tmp/*" --exclude="usr/local/build/autotest/*" '
519 '--sparse -I xz -vcf {1} . && sudo chmod a+r {1}').format(
520 self._chromeos_root, sdk_package)
Luis Lozano036c9232015-12-10 10:47:01 -0800521 rv = self._ce.RunCommand(cmd, print_to_console=True)
Han Shen03d30982014-06-12 11:22:29 -0700522 if rv:
523 self._logger.LogError('Failed to create "built-sdk.tar.xz".')
524 return False
525
526 # Install amd64-host into a new chroot.
527 cmd = ('cd {0} && cros_sdk --chroot new-sdk-chroot --download --replace '
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800528 '--nousepkg --url file://{1}').format(self._chromeos_root,
529 sdk_package)
Luis Lozano036c9232015-12-10 10:47:01 -0800530 rv = self._ce.RunCommand(cmd, print_to_console=True)
Han Shen03d30982014-06-12 11:22:29 -0700531 if rv:
532 self._logger.LogError('Failed to install "built-sdk.tar.xz".')
533 return False
534 self._logger.LogOutput(
535 'Successfully installed built-sdk.tar.xz into a new chroot.\nAll done.')
536
537 # Rename the newly created new-sdk-chroot to chroot.
538 cmd = ('cd {0} && sudo mv chroot chroot-old && '
539 'sudo mv new-sdk-chroot chroot').format(self._chromeos_root)
Luis Lozano036c9232015-12-10 10:47:01 -0800540 rv = self._ce.RunCommand(cmd, print_to_console=True)
Han Shen03d30982014-06-12 11:22:29 -0700541 return rv == 0
542
Han Shen91445322013-03-20 13:43:31 -0700543 def Do(self):
Han Shen819f8622014-04-08 16:57:38 -0700544 """Entrance of the class.
545
546 Returns:
547 True if everything is ok.
548 """
549
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800550 if (self.SubmitToLocalBranch() and self.CheckoutBranch() and
551 self.FindEbuildFile() and self.InplaceModifyEbuildFile()):
Han Shen819f8622014-04-08 16:57:38 -0700552 if self._setup_tool_ebuild_file_only:
553 # Everything is done, we are good.
554 ret = True
555 else:
556 if self._board:
557 ret = self.DoBuildForBoard()
558 else:
559 # This implies '--bootstrap'.
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800560 ret = (self.DoBootstrapping() and (self._disable_2nd_bootstrap or
561 self.BuildAndInstallAmd64Host()))
Han Shen91445322013-03-20 13:43:31 -0700562 else:
563 ret = False
Han Shen91445322013-03-20 13:43:31 -0700564 return ret
565
566
567def Main(argv):
Caroline Tice88272d42016-01-13 09:48:29 -0800568 parser = argparse.ArgumentParser()
569 parser.add_argument('-c',
570 '--chromeos_root',
571 dest='chromeos_root',
572 help=('Optional. ChromeOs root dir. '
573 'When not specified, chromeos root will be deduced'
574 ' from current working directory.'))
575 parser.add_argument('--gcc_branch',
576 dest='gcc_branch',
577 help=('The branch to test against. '
578 'This branch must be a local branch '
579 'inside "src/third_party/gcc". '
580 'Notice, this must not be used with "--gcc_dir".'))
581 parser.add_argument('--binutils_branch',
582 dest='binutils_branch',
583 help=('The branch to test against binutils. '
584 'This branch must be a local branch '
585 'inside "src/third_party/binutils". '
586 'Notice, this must not be used with '
587 '"--binutils_dir".'))
588 parser.add_argument('-g',
589 '--gcc_dir',
590 dest='gcc_dir',
591 help=('Use a local gcc tree to do bootstrapping. '
592 'Notice, this must not be used with '
593 '"--gcc_branch".'))
594 parser.add_argument('--binutils_dir',
595 dest='binutils_dir',
596 help=('Use a local binutils tree to do bootstrapping. '
597 'Notice, this must not be used with '
598 '"--binutils_branch".'))
599 parser.add_argument('--fixperm',
600 dest='fixperm',
601 default=False,
602 action='store_true',
603 help=('Fix the (notorious) permission error '
604 'while trying to bootstrap the chroot. '
605 'Note this takes an extra 10-15 minutes '
606 'and is only needed once per chromiumos tree.'))
607 parser.add_argument('--setup_tool_ebuild_file_only',
608 dest='setup_tool_ebuild_file_only',
609 default=False,
610 action='store_true',
611 help=('Setup gcc and/or binutils ebuild file '
612 'to pick up the branch (--gcc/binutils_branch) or '
613 'use gcc and/or binutils source '
614 '(--gcc/binutils_dir) and exit. Keep chroot as is.'
615 ' This should not be used with '
616 '--gcc/binutils_dir/branch options.'))
617 parser.add_argument('--reset_tool_ebuild_file',
618 dest='reset_tool_ebuild_file',
619 default=False,
620 action='store_true',
621 help=('Reset the modification that is done by this '
622 'script. Note, when this script is running, it '
623 'will modify the active gcc/binutils ebuild file. '
624 'Use this option to reset (what this script has '
625 'done) and exit. This should not be used with -- '
626 'gcc/binutils_dir/branch options.'))
627 parser.add_argument('--board',
628 dest='board',
629 default=None,
630 help=('Only build toolchain for specific board(s). '
631 'Use "host" to build for host. '
632 'Use "," to seperate multiple boards. '
633 'This does not perform a chroot bootstrap.'))
634 parser.add_argument('--bootstrap',
635 dest='bootstrap',
636 default=False,
637 action='store_true',
638 help=('Performs a chroot bootstrap. '
639 'Note, this will *destroy* your current chroot.'))
640 parser.add_argument('--disable-2nd-bootstrap',
641 dest='disable_2nd_bootstrap',
642 default=False,
643 action='store_true',
644 help=('Disable a second bootstrap '
645 '(build of amd64-host stage).'))
Han Shen819f8622014-04-08 16:57:38 -0700646
Caroline Tice88272d42016-01-13 09:48:29 -0800647 options = parser.parse_args(argv)
Han Shen819f8622014-04-08 16:57:38 -0700648 # Trying to deduce chromeos root from current directory.
Han Shen91445322013-03-20 13:43:31 -0700649 if not options.chromeos_root:
Han Shen819f8622014-04-08 16:57:38 -0700650 logger.GetLogger().LogOutput('Trying to deduce chromeos root ...')
651 wdir = os.getcwd()
652 while wdir and wdir != '/':
653 if misc.IsChromeOsTree(wdir):
654 logger.GetLogger().LogOutput('Find chromeos_root: {}'.format(wdir))
655 options.chromeos_root = wdir
656 break
657 wdir = os.path.dirname(wdir)
658
659 if not options.chromeos_root:
660 parser.error('Missing or failing to deduce mandatory option "--chromeos".')
Han Shen91445322013-03-20 13:43:31 -0700661 return 1
662
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800663 options.chromeos_root = os.path.abspath(os.path.expanduser(
664 options.chromeos_root))
Han Shen91445322013-03-20 13:43:31 -0700665
666 if not os.path.isdir(options.chromeos_root):
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800667 logger.GetLogger().LogError('"{0}" does not exist.'.format(
668 options.chromeos_root))
Han Shen91445322013-03-20 13:43:31 -0700669 return 1
670
671 if options.fixperm:
672 # Fix perm error before continuing.
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800673 cmd = (
674 r'sudo find "{0}" \( -name ".cache" -type d -prune \) -o '
675 r'\( -name "chroot" -type d -prune \) -o '
676 r'\( -type f -exec chmod a+r {{}} \; \) -o '
677 r'\( -type d -exec chmod a+rx {{}} \; \)').format(options.chromeos_root)
Han Shen91445322013-03-20 13:43:31 -0700678 logger.GetLogger().LogOutput(
Han Shen819f8622014-04-08 16:57:38 -0700679 'Fixing perm issues for chromeos root, this might take some time.')
Han Shen91445322013-03-20 13:43:31 -0700680 command_executer.GetCommandExecuter().RunCommand(cmd)
681
Han Shen819f8622014-04-08 16:57:38 -0700682 if options.reset_tool_ebuild_file:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800683 if (options.gcc_dir or options.gcc_branch or options.binutils_dir or
684 options.binutils_branch):
Han Shen91445322013-03-20 13:43:31 -0700685 logger.GetLogger().LogWarning(
Han Shen819f8622014-04-08 16:57:38 -0700686 'Ignoring any "--gcc/binutils_dir" and/or "--gcc/binutils_branch".')
687 if options.setup_tool_ebuild_file_only:
688 logger.GetLogger().LogError(
689 ('Conflict options "--reset_tool_ebuild_file" '
690 'and "--setup_tool_ebuild_file_only".'))
Han Shen91445322013-03-20 13:43:31 -0700691 return 1
Han Shen819f8622014-04-08 16:57:38 -0700692 rv = Bootstrapper.ResetToolEbuildFile(options.chromeos_root, 'gcc')
693 rv1 = Bootstrapper.ResetToolEbuildFile(options.chromeos_root, 'binutils')
694 return 0 if (rv and rv1) else 1
Han Shen91445322013-03-20 13:43:31 -0700695
696 if options.gcc_dir:
697 options.gcc_dir = os.path.abspath(os.path.expanduser(options.gcc_dir))
698 if not os.path.isdir(options.gcc_dir):
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800699 logger.GetLogger().LogError('"{0}" does not exist.'.format(
700 options.gcc_dir))
Han Shen91445322013-03-20 13:43:31 -0700701 return 1
702
Han Shen819f8622014-04-08 16:57:38 -0700703 if options.gcc_branch and options.gcc_dir:
704 parser.error('Only one of "--gcc_dir" and "--gcc_branch" can be specified.')
Han Shen91445322013-03-20 13:43:31 -0700705 return 1
Han Shen819f8622014-04-08 16:57:38 -0700706
707 if options.binutils_dir:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800708 options.binutils_dir = os.path.abspath(os.path.expanduser(
709 options.binutils_dir))
Han Shen819f8622014-04-08 16:57:38 -0700710 if not os.path.isdir(options.binutils_dir):
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800711 logger.GetLogger().LogError('"{0}" does not exist.'.format(
712 options.binutils_dir))
Han Shen819f8622014-04-08 16:57:38 -0700713 return 1
714
715 if options.binutils_branch and options.binutils_dir:
716 parser.error('Only one of "--binutils_dir" and '
717 '"--binutils_branch" can be specified.')
718 return 1
719
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800720 if (not (options.binutils_branch or options.binutils_dir or options.gcc_branch
721 or options.gcc_dir)):
Han Shen819f8622014-04-08 16:57:38 -0700722 parser.error(('At least one of "--gcc_dir", "--gcc_branch", '
723 '"--binutils_dir" and "--binutils_branch" must '
724 'be specified.'))
725 return 1
726
727 if not options.board and not options.bootstrap:
728 parser.error('You must specify either "--board" or "--bootstrap".')
729 return 1
730
Han Shen03d30982014-06-12 11:22:29 -0700731 if (options.board and options.bootstrap and
732 not options.setup_tool_ebuild_file_only):
Han Shen819f8622014-04-08 16:57:38 -0700733 parser.error('You must specify only one of "--board" and "--bootstrap".')
Han Shen91445322013-03-20 13:43:31 -0700734 return 1
735
Han Shen03d30982014-06-12 11:22:29 -0700736 if not options.bootstrap and options.disable_2nd_bootstrap:
737 parser.error('"--disable-2nd-bootstrap" has no effect '
738 'without specifying "--bootstrap".')
739 return 1
740
Han Shen91445322013-03-20 13:43:31 -0700741 if Bootstrapper(
Han Shen819f8622014-04-08 16:57:38 -0700742 options.chromeos_root,
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800743 gcc_branch=options.gcc_branch,
744 gcc_dir=options.gcc_dir,
Han Shen819f8622014-04-08 16:57:38 -0700745 binutils_branch=options.binutils_branch,
746 binutils_dir=options.binutils_dir,
747 board=options.board,
Han Shen03d30982014-06-12 11:22:29 -0700748 disable_2nd_bootstrap=options.disable_2nd_bootstrap,
Han Shen819f8622014-04-08 16:57:38 -0700749 setup_tool_ebuild_file_only=options.setup_tool_ebuild_file_only).Do():
Han Shen91445322013-03-20 13:43:31 -0700750 return 0
751 return 1
752
753
754if __name__ == '__main__':
Caroline Tice88272d42016-01-13 09:48:29 -0800755 retval = Main(sys.argv[1:])
Han Shen91445322013-03-20 13:43:31 -0700756 sys.exit(retval)