blob: 0c2fff6563ff423162fbab60f9c054dab6572b8f [file] [log] [blame]
Caroline Ticef6ef4392017-04-06 17:16:05 -07001#!/usr/bin/env python2
Han Shen819f8622014-04-08 16:57:38 -07002"""Script to bootstrap the chroot using new toolchain.
3
4This script allows you to build/install a customized version of gcc/binutils,
5either by specifying branch or a local directory.
6
7This script must be executed outside chroot.
8
9Below is some typical usage -
10
Caroline Tice88272d42016-01-13 09:48:29 -080011## Build gcc located at /local/gcc/dir and do a bootstrap using the new
12## compiler for the chromeos root. The script tries to find a valid chromeos
13## tree all the way up from your current working directory.
Han Shen819f8622014-04-08 16:57:38 -070014./build_tool.py --gcc_dir=/loca/gcc/dir --bootstrap
15
Caroline Tice88272d42016-01-13 09:48:29 -080016## Build binutils, using remote branch "mobile_toolchain_v17" and do a
17## bootstrap using the new binutils for the chromeos root. The script tries to
18## find a valid chromeos tree all the way up from your current working
19## directory.
Han Shen819f8622014-04-08 16:57:38 -070020./build_tool.py --binutils_branch=cros/mobile_toolchain_v17 \
21 --chromeos_root=/chromeos/dir --bootstrap
22
23## Same as above except only do it for board daisy - no bootstrapping involved.
24./build_tool.py --binutils_branch=cros/mobile_toolchain_v16 \
25 --chromeos_root=/chromeos/dir --board=daisy
26"""
Han Shen91445322013-03-20 13:43:31 -070027
Caroline Tice88272d42016-01-13 09:48:29 -080028from __future__ import print_function
29
Han Shen91445322013-03-20 13:43:31 -070030__author__ = 'shenhan@google.com (Han Shen)'
31
Caroline Tice88272d42016-01-13 09:48:29 -080032import argparse
Han Shen91445322013-03-20 13:43:31 -070033import os
34import re
Han Shen91445322013-03-20 13:43:31 -070035import sys
36
Caroline Tice88272d42016-01-13 09:48:29 -080037from cros_utils import command_executer
38from cros_utils import logger
39from cros_utils import misc
Han Shend1c31b22016-04-18 14:58:57 -070040import repo_to_repo
Han Shen91445322013-03-20 13:43:31 -070041
Han Shen819f8622014-04-08 16:57:38 -070042REPO_PATH_PATTERN = 'src/third_party/{0}'
43TEMP_BRANCH_NAME = 'internal_testing_branch_no_use'
44CHROMIUMOS_OVERLAY_PATH = 'src/third_party/chromiumos-overlay'
45EBUILD_PATH_PATTERN = 'src/third_party/chromiumos-overlay/sys-devel/{0}'
46
Han Shen91445322013-03-20 13:43:31 -070047
48class Bootstrapper(object):
Caroline Tice88272d42016-01-13 09:48:29 -080049 """Class that handles bootstrap process."""
Han Shen819f8622014-04-08 16:57:38 -070050
Luis Lozanof2a3ef42015-12-15 13:49:30 -080051 def __init__(self,
52 chromeos_root,
Han Shend1c31b22016-04-18 14:58:57 -070053 ndk_dir,
Luis Lozanof2a3ef42015-12-15 13:49:30 -080054 gcc_branch=None,
55 gcc_dir=None,
56 binutils_branch=None,
57 binutils_dir=None,
58 board=None,
59 disable_2nd_bootstrap=False,
Han Shen03d30982014-06-12 11:22:29 -070060 setup_tool_ebuild_file_only=False):
Han Shen91445322013-03-20 13:43:31 -070061 self._chromeos_root = chromeos_root
Han Shend1c31b22016-04-18 14:58:57 -070062 self._ndk_dir = ndk_dir
Han Shen819f8622014-04-08 16:57:38 -070063
64 self._gcc_branch = gcc_branch
65 self._gcc_branch_tree = None
Han Shen91445322013-03-20 13:43:31 -070066 self._gcc_dir = gcc_dir
Han Shen91445322013-03-20 13:43:31 -070067 self._gcc_ebuild_file = None
68 self._gcc_ebuild_file_name = None
Han Shen819f8622014-04-08 16:57:38 -070069
70 self._binutils_branch = binutils_branch
71 self._binutils_branch_tree = None
72 self._binutils_dir = binutils_dir
73 self._binutils_ebuild_file = None
74 self._binutils_ebuild_file_name = None
75
76 self._setup_tool_ebuild_file_only = setup_tool_ebuild_file_only
77
78 self._ce = command_executer.GetCommandExecuter()
79 self._logger = logger.GetLogger()
80 self._board = board
Han Shen03d30982014-06-12 11:22:29 -070081 self._disable_2nd_bootstrap = disable_2nd_bootstrap
Han Shen819f8622014-04-08 16:57:38 -070082
83 def IsTreeSame(self, t1, t2):
84 diff = 'diff -qr -x .git -x .svn "{0}" "{1}"'.format(t1, t2)
85 if self._ce.RunCommand(diff, print_to_console=False) == 0:
86 self._logger.LogOutput('"{0}" and "{1}" are the same."'.format(t1, t2))
87 return True
88 self._logger.LogWarning('"{0}" and "{1}" are different."'.format(t1, t2))
89 return False
Han Shen91445322013-03-20 13:43:31 -070090
91 def SubmitToLocalBranch(self):
Han Shen819f8622014-04-08 16:57:38 -070092 """Copy source code to the chromium source tree and submit it locally."""
93 if self._gcc_dir:
Caroline Ticef6ef4392017-04-06 17:16:05 -070094 if not self.SubmitToolToLocalBranch(
95 tool_name='gcc', tool_dir=self._gcc_dir):
Han Shen819f8622014-04-08 16:57:38 -070096 return False
97 self._gcc_branch = TEMP_BRANCH_NAME
Han Shen91445322013-03-20 13:43:31 -070098
Han Shen819f8622014-04-08 16:57:38 -070099 if self._binutils_dir:
Caroline Ticef6ef4392017-04-06 17:16:05 -0700100 if not self.SubmitToolToLocalBranch(
101 tool_name='binutils', tool_dir=self._binutils_dir):
Han Shen819f8622014-04-08 16:57:38 -0700102 return False
103 self._binutils_branch = TEMP_BRANCH_NAME
104
105 return True
106
107 def SubmitToolToLocalBranch(self, tool_name, tool_dir):
108 """Copy the source code to local chromium source tree.
109
110 Args:
111 tool_name: either 'gcc' or 'binutils'
112 tool_dir: the tool source dir to be used
Caroline Tice88272d42016-01-13 09:48:29 -0800113
Han Shen819f8622014-04-08 16:57:38 -0700114 Returns:
115 True if all succeeded False otherwise.
116 """
117
118 # The next few steps creates an internal branch to sync with the tool dir
Han Shen91445322013-03-20 13:43:31 -0700119 # user provided.
Han Shen819f8622014-04-08 16:57:38 -0700120 chrome_tool_dir = self.GetChromeOsToolDir(tool_name)
Han Shen91445322013-03-20 13:43:31 -0700121
122 # 0. Test to see if git tree is free of local changes.
Han Shen819f8622014-04-08 16:57:38 -0700123 if not misc.IsGitTreeClean(chrome_tool_dir):
Caroline Ticef6ef4392017-04-06 17:16:05 -0700124 self._logger.LogError(
125 'Git repository "{0}" not clean, aborted.'.format(chrome_tool_dir))
Han Shen91445322013-03-20 13:43:31 -0700126 return False
127
128 # 1. Checkout/create a (new) branch for testing.
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800129 command = 'cd "{0}" && git checkout -B {1}'.format(chrome_tool_dir,
130 TEMP_BRANCH_NAME)
Han Shen91445322013-03-20 13:43:31 -0700131 ret = self._ce.RunCommand(command)
132 if ret:
133 self._logger.LogError('Failed to create a temp branch for test, aborted.')
134 return False
135
Han Shen819f8622014-04-08 16:57:38 -0700136 if self.IsTreeSame(tool_dir, chrome_tool_dir):
Caroline Ticef6ef4392017-04-06 17:16:05 -0700137 self._logger.LogOutput('"{0}" and "{1}" are the same, sync skipped.'.
138 format(tool_dir, chrome_tool_dir))
Han Shen819f8622014-04-08 16:57:38 -0700139 return True
Han Shen91445322013-03-20 13:43:31 -0700140
Han Shen819f8622014-04-08 16:57:38 -0700141 # 2. Sync sources from user provided tool dir to chromiumos tool git.
142 local_tool_repo = repo_to_repo.FileRepo(tool_dir)
143 chrome_tool_repo = repo_to_repo.GitRepo(chrome_tool_dir, TEMP_BRANCH_NAME)
Caroline Tice88272d42016-01-13 09:48:29 -0800144 chrome_tool_repo.SetRoot(chrome_tool_dir)
Han Shen03d30982014-06-12 11:22:29 -0700145 # Delete all stuff except '.git' before start mapping.
146 self._ce.RunCommand(
147 'cd {0} && find . -maxdepth 1 -not -name ".git" -not -name "." '
148 r'\( -type f -exec rm {{}} \; -o '
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800149 r' -type d -exec rm -fr {{}} \; \)'.format(chrome_tool_dir))
Han Shen819f8622014-04-08 16:57:38 -0700150 local_tool_repo.MapSources(chrome_tool_repo.GetRoot())
151
152 # 3. Ensure after sync tree is the same.
153 if self.IsTreeSame(tool_dir, chrome_tool_dir):
Han Shen91445322013-03-20 13:43:31 -0700154 self._logger.LogOutput('Sync successfully done.')
Han Shen819f8622014-04-08 16:57:38 -0700155 else:
156 self._logger.LogError('Sync not successful, aborted.')
157 return False
Han Shen91445322013-03-20 13:43:31 -0700158
159 # 4. Commit all changes.
Han Shen03d30982014-06-12 11:22:29 -0700160 # 4.1 Try to get some information about the tool dir we are using.
161 cmd = 'cd {0} && git log -1 --pretty=oneline'.format(tool_dir)
162 tool_dir_extra_info = None
Luis Lozano036c9232015-12-10 10:47:01 -0800163 ret, tool_dir_extra_info, _ = self._ce.RunCommandWOutput(
Caroline Ticef6ef4392017-04-06 17:16:05 -0700164 cmd, print_to_console=False)
Han Shen03d30982014-06-12 11:22:29 -0700165 commit_message = 'Synced with tool source tree at - "{0}".'.format(tool_dir)
166 if not ret:
Han Shend1c31b22016-04-18 14:58:57 -0700167 commit_message += '\nGit log for {0}:\n{1}'.format(
168 tool_dir, tool_dir_extra_info.strip())
Han Shen03d30982014-06-12 11:22:29 -0700169
170 if chrome_tool_repo.CommitLocally(commit_message):
Caroline Ticef6ef4392017-04-06 17:16:05 -0700171 self._logger.LogError('Commit to local branch "{0}" failed, aborted.'.
172 format(TEMP_BRANCH_NAME))
Han Shen91445322013-03-20 13:43:31 -0700173 return False
174 return True
175
176 def CheckoutBranch(self):
Han Shen819f8622014-04-08 16:57:38 -0700177 """Checkout working branch for the tools.
Han Shen91445322013-03-20 13:43:31 -0700178
Han Shen819f8622014-04-08 16:57:38 -0700179 Returns:
180 True: if operation succeeds.
181 """
Han Shen91445322013-03-20 13:43:31 -0700182
Han Shen819f8622014-04-08 16:57:38 -0700183 if self._gcc_branch:
184 rv = self.CheckoutToolBranch('gcc', self._gcc_branch)
185 if rv:
186 self._gcc_branch_tree = rv
187 else:
188 return False
189
190 if self._binutils_branch:
191 rv = self.CheckoutToolBranch('binutils', self._binutils_branch)
192 if rv:
193 self._binutils_branch_tree = rv
194 else:
195 return False
196
Han Shen91445322013-03-20 13:43:31 -0700197 return True
198
Han Shen819f8622014-04-08 16:57:38 -0700199 def CheckoutToolBranch(self, tool_name, tool_branch):
200 """Checkout the tool branch for a certain tool.
201
202 Args:
203 tool_name: either 'gcc' or 'binutils'
204 tool_branch: tool branch to use
Caroline Tice88272d42016-01-13 09:48:29 -0800205
Han Shen819f8622014-04-08 16:57:38 -0700206 Returns:
207 True: if operation succeeds. Otherwise False.
Han Shen91445322013-03-20 13:43:31 -0700208 """
Han Shen819f8622014-04-08 16:57:38 -0700209
210 chrome_tool_dir = self.GetChromeOsToolDir(tool_name)
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800211 command = 'cd "{0}" && git checkout {1}'.format(chrome_tool_dir,
212 tool_branch)
Han Shen819f8622014-04-08 16:57:38 -0700213 if not self._ce.RunCommand(command, print_to_console=True):
214 # Get 'TREE' value of this commit
215 command = ('cd "{0}" && git cat-file -p {1} '
216 '| grep -E "^tree [a-f0-9]+$" '
217 '| cut -d" " -f2').format(chrome_tool_dir, tool_branch)
Caroline Ticef6ef4392017-04-06 17:16:05 -0700218 ret, stdout, _ = self._ce.RunCommandWOutput(
219 command, print_to_console=False)
Han Shen819f8622014-04-08 16:57:38 -0700220 # Pipe operation always has a zero return value. So need to check if
221 # stdout is valid.
Caroline Ticef6ef4392017-04-06 17:16:05 -0700222 if not ret and stdout and re.match('[0-9a-h]{40}',
223 stdout.strip(), re.IGNORECASE):
Han Shen819f8622014-04-08 16:57:38 -0700224 tool_branch_tree = stdout.strip()
225 self._logger.LogOutput('Find tree for {0} branch "{1}" - "{2}"'.format(
226 tool_name, tool_branch, tool_branch_tree))
227 return tool_branch_tree
228 self._logger.LogError(('Failed to checkout "{0}" or failed to '
229 'get tree value, aborted.').format(tool_branch))
230 return None
231
232 def FindEbuildFile(self):
233 """Find the ebuild files for the tools.
234
235 Returns:
236 True: if operation succeeds.
237 """
238
239 if self._gcc_branch:
240 (rv, ef, efn) = self.FindToolEbuildFile('gcc')
241 if rv:
242 self._gcc_ebuild_file = ef
243 self._gcc_ebuild_file_name = efn
244 else:
245 return False
246
247 if self._binutils_branch:
248 (rv, ef, efn) = self.FindToolEbuildFile('binutils')
249 if rv:
250 self._binutils_ebuild_file = ef
251 self._binutils_ebuild_file_name = efn
252 else:
253 return False
254
255 return True
256
257 def FindToolEbuildFile(self, tool_name):
258 """Find ebuild file for a specific tool.
259
260 Args:
261 tool_name: either "gcc" or "binutils".
Caroline Tice88272d42016-01-13 09:48:29 -0800262
Han Shen819f8622014-04-08 16:57:38 -0700263 Returns:
264 A triplet that consisits of whether operation succeeds or not,
265 tool ebuild file full path and tool ebuild file name.
266 """
267
268 # To get the active gcc ebuild file, we need a workable chroot first.
Caroline Ticef6ef4392017-04-06 17:16:05 -0700269 if not os.path.exists(
270 os.path.join(self._chromeos_root, 'chroot')) and self._ce.RunCommand(
Han Shen819f8622014-04-08 16:57:38 -0700271 'cd "{0}" && cros_sdk --create'.format(self._chromeos_root)):
272 self._logger.LogError(('Failed to install a initial chroot, aborted.\n'
273 'If previous bootstrap failed, do a '
274 '"cros_sdk --delete" to remove '
275 'in-complete chroot.'))
276 return (False, None, None)
277
Luis Lozano036c9232015-12-10 10:47:01 -0800278 rv, stdout, _ = self._ce.ChrootRunCommandWOutput(
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800279 self._chromeos_root,
280 'equery w sys-devel/{0}'.format(tool_name),
Luis Lozano036c9232015-12-10 10:47:01 -0800281 print_to_console=True)
Han Shen819f8622014-04-08 16:57:38 -0700282 if rv:
Caroline Ticef6ef4392017-04-06 17:16:05 -0700283 self._logger.LogError(
284 ('Failed to execute inside chroot '
285 '"equery w sys-devel/{0}", aborted.').format(tool_name))
Han Shen819f8622014-04-08 16:57:38 -0700286 return (False, None, None)
Caroline Ticef6ef4392017-04-06 17:16:05 -0700287 m = re.match(r'^.*/({0}/(.*\.ebuild))$'.format(
288 EBUILD_PATH_PATTERN.format(tool_name)), stdout)
Han Shen819f8622014-04-08 16:57:38 -0700289 if not m:
290 self._logger.LogError(
291 ('Failed to find {0} ebuild file, aborted. '
292 'If previous bootstrap failed, do a "cros_sdk --delete" to remove '
293 'in-complete chroot.').format(tool_name))
294 return (False, None, None)
295 tool_ebuild_file = os.path.join(self._chromeos_root, m.group(1))
296 tool_ebuild_file_name = m.group(2)
297
298 return (True, tool_ebuild_file, tool_ebuild_file_name)
299
300 def InplaceModifyEbuildFile(self):
301 """Modify the ebuild file.
302
303 Returns:
304 True if operation succeeds.
305 """
306
307 # Note we shall not use remote branch name (eg. "cros/gcc.gnu.org/...") in
308 # CROS_WORKON_COMMIT, we have to use GITHASH. So we call GitGetCommitHash on
309 # tool_branch.
Han Shend1c31b22016-04-18 14:58:57 -0700310 tool = None
311 toolbranch = None
Han Shen819f8622014-04-08 16:57:38 -0700312 if self._gcc_branch:
Han Shend1c31b22016-04-18 14:58:57 -0700313 tool = 'gcc'
314 toolbranch = self._gcc_branch
315 tooltree = self._gcc_branch_tree
316 toolebuild = self._gcc_ebuild_file
317 elif self._binutils_branch:
318 tool = 'binutils'
319 toolbranch = self._binutils_branch
320 tooltree = self._binutils_branch_tree
321 toolebuild = self._binutils_ebuild_file
Han Shen819f8622014-04-08 16:57:38 -0700322
Han Shend1c31b22016-04-18 14:58:57 -0700323 assert tool
324
325 # An example for the following variables would be:
326 # tooldir = '~/android/master-ndk/toolchain/gcc/gcc-4.9'
327 # tool_branch_githash = xxxxx
328 # toolcomponents = toolchain/gcc
329 tooldir = self.GetChromeOsToolDir(tool)
330 toolgithash = misc.GitGetCommitHash(tooldir, toolbranch)
331 if not toolgithash:
332 return False
333 toolcomponents = 'toolchain/{}'.format(tool)
Caroline Ticef6ef4392017-04-06 17:16:05 -0700334 return self.InplaceModifyToolEbuildFile(toolcomponents, toolgithash,
335 tooltree, toolebuild)
Han Shen819f8622014-04-08 16:57:38 -0700336
337 @staticmethod
338 def ResetToolEbuildFile(chromeos_root, tool_name):
339 """Reset tool ebuild file to clean state.
340
341 Args:
342 chromeos_root: chromeos source tree
343 tool_name: either "gcc" or "binutils"
Caroline Tice88272d42016-01-13 09:48:29 -0800344
Han Shen819f8622014-04-08 16:57:38 -0700345 Returns:
346 True if operation succeds.
347 """
348 rv = misc.GetGitChangesAsList(
349 os.path.join(chromeos_root, CHROMIUMOS_OVERLAY_PATH),
350 path=('sys-devel/{0}/{0}-*.ebuild'.format(tool_name)),
351 staged=False)
352 if rv:
Caroline Ticef6ef4392017-04-06 17:16:05 -0700353 cmd = 'cd {0} && git checkout --'.format(
354 os.path.join(chromeos_root, CHROMIUMOS_OVERLAY_PATH))
Han Shen819f8622014-04-08 16:57:38 -0700355 for g in rv:
356 cmd += ' ' + g
357 rv = command_executer.GetCommandExecuter().RunCommand(cmd)
358 if rv:
359 logger.GetLogger().LogWarning(
360 'Failed to reset the ebuild file. Please refer to log above.')
361 return False
362 else:
363 logger.GetLogger().LogWarning(
364 'Note - did not find any modified {0} ebuild file.'.format(tool_name))
365 # Fall through
366 return True
367
368 def GetChromeOsToolDir(self, tool_name):
369 """Return the chromeos git dir for a specific tool.
370
Han Shend1c31b22016-04-18 14:58:57 -0700371 Note, after we unified ChromeOs and Android, the tool dir is under
372 ndk_dir/toolchain/[gcc,binutils].
373
Han Shen819f8622014-04-08 16:57:38 -0700374 Args:
375 tool_name: either 'gcc' or 'binutils'.
Caroline Tice88272d42016-01-13 09:48:29 -0800376
Han Shen819f8622014-04-08 16:57:38 -0700377 Returns:
378 Absolute git path for the tool.
379 """
380
Han Shend1c31b22016-04-18 14:58:57 -0700381 tool_toppath = os.path.join(self._ndk_dir, 'toolchain', tool_name)
382 # There may be sub-directories like 'binutils-2.25', 'binutils-2.24',
383 # 'gcc-4.9', 'gcc-4.8', etc. find the newest binutils version.
384 cmd = ('find {} -maxdepth 1 -type d -name "{}-*" '
385 '| sort -r | head -1').format(tool_toppath, tool_name)
386 rv, out, _ = self._ce.RunCommandWOutput(cmd, print_to_console=False)
387 if rv:
388 return None
389 repo = out.strip()
Han Shen819f8622014-04-08 16:57:38 -0700390
Han Shend1c31b22016-04-18 14:58:57 -0700391 # cros-workon eclass expects every CROS_WORKON_PROJECT ends with ".git".
392 self._ce.RunCommand(('cd $(dirname {0}) && '
393 'ln -sf $(basename {0}) $(basename {0}).git').format(
394 repo, print_to_console=True))
395 return repo
396
Caroline Ticef6ef4392017-04-06 17:16:05 -0700397 def InplaceModifyToolEbuildFile(self, tool_components, tool_branch_githash,
398 tool_branch_tree, tool_ebuild_file):
Han Shen819f8622014-04-08 16:57:38 -0700399 """Using sed to fill properly values into the ebuild file.
400
401 Args:
Han Shend1c31b22016-04-18 14:58:57 -0700402 tool_components: either "toolchain/gcc" or "toolchain/binutils"
Han Shen819f8622014-04-08 16:57:38 -0700403 tool_branch_githash: githash for tool_branch
404 tool_branch_tree: treeish for the tool branch
405 tool_ebuild_file: tool ebuild file
Caroline Tice88272d42016-01-13 09:48:29 -0800406
Han Shen819f8622014-04-08 16:57:38 -0700407 Returns:
408 True: if operation succeeded.
409 """
410
411 command = ('sed -i '
Han Shend1c31b22016-04-18 14:58:57 -0700412 '-e \'/^CROS_WORKON_REPO=".*"/i'
413 ' # The following line is modified by script.\' '
414 '-e \'s!^CROS_WORKON_REPO=".*"$!CROS_WORKON_REPO="{0}"!\' '
415 '-e \'/^CROS_WORKON_PROJECT=".*"/i'
416 ' # The following line is modified by script.\' '
417 '-e \'s!^CROS_WORKON_PROJECT=.*$!CROS_WORKON_PROJECT="{1}"!\' '
Han Shen819f8622014-04-08 16:57:38 -0700418 '-e \'/^CROS_WORKON_COMMIT=".*"/i'
419 ' # The following line is modified by script.\' '
Han Shend1c31b22016-04-18 14:58:57 -0700420 '-e \'s!^CROS_WORKON_COMMIT=".*"$!CROS_WORKON_COMMIT="{2}"!\' '
Han Shen819f8622014-04-08 16:57:38 -0700421 '-e \'/^CROS_WORKON_TREE=".*"/i'
422 ' # The following line is modified by script.\' '
Han Shend1c31b22016-04-18 14:58:57 -0700423 '-e \'s!^CROS_WORKON_TREE=".*"$!CROS_WORKON_TREE="{3}"!\' '
424 '{4}').format('/home/{}/ndk-root'.format(os.environ['USER']),
Caroline Ticef6ef4392017-04-06 17:16:05 -0700425 tool_components, tool_branch_githash,
426 tool_branch_tree, tool_ebuild_file)
Han Shen91445322013-03-20 13:43:31 -0700427 rv = self._ce.RunCommand(command)
428 if rv:
429 self._logger.LogError(
Han Shen819f8622014-04-08 16:57:38 -0700430 'Failed to modify commit and tree value for "{0}"", aborted.'.format(
431 tool_ebuild_file))
432 return False
433
434 # Warn that the ebuild file has been modified.
435 self._logger.LogWarning(
436 ('Ebuild file "{0}" is modified, to revert the file - \n'
437 'bootstrap_compiler.py --chromeos_root={1} '
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800438 '--reset_tool_ebuild_file').format(tool_ebuild_file,
439 self._chromeos_root))
Han Shen819f8622014-04-08 16:57:38 -0700440 return True
441
442 def DoBuildForBoard(self):
443 """Build tool for a specific board.
444
445 Returns:
446 True if operation succeeds.
447 """
448
449 if self._gcc_branch:
450 if not self.DoBuildToolForBoard('gcc'):
451 return False
452 if self._binutils_branch:
453 if not self.DoBuildToolForBoard('binutils'):
454 return False
455 return True
456
457 def DoBuildToolForBoard(self, tool_name):
458 """Build a specific tool for a specific board.
459
460 Args:
461 tool_name: either "gcc" or "binutils"
Caroline Tice88272d42016-01-13 09:48:29 -0800462
Han Shen819f8622014-04-08 16:57:38 -0700463 Returns:
464 True if operation succeeds.
465 """
466
Caroline Ticef6ef4392017-04-06 17:16:05 -0700467 chroot_ndk_root = os.path.join(self._chromeos_root, 'chroot', 'home',
468 os.environ['USER'], 'ndk-root')
Han Shend1c31b22016-04-18 14:58:57 -0700469 self._ce.RunCommand('mkdir -p {}'.format(chroot_ndk_root))
Caroline Ticef6ef4392017-04-06 17:16:05 -0700470 if self._ce.RunCommand(
471 'sudo mount --bind {} {}'.format(self._ndk_dir, chroot_ndk_root)):
Han Shend1c31b22016-04-18 14:58:57 -0700472 self._logger.LogError('Failed to mount ndk dir into chroot')
473 return False
Han Shen819f8622014-04-08 16:57:38 -0700474
Han Shend1c31b22016-04-18 14:58:57 -0700475 try:
476 boards_to_build = self._board.split(',')
477 target_built = set()
478 failed = []
479 for board in boards_to_build:
480 if board == 'host':
481 command = 'sudo emerge sys-devel/{0}'.format(tool_name)
482 else:
483 target = misc.GetCtargetFromBoard(board, self._chromeos_root)
484 if not target:
485 self._logger.LogError(
486 'Unsupported board "{0}", skip.'.format(board))
487 failed.append(board)
488 continue
489 # Skip this board if we have already built for a board that has the
490 # same target.
491 if target in target_built:
492 self._logger.LogWarning(
493 'Skipping toolchain for board "{}"'.format(board))
494 continue
495 target_built.add(target)
496 command = 'sudo emerge cross-{0}/{1}'.format(target, tool_name)
497
Caroline Ticef6ef4392017-04-06 17:16:05 -0700498 rv = self._ce.ChrootRunCommand(
499 self._chromeos_root, command, print_to_console=True)
Han Shend1c31b22016-04-18 14:58:57 -0700500 if rv:
Caroline Ticef6ef4392017-04-06 17:16:05 -0700501 self._logger.LogError(
502 'Build {0} failed for {1}, aborted.'.format(tool_name, board))
Han Shenfe3001c2014-04-28 16:36:28 -0700503 failed.append(board)
Han Shend1c31b22016-04-18 14:58:57 -0700504 else:
Caroline Ticef6ef4392017-04-06 17:16:05 -0700505 self._logger.LogOutput(
506 'Successfully built {0} for board {1}.'.format(tool_name, board))
Han Shend1c31b22016-04-18 14:58:57 -0700507 finally:
508 # Make sure we un-mount ndk-root before we leave here, regardless of the
509 # build result of the tool. Otherwise we may inadvertently delete ndk-root
510 # dir, which is not part of the chroot and could be disastrous.
511 if chroot_ndk_root:
512 if self._ce.RunCommand('sudo umount {}'.format(chroot_ndk_root)):
Caroline Ticef6ef4392017-04-06 17:16:05 -0700513 self._logger.LogWarning(
514 ('Failed to umount "{}", please check '
515 'before deleting chroot.').format(chroot_ndk_root))
Han Shenfe3001c2014-04-28 16:36:28 -0700516
Han Shend1c31b22016-04-18 14:58:57 -0700517 # Clean up soft links created during build.
518 self._ce.RunCommand('cd {}/toolchain/{} && git clean -df'.format(
519 self._ndk_dir, tool_name))
Han Shenfe3001c2014-04-28 16:36:28 -0700520
Han Shen8dfdabd2014-05-08 10:15:09 -0700521 if failed:
Han Shen819f8622014-04-08 16:57:38 -0700522 self._logger.LogError(
Han Shenfe3001c2014-04-28 16:36:28 -0700523 'Failed to build {0} for the following board(s): "{1}"'.format(
524 tool_name, ' '.join(failed)))
Han Shen91445322013-03-20 13:43:31 -0700525 return False
Han Shenfe3001c2014-04-28 16:36:28 -0700526 # All boards build successfully
Han Shen91445322013-03-20 13:43:31 -0700527 return True
528
529 def DoBootstrapping(self):
Han Shen819f8622014-04-08 16:57:38 -0700530 """Do bootstrapping the chroot.
531
Han Shen03d30982014-06-12 11:22:29 -0700532 This step firstly downloads a prestine sdk, then use this sdk to build the
533 new sdk, finally use the new sdk to build every host package.
534
Han Shen819f8622014-04-08 16:57:38 -0700535 Returns:
536 True if operation succeeds.
537 """
538
Han Shen91445322013-03-20 13:43:31 -0700539 logfile = os.path.join(self._chromeos_root, 'bootstrap.log')
Han Shen819f8622014-04-08 16:57:38 -0700540 command = 'cd "{0}" && cros_sdk --delete --bootstrap |& tee "{1}"'.format(
541 self._chromeos_root, logfile)
Luis Lozano036c9232015-12-10 10:47:01 -0800542 rv = self._ce.RunCommand(command, print_to_console=True)
Han Shen91445322013-03-20 13:43:31 -0700543 if rv:
Caroline Ticef6ef4392017-04-06 17:16:05 -0700544 self._logger.LogError(
545 'Bootstrapping failed, log file - "{0}"\n'.format(logfile))
Han Shen91445322013-03-20 13:43:31 -0700546 return False
547
548 self._logger.LogOutput('Bootstrap succeeded.')
549 return True
550
Han Shen03d30982014-06-12 11:22:29 -0700551 def BuildAndInstallAmd64Host(self):
552 """Build amd64-host (host) packages.
553
554 Build all host packages in the newly-bootstrapped 'chroot' using *NEW*
555 toolchain.
556
557 So actually we perform 2 builds of all host packages -
558 1. build new toolchain using old toolchain and build all host packages
559 using the newly built toolchain
560 2. build the new toolchain again but using new toolchain built in step 1,
561 and build all host packages using the newly built toolchain
562
563 Returns:
564 True if operation succeeds.
565 """
566
Alex Klein85214fa2019-01-15 13:59:05 -0700567 cmd = 'cd {0} && cros_sdk -- -- ./build_sdk_board'.format(
568 self._chromeos_root)
Luis Lozano036c9232015-12-10 10:47:01 -0800569 rv = self._ce.RunCommand(cmd, print_to_console=True)
Han Shen03d30982014-06-12 11:22:29 -0700570 if rv:
571 self._logger.LogError('Build amd64-host failed.')
572 return False
573
574 # Package amd64-host into 'built-sdk.tar.xz'.
575 sdk_package = os.path.join(self._chromeos_root, 'built-sdk.tar.xz')
576 cmd = ('cd {0}/chroot/build/amd64-host && sudo XZ_OPT="-e9" '
577 'tar --exclude="usr/lib/debug/*" --exclude="packages/*" '
578 '--exclude="tmp/*" --exclude="usr/local/build/autotest/*" '
579 '--sparse -I xz -vcf {1} . && sudo chmod a+r {1}').format(
580 self._chromeos_root, sdk_package)
Luis Lozano036c9232015-12-10 10:47:01 -0800581 rv = self._ce.RunCommand(cmd, print_to_console=True)
Han Shen03d30982014-06-12 11:22:29 -0700582 if rv:
583 self._logger.LogError('Failed to create "built-sdk.tar.xz".')
584 return False
585
586 # Install amd64-host into a new chroot.
587 cmd = ('cd {0} && cros_sdk --chroot new-sdk-chroot --download --replace '
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800588 '--nousepkg --url file://{1}').format(self._chromeos_root,
589 sdk_package)
Luis Lozano036c9232015-12-10 10:47:01 -0800590 rv = self._ce.RunCommand(cmd, print_to_console=True)
Han Shen03d30982014-06-12 11:22:29 -0700591 if rv:
592 self._logger.LogError('Failed to install "built-sdk.tar.xz".')
593 return False
594 self._logger.LogOutput(
595 'Successfully installed built-sdk.tar.xz into a new chroot.\nAll done.')
596
597 # Rename the newly created new-sdk-chroot to chroot.
598 cmd = ('cd {0} && sudo mv chroot chroot-old && '
599 'sudo mv new-sdk-chroot chroot').format(self._chromeos_root)
Luis Lozano036c9232015-12-10 10:47:01 -0800600 rv = self._ce.RunCommand(cmd, print_to_console=True)
Han Shen03d30982014-06-12 11:22:29 -0700601 return rv == 0
602
Han Shen91445322013-03-20 13:43:31 -0700603 def Do(self):
Han Shen819f8622014-04-08 16:57:38 -0700604 """Entrance of the class.
605
606 Returns:
607 True if everything is ok.
608 """
609
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800610 if (self.SubmitToLocalBranch() and self.CheckoutBranch() and
611 self.FindEbuildFile() and self.InplaceModifyEbuildFile()):
Han Shen819f8622014-04-08 16:57:38 -0700612 if self._setup_tool_ebuild_file_only:
613 # Everything is done, we are good.
614 ret = True
615 else:
616 if self._board:
617 ret = self.DoBuildForBoard()
618 else:
619 # This implies '--bootstrap'.
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800620 ret = (self.DoBootstrapping() and (self._disable_2nd_bootstrap or
621 self.BuildAndInstallAmd64Host()))
Han Shen91445322013-03-20 13:43:31 -0700622 else:
623 ret = False
Han Shen91445322013-03-20 13:43:31 -0700624 return ret
625
626
627def Main(argv):
Caroline Tice88272d42016-01-13 09:48:29 -0800628 parser = argparse.ArgumentParser()
Caroline Ticef6ef4392017-04-06 17:16:05 -0700629 parser.add_argument(
630 '-c',
631 '--chromeos_root',
632 dest='chromeos_root',
633 help=('Optional. ChromeOs root dir. '
634 'When not specified, chromeos root will be deduced'
635 ' from current working directory.'))
636 parser.add_argument(
637 '--ndk_dir',
638 dest='ndk_dir',
639 help=('Topmost android ndk dir, required. '
640 'Do not need to include the "toolchain/*" part.'))
641 parser.add_argument(
642 '--gcc_branch',
643 dest='gcc_branch',
644 help=('The branch to test against. '
645 'This branch must be a local branch '
646 'inside "src/third_party/gcc". '
647 'Notice, this must not be used with "--gcc_dir".'))
648 parser.add_argument(
649 '--binutils_branch',
650 dest='binutils_branch',
651 help=('The branch to test against binutils. '
652 'This branch must be a local branch '
653 'inside "src/third_party/binutils". '
654 'Notice, this must not be used with '
655 '"--binutils_dir".'))
656 parser.add_argument(
657 '-g',
658 '--gcc_dir',
659 dest='gcc_dir',
660 help=('Use a local gcc tree to do bootstrapping. '
661 'Notice, this must not be used with '
662 '"--gcc_branch".'))
663 parser.add_argument(
664 '--binutils_dir',
665 dest='binutils_dir',
666 help=('Use a local binutils tree to do bootstrapping. '
667 'Notice, this must not be used with '
668 '"--binutils_branch".'))
669 parser.add_argument(
670 '--fixperm',
671 dest='fixperm',
672 default=False,
673 action='store_true',
674 help=('Fix the (notorious) permission error '
675 'while trying to bootstrap the chroot. '
676 'Note this takes an extra 10-15 minutes '
677 'and is only needed once per chromiumos tree.'))
678 parser.add_argument(
679 '--setup_tool_ebuild_file_only',
680 dest='setup_tool_ebuild_file_only',
681 default=False,
682 action='store_true',
683 help=('Setup gcc and/or binutils ebuild file '
684 'to pick up the branch (--gcc/binutils_branch) or '
685 'use gcc and/or binutils source '
686 '(--gcc/binutils_dir) and exit. Keep chroot as is.'
687 ' This should not be used with '
688 '--gcc/binutils_dir/branch options.'))
689 parser.add_argument(
690 '--reset_tool_ebuild_file',
691 dest='reset_tool_ebuild_file',
692 default=False,
693 action='store_true',
694 help=('Reset the modification that is done by this '
695 'script. Note, when this script is running, it '
696 'will modify the active gcc/binutils ebuild file. '
697 'Use this option to reset (what this script has '
698 'done) and exit. This should not be used with -- '
699 'gcc/binutils_dir/branch options.'))
700 parser.add_argument(
701 '--board',
702 dest='board',
703 default=None,
704 help=('Only build toolchain for specific board(s). '
705 'Use "host" to build for host. '
706 'Use "," to seperate multiple boards. '
707 'This does not perform a chroot bootstrap.'))
708 parser.add_argument(
709 '--bootstrap',
710 dest='bootstrap',
711 default=False,
712 action='store_true',
713 help=('Performs a chroot bootstrap. '
714 'Note, this will *destroy* your current chroot.'))
715 parser.add_argument(
716 '--disable-2nd-bootstrap',
717 dest='disable_2nd_bootstrap',
718 default=False,
719 action='store_true',
720 help=('Disable a second bootstrap '
721 '(build of amd64-host stage).'))
Han Shen819f8622014-04-08 16:57:38 -0700722
Caroline Tice88272d42016-01-13 09:48:29 -0800723 options = parser.parse_args(argv)
Han Shen819f8622014-04-08 16:57:38 -0700724 # Trying to deduce chromeos root from current directory.
Han Shen91445322013-03-20 13:43:31 -0700725 if not options.chromeos_root:
Han Shen819f8622014-04-08 16:57:38 -0700726 logger.GetLogger().LogOutput('Trying to deduce chromeos root ...')
727 wdir = os.getcwd()
728 while wdir and wdir != '/':
729 if misc.IsChromeOsTree(wdir):
730 logger.GetLogger().LogOutput('Find chromeos_root: {}'.format(wdir))
731 options.chromeos_root = wdir
732 break
733 wdir = os.path.dirname(wdir)
734
735 if not options.chromeos_root:
736 parser.error('Missing or failing to deduce mandatory option "--chromeos".')
Han Shen91445322013-03-20 13:43:31 -0700737 return 1
738
Caroline Ticef6ef4392017-04-06 17:16:05 -0700739 options.chromeos_root = os.path.abspath(
740 os.path.expanduser(options.chromeos_root))
Han Shen91445322013-03-20 13:43:31 -0700741
742 if not os.path.isdir(options.chromeos_root):
Caroline Ticef6ef4392017-04-06 17:16:05 -0700743 logger.GetLogger().LogError(
744 '"{0}" does not exist.'.format(options.chromeos_root))
Han Shen91445322013-03-20 13:43:31 -0700745 return 1
746
Han Shend1c31b22016-04-18 14:58:57 -0700747 options.ndk_dir = os.path.expanduser(options.ndk_dir)
748 if not options.ndk_dir:
749 parser.error('Missing mandatory option "--ndk_dir".')
750 return 1
751
752 # Some tolerance regarding user input. We only need the ndk_root part, do not
753 # include toolchain/(gcc|binutils)/ part in this option.
Caroline Ticef6ef4392017-04-06 17:16:05 -0700754 options.ndk_dir = re.sub('/toolchain(/gcc|/binutils)?/?$', '',
755 options.ndk_dir)
Han Shend1c31b22016-04-18 14:58:57 -0700756
757 if not (os.path.isdir(options.ndk_dir) and
758 os.path.isdir(os.path.join(options.ndk_dir, 'toolchain'))):
759 logger.GetLogger().LogError(
760 '"toolchain" directory not found under "{0}".'.format(options.ndk_dir))
761 return 1
762
Han Shen91445322013-03-20 13:43:31 -0700763 if options.fixperm:
764 # Fix perm error before continuing.
Caroline Ticef6ef4392017-04-06 17:16:05 -0700765 cmd = (r'sudo find "{0}" \( -name ".cache" -type d -prune \) -o '
766 r'\( -name "chroot" -type d -prune \) -o '
767 r'\( -type f -exec chmod a+r {{}} \; \) -o '
768 r'\( -type d -exec chmod a+rx {{}} \; \)'
769 ).format(options.chromeos_root)
Han Shen91445322013-03-20 13:43:31 -0700770 logger.GetLogger().LogOutput(
Han Shen819f8622014-04-08 16:57:38 -0700771 'Fixing perm issues for chromeos root, this might take some time.')
Han Shen91445322013-03-20 13:43:31 -0700772 command_executer.GetCommandExecuter().RunCommand(cmd)
773
Han Shen819f8622014-04-08 16:57:38 -0700774 if options.reset_tool_ebuild_file:
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800775 if (options.gcc_dir or options.gcc_branch or options.binutils_dir or
776 options.binutils_branch):
Han Shen91445322013-03-20 13:43:31 -0700777 logger.GetLogger().LogWarning(
Han Shen819f8622014-04-08 16:57:38 -0700778 'Ignoring any "--gcc/binutils_dir" and/or "--gcc/binutils_branch".')
779 if options.setup_tool_ebuild_file_only:
780 logger.GetLogger().LogError(
781 ('Conflict options "--reset_tool_ebuild_file" '
782 'and "--setup_tool_ebuild_file_only".'))
Han Shen91445322013-03-20 13:43:31 -0700783 return 1
Han Shen819f8622014-04-08 16:57:38 -0700784 rv = Bootstrapper.ResetToolEbuildFile(options.chromeos_root, 'gcc')
785 rv1 = Bootstrapper.ResetToolEbuildFile(options.chromeos_root, 'binutils')
786 return 0 if (rv and rv1) else 1
Han Shen91445322013-03-20 13:43:31 -0700787
788 if options.gcc_dir:
789 options.gcc_dir = os.path.abspath(os.path.expanduser(options.gcc_dir))
790 if not os.path.isdir(options.gcc_dir):
Caroline Ticef6ef4392017-04-06 17:16:05 -0700791 logger.GetLogger().LogError(
792 '"{0}" does not exist.'.format(options.gcc_dir))
Han Shen91445322013-03-20 13:43:31 -0700793 return 1
794
Han Shen819f8622014-04-08 16:57:38 -0700795 if options.gcc_branch and options.gcc_dir:
796 parser.error('Only one of "--gcc_dir" and "--gcc_branch" can be specified.')
Han Shen91445322013-03-20 13:43:31 -0700797 return 1
Han Shen819f8622014-04-08 16:57:38 -0700798
799 if options.binutils_dir:
Caroline Ticef6ef4392017-04-06 17:16:05 -0700800 options.binutils_dir = os.path.abspath(
801 os.path.expanduser(options.binutils_dir))
Han Shen819f8622014-04-08 16:57:38 -0700802 if not os.path.isdir(options.binutils_dir):
Caroline Ticef6ef4392017-04-06 17:16:05 -0700803 logger.GetLogger().LogError(
804 '"{0}" does not exist.'.format(options.binutils_dir))
Han Shen819f8622014-04-08 16:57:38 -0700805 return 1
806
807 if options.binutils_branch and options.binutils_dir:
808 parser.error('Only one of "--binutils_dir" and '
809 '"--binutils_branch" can be specified.')
810 return 1
811
Caroline Ticef6ef4392017-04-06 17:16:05 -0700812 if (not (options.binutils_branch or options.binutils_dir or
813 options.gcc_branch or options.gcc_dir)):
Han Shen819f8622014-04-08 16:57:38 -0700814 parser.error(('At least one of "--gcc_dir", "--gcc_branch", '
815 '"--binutils_dir" and "--binutils_branch" must '
816 'be specified.'))
817 return 1
818
819 if not options.board and not options.bootstrap:
820 parser.error('You must specify either "--board" or "--bootstrap".')
821 return 1
822
Han Shen03d30982014-06-12 11:22:29 -0700823 if (options.board and options.bootstrap and
824 not options.setup_tool_ebuild_file_only):
Han Shen819f8622014-04-08 16:57:38 -0700825 parser.error('You must specify only one of "--board" and "--bootstrap".')
Han Shen91445322013-03-20 13:43:31 -0700826 return 1
827
Han Shen03d30982014-06-12 11:22:29 -0700828 if not options.bootstrap and options.disable_2nd_bootstrap:
829 parser.error('"--disable-2nd-bootstrap" has no effect '
830 'without specifying "--bootstrap".')
831 return 1
832
Han Shen91445322013-03-20 13:43:31 -0700833 if Bootstrapper(
Han Shen819f8622014-04-08 16:57:38 -0700834 options.chromeos_root,
Han Shend1c31b22016-04-18 14:58:57 -0700835 options.ndk_dir,
Luis Lozanof2a3ef42015-12-15 13:49:30 -0800836 gcc_branch=options.gcc_branch,
837 gcc_dir=options.gcc_dir,
Han Shen819f8622014-04-08 16:57:38 -0700838 binutils_branch=options.binutils_branch,
839 binutils_dir=options.binutils_dir,
840 board=options.board,
Han Shen03d30982014-06-12 11:22:29 -0700841 disable_2nd_bootstrap=options.disable_2nd_bootstrap,
Han Shen819f8622014-04-08 16:57:38 -0700842 setup_tool_ebuild_file_only=options.setup_tool_ebuild_file_only).Do():
Han Shen91445322013-03-20 13:43:31 -0700843 return 0
844 return 1
845
846
847if __name__ == '__main__':
Caroline Tice88272d42016-01-13 09:48:29 -0800848 retval = Main(sys.argv[1:])
Han Shen91445322013-03-20 13:43:31 -0700849 sys.exit(retval)