blob: 3c9a78036df44335beb287d04b0f5db1ec494e3c [file] [log] [blame]
Don Garrettc4114cc2016-11-01 20:04:06 -07001# Copyright 2016 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Bootstrap for cbuildbot.
6
7This script is intended to checkout chromite on the branch specified by -b or
8--branch (as normally accepted by cbuildbot), and then invoke cbuildbot. Most
9arguments are not parsed, only passed along. If a branch is not specified, this
10script will use 'master'.
11
12Among other things, this allows us to invoke build configs that exist on a given
13branch, but not on TOT.
14"""
15
16from __future__ import print_function
17
Don Garrett125d4dc2017-04-25 16:26:03 -070018import functools
Don Garrettc4114cc2016-11-01 20:04:06 -070019import os
20
21from chromite.cbuildbot import repository
Don Garrett597ddff2017-02-17 18:29:37 -080022from chromite.cbuildbot.stages import sync_stages
Don Garrett86881cb2017-02-15 15:41:55 -080023from chromite.lib import config_lib
Don Garrettc4114cc2016-11-01 20:04:06 -070024from chromite.lib import cros_build_lib
25from chromite.lib import cros_logging as logging
Don Garrettacbb2392017-05-11 18:27:41 -070026from chromite.lib import metrics
Don Garrettc4114cc2016-11-01 20:04:06 -070027from chromite.lib import osutils
Don Garrettacbb2392017-05-11 18:27:41 -070028from chromite.lib import ts_mon_config
Don Garrett86881cb2017-02-15 15:41:55 -080029from chromite.scripts import cbuildbot
Don Garrettc4114cc2016-11-01 20:04:06 -070030
Don Garrett597ddff2017-02-17 18:29:37 -080031
Don Garrett60967922017-04-12 18:51:44 -070032# This number should be incremented when we change the layout of the buildroot
33# in a non-backwards compatible way. This wipes all buildroots.
Don Garrettbf90cdf2017-05-19 15:54:02 -070034BUILDROOT_BUILDROOT_LAYOUT = 2
Don Garrett60967922017-04-12 18:51:44 -070035
Don Garrettacbb2392017-05-11 18:27:41 -070036# Metrics reported to Monarch.
Don Garrett45e77412017-06-14 16:57:55 -070037METRIC_ACTIVE = 'chromeos/chromite/cbuildbot_launch/active'
Don Garrettacbb2392017-05-11 18:27:41 -070038METRIC_INVOKED = 'chromeos/chromite/cbuildbot_launch/invoked'
39METRIC_COMPLETED = 'chromeos/chromite/cbuildbot_launch/completed'
40METRIC_PREP = 'chromeos/chromite/cbuildbot_launch/prep_completed'
41METRIC_CLEAN = 'chromeos/chromite/cbuildbot_launch/clean_buildroot_durations'
42METRIC_INITIAL = 'chromeos/chromite/cbuildbot_launch/initial_checkout_durations'
43METRIC_CBUILDBOT = 'chromeos/chromite/cbuildbot_launch/cbuildbot_durations'
44METRIC_CLOBBER = 'chromeos/chromite/cbuildbot_launch/clobber'
45METRIC_BRANCH_CLEANUP = 'chromeos/chromite/cbuildbot_launch/branch_cleanup'
46
Don Garrett60967922017-04-12 18:51:44 -070047
Don Garrett125d4dc2017-04-25 16:26:03 -070048def StageDecorator(functor):
49 """A Decorator that adds buildbot stage tags around a method.
50
Don Garrettacbb2392017-05-11 18:27:41 -070051 It uses the method name as the stage name, and assumes failure on a true
52 return value, or an exception.
Don Garrett125d4dc2017-04-25 16:26:03 -070053 """
54 @functools.wraps(functor)
55 def wrapped_functor(*args, **kwargs):
56 try:
57 logging.PrintBuildbotStepName(functor.__name__)
Don Garrettacbb2392017-05-11 18:27:41 -070058 result = functor(*args, **kwargs)
Don Garrett125d4dc2017-04-25 16:26:03 -070059 except Exception:
60 logging.PrintBuildbotStepFailure()
61 raise
62
Don Garrettacbb2392017-05-11 18:27:41 -070063 if result:
64 logging.PrintBuildbotStepFailure()
65 return result
66
Don Garrett125d4dc2017-04-25 16:26:03 -070067 return wrapped_functor
68
69
Don Garrettacbb2392017-05-11 18:27:41 -070070def field(fields, **kwargs):
71 """Helper for inserting more fields into a metrics fields dictionary.
72
73 Args:
74 fields: Dictionary of metrics fields.
75 kwargs: Each argument is a key/value pair to insert into dict.
76
77 Returns:
78 Copy of original dictionary with kwargs set as fields.
79 """
80 f = fields.copy()
81 f.update(kwargs)
82 return f
83
Don Garrett86881cb2017-02-15 15:41:55 -080084def PreParseArguments(argv):
Don Garrettc4114cc2016-11-01 20:04:06 -070085 """Extract the branch name from cbuildbot command line arguments.
86
Don Garrettc4114cc2016-11-01 20:04:06 -070087 Args:
88 argv: The command line arguments to parse.
89
90 Returns:
91 Branch as a string ('master' if nothing is specified).
92 """
Don Garrett86881cb2017-02-15 15:41:55 -080093 parser = cbuildbot.CreateParser()
Mike Frysinger80bba8a2017-08-18 15:28:36 -040094 options = cbuildbot.ParseCommandLine(parser, argv)
Don Garrettd1d90dd2017-06-13 17:35:52 -070095 options.Freeze()
Don Garrett86881cb2017-02-15 15:41:55 -080096
97 # This option isn't required for cbuildbot, but is for us.
98 if not options.buildroot:
99 cros_build_lib.Die('--buildroot is a required option.')
100
Don Garrettd1d90dd2017-06-13 17:35:52 -0700101 if len(options.build_targets) != 1:
102 cros_build_lib.Die('Exactly one build target required, got: %s',
103 ', '.join(options.build_targets) or 'None')
Don Garrett597ddff2017-02-17 18:29:37 -0800104
Don Garrett86881cb2017-02-15 15:41:55 -0800105 return options
Don Garrettc4114cc2016-11-01 20:04:06 -0700106
107
Don Garrettbf90cdf2017-05-19 15:54:02 -0700108def GetState(root):
109 """Fetch the current state of our working directory.
110
111 Will return with a default result if there is no known state.
112
113 Args:
114 root: Root of the working directory tree as a string.
115
116 Returns:
117 Layout version as an integer (0 for unknown).
118 Previous branch as a string ('' for unknown).
119 """
120 state_file = os.path.join(root, '.cbuildbot_launch_state')
Don Garrett60967922017-04-12 18:51:44 -0700121
122 try:
123 state = osutils.ReadFile(state_file)
124 buildroot_layout, branchname = state.split()
125 buildroot_layout = int(buildroot_layout)
126 return buildroot_layout, branchname
127 except (IOError, ValueError):
128 # If we are unable to either read or parse the state file, we get here.
129 return 0, ''
130
131
Don Garrettbf90cdf2017-05-19 15:54:02 -0700132def SetState(branchname, root):
133 """Save the current state of our working directory.
134
135 Args:
136 branchname: Name of branch we prepped for as a string.
137 root: Root of the working directory tree as a string.
138 """
Don Garrett60967922017-04-12 18:51:44 -0700139 assert branchname
Don Garrettbf90cdf2017-05-19 15:54:02 -0700140 state_file = os.path.join(root, '.cbuildbot_launch_state')
Don Garrett60967922017-04-12 18:51:44 -0700141 new_state = '%d %s' % (BUILDROOT_BUILDROOT_LAYOUT, branchname)
142 osutils.WriteFile(state_file, new_state)
143
144
Don Garrett125d4dc2017-04-25 16:26:03 -0700145@StageDecorator
Don Garrettbf90cdf2017-05-19 15:54:02 -0700146def CleanBuildRoot(root, repo, metrics_fields):
Don Garrett7ade05a2017-02-17 13:31:47 -0800147 """Some kinds of branch transitions break builds.
148
Don Garrettbf90cdf2017-05-19 15:54:02 -0700149 This method ensures that cbuildbot's buildroot is a clean checkout on the
150 given branch when it starts. If necessary (a branch transition) it will wipe
151 assorted state that cannot be safely reused from the previous build.
Don Garrett7ade05a2017-02-17 13:31:47 -0800152
Don Garrett7ade05a2017-02-17 13:31:47 -0800153 Args:
Don Garrettbf90cdf2017-05-19 15:54:02 -0700154 root: Root directory owned by cbuildbot_launch.
Don Garrettf324bc32017-05-23 14:00:53 -0700155 repo: repository.RepoRepository instance.
Don Garrettacbb2392017-05-11 18:27:41 -0700156 metrics_fields: Dictionary of fields to include in metrics.
Don Garrett7ade05a2017-02-17 13:31:47 -0800157 """
Don Garrettbf90cdf2017-05-19 15:54:02 -0700158 old_buildroot_layout, old_branch = GetState(root)
Don Garrette17e1d92017-04-12 15:28:19 -0700159
Don Garrett60967922017-04-12 18:51:44 -0700160 if old_buildroot_layout != BUILDROOT_BUILDROOT_LAYOUT:
Don Garrett125d4dc2017-04-25 16:26:03 -0700161 logging.PrintBuildbotStepText('Unknown layout: Wiping buildroot.')
Don Garrettacbb2392017-05-11 18:27:41 -0700162 metrics.Counter(METRIC_CLOBBER).increment(
163 field(metrics_fields, reason='layout_change'))
Don Garrettbf90cdf2017-05-19 15:54:02 -0700164 osutils.RmDir(root, ignore_missing=True, sudo=True)
Don Garrettf324bc32017-05-23 14:00:53 -0700165 else:
166 if old_branch != repo.branch:
167 logging.PrintBuildbotStepText('Branch change: Cleaning buildroot.')
168 logging.info('Unmatched branch: %s -> %s', old_branch, repo.branch)
Don Garrettacbb2392017-05-11 18:27:41 -0700169 metrics.Counter(METRIC_BRANCH_CLEANUP).increment(
170 field(metrics_fields, old_branch=old_branch))
Don Garrett39963602017-02-27 14:41:58 -0800171
Don Garrettf324bc32017-05-23 14:00:53 -0700172 logging.info('Remove Chroot.')
Don Garrettbf90cdf2017-05-19 15:54:02 -0700173 osutils.RmDir(os.path.join(repo.directory, 'chroot'),
Don Garrettf324bc32017-05-23 14:00:53 -0700174 ignore_missing=True, sudo=True)
Don Garrett7ade05a2017-02-17 13:31:47 -0800175
Don Garrettf324bc32017-05-23 14:00:53 -0700176 logging.info('Remove Chrome checkout.')
Don Garrettbf90cdf2017-05-19 15:54:02 -0700177 osutils.RmDir(os.path.join(repo.directory, '.cache', 'distfiles'),
Don Garrettf324bc32017-05-23 14:00:53 -0700178 ignore_missing=True, sudo=True)
179
180 try:
181 # If there is any failure doing the cleanup, wipe everything.
182 repo.BuildRootGitCleanup(prune_all=True)
183 except Exception:
184 logging.info('Checkout cleanup failed, wiping buildroot:', exc_info=True)
Don Garrettacbb2392017-05-11 18:27:41 -0700185 metrics.Counter(METRIC_CLOBBER).increment(
186 field(metrics_fields, reason='repo_cleanup_failure'))
Don Garrettbf90cdf2017-05-19 15:54:02 -0700187 repository.ClearBuildRoot(repo.directory)
Don Garrett39963602017-02-27 14:41:58 -0800188
Don Garrettbf90cdf2017-05-19 15:54:02 -0700189 # Ensure buildroot exists. Save the state we are prepped for.
190 osutils.SafeMakedirs(repo.directory)
191 SetState(repo.branch, root)
Don Garrett7ade05a2017-02-17 13:31:47 -0800192
193
Don Garrett125d4dc2017-04-25 16:26:03 -0700194@StageDecorator
Don Garrettf324bc32017-05-23 14:00:53 -0700195def InitialCheckout(repo):
Don Garrett86881cb2017-02-15 15:41:55 -0800196 """Preliminary ChromeOS checkout.
197
198 Perform a complete checkout of ChromeOS on the specified branch. This does NOT
199 match what the build needs, but ensures the buildroot both has a 'hot'
200 checkout, and is close enough that the branched cbuildbot can successfully get
201 the right checkout.
202
203 This checks out full ChromeOS, even if a ChromiumOS build is going to be
204 performed. This is because we have no knowledge of the build config to be
205 used.
Don Garrettc4114cc2016-11-01 20:04:06 -0700206
207 Args:
Don Garrettf324bc32017-05-23 14:00:53 -0700208 repo: repository.RepoRepository instance.
Don Garrettc4114cc2016-11-01 20:04:06 -0700209 """
Don Garrettf324bc32017-05-23 14:00:53 -0700210 logging.PrintBuildbotStepText('Branch: %s' % repo.branch)
Don Garrett7ade05a2017-02-17 13:31:47 -0800211 logging.info('Bootstrap script starting initial sync on branch: %s',
Don Garrettf324bc32017-05-23 14:00:53 -0700212 repo.branch)
Don Garrett76496912017-05-11 16:59:11 -0700213 repo.Sync(detach=True)
Don Garrettc4114cc2016-11-01 20:04:06 -0700214
215
Don Garrett125d4dc2017-04-25 16:26:03 -0700216@StageDecorator
Don Garrettd1d90dd2017-06-13 17:35:52 -0700217def RunCbuildbot(buildroot, argv):
Don Garrettc4114cc2016-11-01 20:04:06 -0700218 """Start cbuildbot in specified directory with all arguments.
219
220 Args:
Don Garrettbf90cdf2017-05-19 15:54:02 -0700221 buildroot: Directory to be passed to cbuildbot with --buildroot.
Don Garrettd1d90dd2017-06-13 17:35:52 -0700222 argv: Command line options passed to cbuildbot_launch.
Don Garrettc4114cc2016-11-01 20:04:06 -0700223
224 Returns:
225 Return code of cbuildbot as an integer.
226 """
Don Garrettbf90cdf2017-05-19 15:54:02 -0700227 logging.info('Bootstrap cbuildbot in: %s', buildroot)
Don Garrettbf90cdf2017-05-19 15:54:02 -0700228
Don Garrettd1d90dd2017-06-13 17:35:52 -0700229 # Fixup buildroot parameter.
230 argv = argv[:]
231 for i in xrange(len(argv)):
232 if argv[i] in ('-r', '--buildroot'):
233 argv[i+1] = buildroot
Don Garrett597ddff2017-02-17 18:29:37 -0800234
Don Garrettd1d90dd2017-06-13 17:35:52 -0700235 # This filters out command line arguments not supported by older versions
236 # of cbuildbot.
237 parser = cbuildbot.CreateParser()
Mike Frysinger80bba8a2017-08-18 15:28:36 -0400238 options = cbuildbot.ParseCommandLine(parser, argv)
Don Garrettd1d90dd2017-06-13 17:35:52 -0700239 cbuildbot_path = os.path.join(buildroot, 'chromite', 'bin', 'cbuildbot')
Don Garrett597ddff2017-02-17 18:29:37 -0800240 cmd = sync_stages.BootstrapStage.FilterArgsForTargetCbuildbot(
Don Garrettbf90cdf2017-05-19 15:54:02 -0700241 buildroot, cbuildbot_path, options)
Don Garrett597ddff2017-02-17 18:29:37 -0800242
Don Garrettd1d90dd2017-06-13 17:35:52 -0700243 # Actually run cbuildbot with the fixed up command line options.
244 result = cros_build_lib.RunCommand(cmd, error_code_ok=True, cwd=buildroot)
Don Garrettacbb2392017-05-11 18:27:41 -0700245 return result.returncode
Don Garrettc4114cc2016-11-01 20:04:06 -0700246
Don Garrett60967922017-04-12 18:51:44 -0700247
Don Garrettf15d65b2017-04-12 12:39:55 -0700248def ConfigureGlobalEnvironment():
249 """Setup process wide environmental changes."""
Don Garrettf15d65b2017-04-12 12:39:55 -0700250 # Set umask to 022 so files created by buildbot are readable.
251 os.umask(0o22)
252
Don Garrett86fec482017-05-17 18:13:33 -0700253 # These variables can interfere with LANG / locale behavior.
254 unwanted_local_vars = [
255 'LC_ALL', 'LC_CTYPE', 'LC_COLLATE', 'LC_TIME', 'LC_NUMERIC',
256 'LC_MONETARY', 'LC_MESSAGES', 'LC_PAPER', 'LC_NAME', 'LC_ADDRESS',
257 'LC_TELEPHONE', 'LC_MEASUREMENT', 'LC_IDENTIFICATION',
258 ]
259 for v in unwanted_local_vars:
260 os.environ.pop(v, None)
261
262 # This variable is required for repo sync's to work in all cases.
263 os.environ['LANG'] = 'en_US.UTF-8'
264
Don Garrettc4114cc2016-11-01 20:04:06 -0700265
Don Garrettacbb2392017-05-11 18:27:41 -0700266def _main(argv):
Don Garrettc4114cc2016-11-01 20:04:06 -0700267 """main method of script.
268
269 Args:
270 argv: All command line arguments to pass as list of strings.
271
272 Returns:
273 Return code of cbuildbot as an integer.
274 """
Don Garrettd1d90dd2017-06-13 17:35:52 -0700275 options = PreParseArguments(argv)
276
277 branchname = options.branch or 'master'
278 root = options.buildroot
279 buildroot = os.path.join(root, 'repository')
280 build_config = options.build_targets[0]
281
282 metrics_fields = {
283 'branch_name': branchname,
284 'build_config': build_config,
285 'tryjob': options.remote_trybot,
286 }
Don Garrettf15d65b2017-04-12 12:39:55 -0700287
Don Garrettacbb2392017-05-11 18:27:41 -0700288 # Does the entire build pass or fail.
Don Garrett45e77412017-06-14 16:57:55 -0700289 with metrics.Presence(METRIC_ACTIVE, metrics_fields), \
290 metrics.SuccessCounter(METRIC_COMPLETED, metrics_fields) as s_fields:
Don Garrettc4114cc2016-11-01 20:04:06 -0700291
Don Garrettacbb2392017-05-11 18:27:41 -0700292 # Preliminary set, mostly command line parsing.
Don Garrettd1d90dd2017-06-13 17:35:52 -0700293 with metrics.SuccessCounter(METRIC_INVOKED, metrics_fields):
Don Garrettacbb2392017-05-11 18:27:41 -0700294 logging.EnableBuildbotMarkers()
295 ConfigureGlobalEnvironment()
Don Garrett86881cb2017-02-15 15:41:55 -0800296
Don Garrettacbb2392017-05-11 18:27:41 -0700297 # Prepare the buildroot with source for the build.
298 with metrics.SuccessCounter(METRIC_PREP, metrics_fields):
299 site_config = config_lib.GetConfig()
300 manifest_url = site_config.params['MANIFEST_INT_URL']
301 repo = repository.RepoRepository(manifest_url, buildroot,
302 branch=branchname,
Don Garrettd1d90dd2017-06-13 17:35:52 -0700303 git_cache_dir=options.git_cache_dir)
Don Garrett86881cb2017-02-15 15:41:55 -0800304
Don Garrettacbb2392017-05-11 18:27:41 -0700305 # Clean up the buildroot to a safe state.
306 with metrics.SecondsTimer(METRIC_CLEAN, fields=metrics_fields):
Don Garrettbf90cdf2017-05-19 15:54:02 -0700307 CleanBuildRoot(root, repo, metrics_fields)
Don Garrettacbb2392017-05-11 18:27:41 -0700308
309 # Get a checkout close enough the branched cbuildbot can handle it.
310 with metrics.SecondsTimer(METRIC_INITIAL, fields=metrics_fields):
311 InitialCheckout(repo)
312
313 # Run cbuildbot inside the full ChromeOS checkout, on the specified branch.
314 with metrics.SecondsTimer(METRIC_CBUILDBOT, fields=metrics_fields):
Don Garrettd1d90dd2017-06-13 17:35:52 -0700315 result = RunCbuildbot(buildroot, argv)
316 s_fields['success'] = (result == 0)
Don Garrettacbb2392017-05-11 18:27:41 -0700317 return result
318
319
320def main(argv):
321 # Enable Monarch metrics gathering.
322 with ts_mon_config.SetupTsMonGlobalState('cbuildbot_launch', indirect=True):
323 return _main(argv)