blob: d7c0e46482cbc8470ace3d8c510cb97630181420 [file] [log] [blame]
David Rileyc0da9d92016-02-01 12:11:01 -08001# Copyright 2016 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""This module uprevs Android for cbuildbot.
6
7After calling, it prints outs ANDROID_VERSION_ATOM=(version atom string). A
8caller could then use this atom with emerge to build the newly uprevved version
9of Android e.g.
10
11./cros_mark_android_as_stable
12Returns chromeos-base/android-container-2559197
13
14emerge-veyron_minnie-cheets =chromeos-base/android-container-2559197-r1
15"""
16
17from __future__ import print_function
18
19import filecmp
20import glob
21import os
Hidehiko Abe12727dd2016-05-27 23:23:45 +090022import re
David Rileyc0da9d92016-02-01 12:11:01 -080023
24from chromite.cbuildbot import constants
25from chromite.lib import commandline
26from chromite.lib import cros_build_lib
27from chromite.lib import cros_logging as logging
28from chromite.lib import git
29from chromite.lib import gs
30from chromite.lib import portage_util
31from chromite.scripts import cros_mark_as_stable
32
33
34# Dir where all the action happens.
35_OVERLAY_DIR = '%(srcroot)s/private-overlays/project-cheets-private/'
36
37_GIT_COMMIT_MESSAGE = ('Marking latest for %(android_pn)s ebuild '
38 'with version %(android_version)s as stable.')
39
40# URLs that print lists of Android revisions between two build ids.
41_ANDROID_VERSION_URL = ('http://android-build-uber.corp.google.com/repo.html?'
42 'last_bid=%(old)s&bid=%(new)s&branch=%(branch)s')
43
44
45def IsBuildIdValid(bucket_url, build_branch, build_id):
46 """Checks that a specific build_id is valid.
47
48 Looks for that build_id for all builds. Confirms that the subpath can
49 be found and that the zip file is present in that subdirectory.
50
51 Args:
52 bucket_url: URL of Android build gs bucket
53 build_branch: branch of Android builds
54 build_id: A string. The Android build id number to check.
55
56 Returns:
57 Returns subpaths dictionary if build_id is valid.
58 None if the build_id is not valid.
59 """
60 gs_context = gs.GSContext()
61 subpaths_dict = {}
Hidehiko Abe12727dd2016-05-27 23:23:45 +090062 for build, (target, _) in constants.ANDROID_BUILD_TARGETS.iteritems():
David Rileyc0da9d92016-02-01 12:11:01 -080063 build_dir = '%s-%s' % (build_branch, target)
64 build_id_path = os.path.join(bucket_url, build_dir, build_id)
65
66 # Find name of subpath.
67 try:
68 subpaths = gs_context.List(build_id_path)
69 except gs.GSNoSuchKey:
70 logging.warn(
71 'Directory [%s] does not contain any subpath, ignoring it.',
72 build_id_path)
73 return None
74 if len(subpaths) > 1:
75 logging.warn(
76 'Directory [%s] contains more than one subpath, ignoring it.',
77 build_id_path)
78 return None
79
80 subpath_dir = subpaths[0].url.rstrip('/')
81 subpath_name = os.path.basename(subpath_dir)
82
83 # Look for a zipfile ending in the build_id number.
84 try:
Hidehiko Abe12727dd2016-05-27 23:23:45 +090085 gs_context.List(subpath_dir)
David Rileyc0da9d92016-02-01 12:11:01 -080086 except gs.GSNoSuchKey:
87 logging.warn(
Hidehiko Abe12727dd2016-05-27 23:23:45 +090088 'Did not find a file for build id [%s] in directory [%s].',
David Rileyc0da9d92016-02-01 12:11:01 -080089 build_id, subpath_dir)
90 return None
91
92 # Record subpath for the build.
93 subpaths_dict[build] = subpath_name
94
95 # If we got here, it means we found an appropriate build for all platforms.
96 return subpaths_dict
97
98
99def GetLatestBuild(bucket_url, build_branch):
100 """Searches the gs bucket for the latest green build.
101
102 Args:
103 bucket_url: URL of Android build gs bucket
104 build_branch: branch of Android builds
105
106 Returns:
107 Tuple of (latest version string, subpaths dictionary)
108 If no latest build can be found, returns None, None
109 """
110 gs_context = gs.GSContext()
111 common_build_ids = None
112 # Find builds for each target.
Hidehiko Abe12727dd2016-05-27 23:23:45 +0900113 for target, _ in constants.ANDROID_BUILD_TARGETS.itervalues():
David Rileyc0da9d92016-02-01 12:11:01 -0800114 build_dir = '-'.join((build_branch, target))
115 base_path = os.path.join(bucket_url, build_dir)
116 build_ids = []
117 for gs_result in gs_context.List(base_path):
118 # Remove trailing slashes and get the base name, which is the build_id.
119 build_id = os.path.basename(gs_result.url.rstrip('/'))
120 if not build_id.isdigit():
121 logging.warn('Directory [%s] does not look like a valid build_id.',
122 gs_result.url)
123 continue
124 build_ids.append(build_id)
125
126 # Update current list of builds.
127 if common_build_ids is None:
128 # First run, populate it with the first platform.
129 common_build_ids = set(build_ids)
130 else:
131 # Already populated, find the ones that are common.
132 common_build_ids.intersection_update(build_ids)
133
134 if common_build_ids is None:
135 logging.warn('Did not find a build_id common to all platforms.')
136 return None, None
137
138 # Otherwise, find the most recent one that is valid.
139 for build_id in sorted(common_build_ids, key=int, reverse=True):
140 subpaths = IsBuildIdValid(bucket_url, build_branch, build_id)
141 if subpaths:
142 return build_id, subpaths
143
144 # If not found, no build_id is valid.
145 logging.warn('Did not find a build_id valid on all platforms.')
146 return None, None
147
148
149def FindAndroidCandidates(package_dir):
150 """Return a tuple of Android's unstable ebuild and stable ebuilds.
151
152 Args:
153 package_dir: The path to where the package ebuild is stored.
154
155 Returns:
156 Tuple [unstable_ebuild, stable_ebuilds].
157
158 Raises:
159 Exception: if no unstable ebuild exists for Android.
160 """
161 stable_ebuilds = []
162 unstable_ebuilds = []
163 for path in glob.glob(os.path.join(package_dir, '*.ebuild')):
164 ebuild = portage_util.EBuild(path)
165 if ebuild.version == '9999':
166 unstable_ebuilds.append(ebuild)
167 else:
168 stable_ebuilds.append(ebuild)
169
170 # Apply some sanity checks.
171 if not unstable_ebuilds:
172 raise Exception('Missing 9999 ebuild for %s' % package_dir)
173 if not stable_ebuilds:
174 logging.warning('Missing stable ebuild for %s' % package_dir)
175
176 return portage_util.BestEBuild(unstable_ebuilds), stable_ebuilds
177
178
David Riley73f00d92016-02-16 18:54:20 -0800179def CopyToArcBucket(android_bucket_url, build_branch, build_id, subpaths,
180 arc_bucket_url, acls):
181 """Copies from source Android bucket to ARC++ specific bucket.
182
183 Copies each build to the ARC bucket eliminating the subpath.
184 Applies build specific ACLs for each file.
185
186 Args:
187 android_bucket_url: URL of Android build gs bucket
188 build_branch: branch of Android builds
189 build_id: A string. The Android build id number to check.
190 subpaths: Subpath dictionary for each build to copy.
191 arc_bucket_url: URL of the target ARC build gs bucket
192 acls: ACLs dictionary for each build to copy.
193 """
194 gs_context = gs.GSContext()
195 for build, subpath in subpaths.iteritems():
Hidehiko Abe12727dd2016-05-27 23:23:45 +0900196 target, pattern = constants.ANDROID_BUILD_TARGETS[build]
David Riley73f00d92016-02-16 18:54:20 -0800197 build_dir = '%s-%s' % (build_branch, target)
198 android_dir = os.path.join(android_bucket_url, build_dir, build_id, subpath)
199 arc_dir = os.path.join(arc_bucket_url, build_dir, build_id)
200
Hidehiko Abe12727dd2016-05-27 23:23:45 +0900201 # Copy all target files from android_dir to arc_dir, setting ACLs.
202 for targetfile in gs_context.List(android_dir):
203 if re.search(pattern, targetfile.url):
204 basename = os.path.basename(targetfile.url)
205 arc_path = os.path.join(arc_dir, basename)
David Riley73f00d92016-02-16 18:54:20 -0800206 acl = acls[build]
207 needs_copy = True
208
209 # Check a pre-existing file with the original source.
210 if gs_context.Exists(arc_path):
Hidehiko Abe12727dd2016-05-27 23:23:45 +0900211 if (gs_context.Stat(targetfile.url).hash_crc32c !=
Elijah Taylorbfc30692016-04-22 14:05:23 -0700212 gs_context.Stat(arc_path).hash_crc32c):
David Riley73f00d92016-02-16 18:54:20 -0800213 logging.warn('Removing incorrect file %s', arc_path)
214 gs_context.Remove(arc_path)
215 else:
216 logging.info('Skipping already copied file %s', arc_path)
217 needs_copy = False
218
219 # Copy if necessary, and set the ACL unconditionally.
220 # The Stat() call above doesn't verify the ACL is correct and
221 # the ChangeACL should be relatively cheap compared to the copy.
222 # This covers the following caes:
223 # - handling an interrupted copy from a previous run.
224 # - rerunning the copy in case one of the googlestorage_acl_X.txt
225 # files changes (e.g. we add a new variant which reuses a build).
226 if needs_copy:
Hidehiko Abe12727dd2016-05-27 23:23:45 +0900227 logging.info('Copying %s -> %s (acl %s)',
228 targetfile.url, arc_path, acl)
229 gs_context.Copy(targetfile.url, arc_path, version=0)
David Riley73f00d92016-02-16 18:54:20 -0800230 gs_context.ChangeACL(arc_path, acl_args_file=acl)
231
232
233def MakeAclDict(package_dir):
234 """Creates a dictionary of acl files for each build type.
235
236 Args:
237 package_dir: The path to where the package acl files are stored.
238
239 Returns:
240 Returns acls dictionary.
241 """
242 return dict(
243 (k, os.path.join(package_dir, v))
244 for k, v in constants.ARC_BUCKET_ACLS.items()
245 )
246
247
David Rileyc0da9d92016-02-01 12:11:01 -0800248def GetAndroidRevisionListLink(build_branch, old_android, new_android):
249 """Returns a link to the list of revisions between two Android versions
250
251 Given two AndroidEBuilds, generate a link to a page that prints the
252 Android changes between those two revisions, inclusive.
253
254 Args:
255 build_branch: branch of Android builds
256 old_android: ebuild for the version to diff from
257 new_android: ebuild for the version to which to diff
258
259 Returns:
260 The desired URL.
261 """
262 return _ANDROID_VERSION_URL % {'branch': build_branch,
263 'old': old_android.version,
264 'new': new_android.version}
265
266
267def MarkAndroidEBuildAsStable(stable_candidate, unstable_ebuild, android_pn,
David Riley73f00d92016-02-16 18:54:20 -0800268 android_version, package_dir, build_branch,
269 arc_bucket_url):
David Rileyc0da9d92016-02-01 12:11:01 -0800270 r"""Uprevs the Android ebuild.
271
272 This is the main function that uprevs from a stable candidate
273 to its new version.
274
275 Args:
276 stable_candidate: ebuild that corresponds to the stable ebuild we are
277 revving from. If None, builds the a new ebuild given the version
278 with revision set to 1.
279 unstable_ebuild: ebuild corresponding to the unstable ebuild for Android.
280 android_pn: package name.
281 android_version: The \d+ build id of Android.
David Rileyc0da9d92016-02-01 12:11:01 -0800282 package_dir: Path to the android-container package dir.
David Riley73f00d92016-02-16 18:54:20 -0800283 build_branch: branch of Android builds.
284 arc_bucket_url: URL of the target ARC build gs bucket.
David Rileyc0da9d92016-02-01 12:11:01 -0800285
286 Returns:
287 Full portage version atom (including rc's, etc) that was revved.
288 """
289 def IsTheNewEBuildRedundant(new_ebuild, stable_ebuild):
290 """Returns True if the new ebuild is redundant.
291
292 This is True if there if the current stable ebuild is the exact same copy
293 of the new one.
294 """
295 if not stable_ebuild:
296 return False
297
David Riley676f5402016-02-12 17:24:23 -0800298 if stable_candidate.version_no_rev == new_ebuild.version_no_rev:
David Rileyc0da9d92016-02-01 12:11:01 -0800299 return filecmp.cmp(
300 new_ebuild.ebuild_path, stable_ebuild.ebuild_path, shallow=False)
301
302 # Case where we have the last stable candidate with same version just rev.
David Riley676f5402016-02-12 17:24:23 -0800303 if stable_candidate and stable_candidate.version_no_rev == android_version:
David Rileyc0da9d92016-02-01 12:11:01 -0800304 new_ebuild_path = '%s-r%d.ebuild' % (
305 stable_candidate.ebuild_path_no_revision,
306 stable_candidate.current_revision + 1)
307 else:
308 pf = '%s-%s-r1' % (android_pn, android_version)
309 new_ebuild_path = os.path.join(package_dir, '%s.ebuild' % pf)
310
David Riley73f00d92016-02-16 18:54:20 -0800311 variables = {'BASE_URL': arc_bucket_url}
Hidehiko Abe12727dd2016-05-27 23:23:45 +0900312 for build, (target, _) in constants.ANDROID_BUILD_TARGETS.iteritems():
David Riley73f00d92016-02-16 18:54:20 -0800313 variables[build + '_TARGET'] = '%s-%s' % (build_branch, target)
David Rileyc0da9d92016-02-01 12:11:01 -0800314
315 portage_util.EBuild.MarkAsStable(
316 unstable_ebuild.ebuild_path, new_ebuild_path,
317 variables, make_stable=True)
318 new_ebuild = portage_util.EBuild(new_ebuild_path)
319
320 # Determine whether this is ebuild is redundant.
321 if IsTheNewEBuildRedundant(new_ebuild, stable_candidate):
322 msg = 'Previous ebuild with same version found and ebuild is redundant.'
323 logging.info(msg)
324 os.unlink(new_ebuild_path)
325 return None
326
327 if stable_candidate:
328 logging.PrintBuildbotLink('Android revisions',
329 GetAndroidRevisionListLink(build_branch,
330 stable_candidate,
331 new_ebuild))
332
333 git.RunGit(package_dir, ['add', new_ebuild_path])
334 if stable_candidate and not stable_candidate.IsSticky():
335 git.RunGit(package_dir, ['rm', stable_candidate.ebuild_path])
336
337 # Update ebuild manifest and git add it.
338 gen_manifest_cmd = ['ebuild', new_ebuild_path, 'manifest', '--force']
339 cros_build_lib.RunCommand(gen_manifest_cmd,
340 extra_env=None, print_cmd=True)
341 git.RunGit(package_dir, ['add', 'Manifest'])
342
343 portage_util.EBuild.CommitChange(
344 _GIT_COMMIT_MESSAGE % {'android_pn': android_pn,
345 'android_version': android_version},
346 package_dir)
347
348 return '%s-%s' % (new_ebuild.package, new_ebuild.version)
349
350
351def GetParser():
352 """Creates the argument parser."""
353 parser = commandline.ArgumentParser()
354 parser.add_argument('-b', '--boards')
355 parser.add_argument('--android_bucket_url',
David Riley73f00d92016-02-16 18:54:20 -0800356 default=constants.ANDROID_BUCKET_URL,
357 type='gs_path')
David Rileyc0da9d92016-02-01 12:11:01 -0800358 parser.add_argument('--android_build_branch',
359 default=constants.ANDROID_BUILD_BRANCH)
David Riley73f00d92016-02-16 18:54:20 -0800360 parser.add_argument('--arc_bucket_url',
361 default=constants.ARC_BUCKET_URL,
362 type='gs_path')
David Rileyc0da9d92016-02-01 12:11:01 -0800363 parser.add_argument('-f', '--force_version',
364 help='Android build id to use')
365 parser.add_argument('-s', '--srcroot',
366 default=os.path.join(os.environ['HOME'], 'trunk', 'src'),
367 help='Path to the src directory')
368 parser.add_argument('-t', '--tracking_branch', default='cros/master',
369 help='Branch we are tracking changes against')
370 return parser
371
372
373def main(argv):
374 parser = GetParser()
375 options = parser.parse_args(argv)
376 options.Freeze()
377
378 overlay_dir = os.path.abspath(_OVERLAY_DIR % {'srcroot': options.srcroot})
379 android_package_dir = os.path.join(overlay_dir, constants.ANDROID_CP)
380 version_to_uprev = None
381 subpaths = None
382
383 (unstable_ebuild, stable_ebuilds) = FindAndroidCandidates(android_package_dir)
384
385 if options.force_version:
386 version_to_uprev = options.force_version
387 subpaths = IsBuildIdValid(options.android_bucket_url,
388 options.android_build_branch, version_to_uprev)
389 if not subpaths:
390 logging.error('Requested build %s is not valid' % version_to_uprev)
391 else:
392 version_to_uprev, subpaths = GetLatestBuild(options.android_bucket_url,
393 options.android_build_branch)
394
David Riley73f00d92016-02-16 18:54:20 -0800395 acls = MakeAclDict(android_package_dir)
396 CopyToArcBucket(options.android_bucket_url, options.android_build_branch,
397 version_to_uprev, subpaths, options.arc_bucket_url, acls)
398
David Rileyc0da9d92016-02-01 12:11:01 -0800399 stable_candidate = portage_util.BestEBuild(stable_ebuilds)
400
401 if stable_candidate:
David Riley676f5402016-02-12 17:24:23 -0800402 logging.info('Stable candidate found %s' % stable_candidate.version)
David Rileyc0da9d92016-02-01 12:11:01 -0800403 else:
404 logging.info('No stable candidate found.')
405
406 tracking_branch = 'remotes/m/%s' % os.path.basename(options.tracking_branch)
407 existing_branch = git.GetCurrentBranch(android_package_dir)
408 work_branch = cros_mark_as_stable.GitBranch(constants.STABLE_EBUILD_BRANCH,
409 tracking_branch,
410 android_package_dir)
411 work_branch.CreateBranch()
412
413 # In the case of uprevving overlays that have patches applied to them,
414 # include the patched changes in the stabilizing branch.
415 if existing_branch:
416 git.RunGit(overlay_dir, ['rebase', existing_branch])
417
418 android_version_atom = MarkAndroidEBuildAsStable(
419 stable_candidate, unstable_ebuild, constants.ANDROID_PN,
David Riley73f00d92016-02-16 18:54:20 -0800420 version_to_uprev, android_package_dir,
421 options.android_build_branch, options.arc_bucket_url)
David Rileyc0da9d92016-02-01 12:11:01 -0800422 if android_version_atom:
423 if options.boards:
424 cros_mark_as_stable.CleanStalePackages(options.srcroot,
425 options.boards.split(':'),
426 [android_version_atom])
427
428 # Explicit print to communicate to caller.
429 print('ANDROID_VERSION_ATOM=%s' % android_version_atom)