blob: f4b9cdf79afdd643ec3abc45da5322eec9a08e8b [file] [log] [blame]
David Rileyc0da9d92016-02-01 12:11:01 -08001# Copyright 2016 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""This module uprevs Android for cbuildbot.
6
7After calling, it prints outs ANDROID_VERSION_ATOM=(version atom string). A
8caller could then use this atom with emerge to build the newly uprevved version
9of Android e.g.
10
11./cros_mark_android_as_stable
12Returns chromeos-base/android-container-2559197
13
14emerge-veyron_minnie-cheets =chromeos-base/android-container-2559197-r1
15"""
16
17from __future__ import print_function
18
19import filecmp
20import glob
21import os
Hidehiko Abe12727dd2016-05-27 23:23:45 +090022import re
David Rileyc0da9d92016-02-01 12:11:01 -080023
24from chromite.cbuildbot import constants
25from chromite.lib import commandline
26from chromite.lib import cros_build_lib
27from chromite.lib import cros_logging as logging
28from chromite.lib import git
29from chromite.lib import gs
30from chromite.lib import portage_util
31from chromite.scripts import cros_mark_as_stable
32
33
34# Dir where all the action happens.
35_OVERLAY_DIR = '%(srcroot)s/private-overlays/project-cheets-private/'
36
37_GIT_COMMIT_MESSAGE = ('Marking latest for %(android_pn)s ebuild '
38 'with version %(android_version)s as stable.')
39
40# URLs that print lists of Android revisions between two build ids.
41_ANDROID_VERSION_URL = ('http://android-build-uber.corp.google.com/repo.html?'
42 'last_bid=%(old)s&bid=%(new)s&branch=%(branch)s')
43
44
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +090045def IsBuildIdValid(bucket_url, build_branch, build_id, targets):
David Rileyc0da9d92016-02-01 12:11:01 -080046 """Checks that a specific build_id is valid.
47
48 Looks for that build_id for all builds. Confirms that the subpath can
49 be found and that the zip file is present in that subdirectory.
50
51 Args:
52 bucket_url: URL of Android build gs bucket
53 build_branch: branch of Android builds
54 build_id: A string. The Android build id number to check.
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +090055 targets: Dict from build key to (targe build suffix, artifact file pattern)
56 pair.
David Rileyc0da9d92016-02-01 12:11:01 -080057
58 Returns:
59 Returns subpaths dictionary if build_id is valid.
60 None if the build_id is not valid.
61 """
62 gs_context = gs.GSContext()
63 subpaths_dict = {}
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +090064 for build, (target, _) in targets.iteritems():
David Rileyc0da9d92016-02-01 12:11:01 -080065 build_dir = '%s-%s' % (build_branch, target)
66 build_id_path = os.path.join(bucket_url, build_dir, build_id)
67
68 # Find name of subpath.
69 try:
70 subpaths = gs_context.List(build_id_path)
71 except gs.GSNoSuchKey:
72 logging.warn(
73 'Directory [%s] does not contain any subpath, ignoring it.',
74 build_id_path)
75 return None
76 if len(subpaths) > 1:
77 logging.warn(
78 'Directory [%s] contains more than one subpath, ignoring it.',
79 build_id_path)
80 return None
81
82 subpath_dir = subpaths[0].url.rstrip('/')
83 subpath_name = os.path.basename(subpath_dir)
84
85 # Look for a zipfile ending in the build_id number.
86 try:
Hidehiko Abe12727dd2016-05-27 23:23:45 +090087 gs_context.List(subpath_dir)
David Rileyc0da9d92016-02-01 12:11:01 -080088 except gs.GSNoSuchKey:
89 logging.warn(
Hidehiko Abe12727dd2016-05-27 23:23:45 +090090 'Did not find a file for build id [%s] in directory [%s].',
David Rileyc0da9d92016-02-01 12:11:01 -080091 build_id, subpath_dir)
92 return None
93
94 # Record subpath for the build.
95 subpaths_dict[build] = subpath_name
96
97 # If we got here, it means we found an appropriate build for all platforms.
98 return subpaths_dict
99
100
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900101def GetLatestBuild(bucket_url, build_branch, targets):
David Rileyc0da9d92016-02-01 12:11:01 -0800102 """Searches the gs bucket for the latest green build.
103
104 Args:
105 bucket_url: URL of Android build gs bucket
106 build_branch: branch of Android builds
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900107 targets: Dict from build key to (targe build suffix, artifact file pattern)
108 pair.
David Rileyc0da9d92016-02-01 12:11:01 -0800109
110 Returns:
111 Tuple of (latest version string, subpaths dictionary)
112 If no latest build can be found, returns None, None
113 """
114 gs_context = gs.GSContext()
115 common_build_ids = None
116 # Find builds for each target.
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900117 for target, _ in targets.itervalues():
David Rileyc0da9d92016-02-01 12:11:01 -0800118 build_dir = '-'.join((build_branch, target))
119 base_path = os.path.join(bucket_url, build_dir)
120 build_ids = []
121 for gs_result in gs_context.List(base_path):
122 # Remove trailing slashes and get the base name, which is the build_id.
123 build_id = os.path.basename(gs_result.url.rstrip('/'))
124 if not build_id.isdigit():
125 logging.warn('Directory [%s] does not look like a valid build_id.',
126 gs_result.url)
127 continue
128 build_ids.append(build_id)
129
130 # Update current list of builds.
131 if common_build_ids is None:
132 # First run, populate it with the first platform.
133 common_build_ids = set(build_ids)
134 else:
135 # Already populated, find the ones that are common.
136 common_build_ids.intersection_update(build_ids)
137
138 if common_build_ids is None:
139 logging.warn('Did not find a build_id common to all platforms.')
140 return None, None
141
142 # Otherwise, find the most recent one that is valid.
143 for build_id in sorted(common_build_ids, key=int, reverse=True):
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900144 subpaths = IsBuildIdValid(bucket_url, build_branch, build_id, targets)
David Rileyc0da9d92016-02-01 12:11:01 -0800145 if subpaths:
146 return build_id, subpaths
147
148 # If not found, no build_id is valid.
149 logging.warn('Did not find a build_id valid on all platforms.')
150 return None, None
151
152
153def FindAndroidCandidates(package_dir):
154 """Return a tuple of Android's unstable ebuild and stable ebuilds.
155
156 Args:
157 package_dir: The path to where the package ebuild is stored.
158
159 Returns:
160 Tuple [unstable_ebuild, stable_ebuilds].
161
162 Raises:
163 Exception: if no unstable ebuild exists for Android.
164 """
165 stable_ebuilds = []
166 unstable_ebuilds = []
167 for path in glob.glob(os.path.join(package_dir, '*.ebuild')):
168 ebuild = portage_util.EBuild(path)
169 if ebuild.version == '9999':
170 unstable_ebuilds.append(ebuild)
171 else:
172 stable_ebuilds.append(ebuild)
173
174 # Apply some sanity checks.
175 if not unstable_ebuilds:
176 raise Exception('Missing 9999 ebuild for %s' % package_dir)
177 if not stable_ebuilds:
178 logging.warning('Missing stable ebuild for %s' % package_dir)
179
180 return portage_util.BestEBuild(unstable_ebuilds), stable_ebuilds
181
182
David Riley73f00d92016-02-16 18:54:20 -0800183def CopyToArcBucket(android_bucket_url, build_branch, build_id, subpaths,
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900184 targets, arc_bucket_url, acls):
David Riley73f00d92016-02-16 18:54:20 -0800185 """Copies from source Android bucket to ARC++ specific bucket.
186
187 Copies each build to the ARC bucket eliminating the subpath.
188 Applies build specific ACLs for each file.
189
190 Args:
191 android_bucket_url: URL of Android build gs bucket
192 build_branch: branch of Android builds
193 build_id: A string. The Android build id number to check.
194 subpaths: Subpath dictionary for each build to copy.
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900195 targets: Dict from build key to (targe build suffix, artifact file pattern)
196 pair.
David Riley73f00d92016-02-16 18:54:20 -0800197 arc_bucket_url: URL of the target ARC build gs bucket
198 acls: ACLs dictionary for each build to copy.
199 """
200 gs_context = gs.GSContext()
201 for build, subpath in subpaths.iteritems():
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900202 target, pattern = targets[build]
David Riley73f00d92016-02-16 18:54:20 -0800203 build_dir = '%s-%s' % (build_branch, target)
204 android_dir = os.path.join(android_bucket_url, build_dir, build_id, subpath)
205 arc_dir = os.path.join(arc_bucket_url, build_dir, build_id)
206
Hidehiko Abe12727dd2016-05-27 23:23:45 +0900207 # Copy all target files from android_dir to arc_dir, setting ACLs.
208 for targetfile in gs_context.List(android_dir):
209 if re.search(pattern, targetfile.url):
210 basename = os.path.basename(targetfile.url)
211 arc_path = os.path.join(arc_dir, basename)
David Riley73f00d92016-02-16 18:54:20 -0800212 acl = acls[build]
213 needs_copy = True
214
215 # Check a pre-existing file with the original source.
216 if gs_context.Exists(arc_path):
Hidehiko Abe12727dd2016-05-27 23:23:45 +0900217 if (gs_context.Stat(targetfile.url).hash_crc32c !=
Elijah Taylorbfc30692016-04-22 14:05:23 -0700218 gs_context.Stat(arc_path).hash_crc32c):
David Riley73f00d92016-02-16 18:54:20 -0800219 logging.warn('Removing incorrect file %s', arc_path)
220 gs_context.Remove(arc_path)
221 else:
222 logging.info('Skipping already copied file %s', arc_path)
223 needs_copy = False
224
225 # Copy if necessary, and set the ACL unconditionally.
226 # The Stat() call above doesn't verify the ACL is correct and
227 # the ChangeACL should be relatively cheap compared to the copy.
228 # This covers the following caes:
229 # - handling an interrupted copy from a previous run.
230 # - rerunning the copy in case one of the googlestorage_acl_X.txt
231 # files changes (e.g. we add a new variant which reuses a build).
232 if needs_copy:
Hidehiko Abe12727dd2016-05-27 23:23:45 +0900233 logging.info('Copying %s -> %s (acl %s)',
234 targetfile.url, arc_path, acl)
235 gs_context.Copy(targetfile.url, arc_path, version=0)
David Riley73f00d92016-02-16 18:54:20 -0800236 gs_context.ChangeACL(arc_path, acl_args_file=acl)
237
238
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900239def MirrorArtifacts(android_bucket_url, android_build_branch, arc_bucket_url,
240 acls, targets, version=None):
241 """Mirrors artifacts from Android bucket to ARC bucket.
242
243 First, this function identifies which build version should be copied,
244 if not given. Please see GetLatestBuild() and IsBuildIdValid() for details.
245
246 On build version identified, then copies target artifacts to the ARC bucket,
247 with setting ACLs.
248
249 Args:
250 android_bucket_url: URL of Android build gs bucket
251 android_build_branch: branch of Android builds
252 arc_bucket_url: URL of the target ARC build gs bucket
253 acls: ACLs dictionary for each build to copy.
254 targets: Dict from build key to (targe build suffix, artifact file pattern)
255 pair.
256 version: (optional) A string. The Android build id number to check.
257 If not passed, detect latest good build version.
258
259 Returns:
260 Mirrored version.
261 """
262 if version:
263 subpaths = IsBuildIdValid(
264 android_bucket_url, android_build_branch, version, targets)
265 if not subpaths:
266 logging.error('Requested build %s is not valid' % version)
267 else:
268 version, subpaths = GetLatestBuild(
269 android_bucket_url, android_build_branch, targets)
270
271 CopyToArcBucket(android_bucket_url, android_build_branch, version, subpaths,
272 targets, arc_bucket_url, acls)
273 return version
274
275
David Riley73f00d92016-02-16 18:54:20 -0800276def MakeAclDict(package_dir):
277 """Creates a dictionary of acl files for each build type.
278
279 Args:
280 package_dir: The path to where the package acl files are stored.
281
282 Returns:
283 Returns acls dictionary.
284 """
285 return dict(
286 (k, os.path.join(package_dir, v))
287 for k, v in constants.ARC_BUCKET_ACLS.items()
288 )
289
290
David Rileyc0da9d92016-02-01 12:11:01 -0800291def GetAndroidRevisionListLink(build_branch, old_android, new_android):
292 """Returns a link to the list of revisions between two Android versions
293
294 Given two AndroidEBuilds, generate a link to a page that prints the
295 Android changes between those two revisions, inclusive.
296
297 Args:
298 build_branch: branch of Android builds
299 old_android: ebuild for the version to diff from
300 new_android: ebuild for the version to which to diff
301
302 Returns:
303 The desired URL.
304 """
305 return _ANDROID_VERSION_URL % {'branch': build_branch,
306 'old': old_android.version,
307 'new': new_android.version}
308
309
310def MarkAndroidEBuildAsStable(stable_candidate, unstable_ebuild, android_pn,
David Riley73f00d92016-02-16 18:54:20 -0800311 android_version, package_dir, build_branch,
312 arc_bucket_url):
David Rileyc0da9d92016-02-01 12:11:01 -0800313 r"""Uprevs the Android ebuild.
314
315 This is the main function that uprevs from a stable candidate
316 to its new version.
317
318 Args:
319 stable_candidate: ebuild that corresponds to the stable ebuild we are
320 revving from. If None, builds the a new ebuild given the version
321 with revision set to 1.
322 unstable_ebuild: ebuild corresponding to the unstable ebuild for Android.
323 android_pn: package name.
324 android_version: The \d+ build id of Android.
David Rileyc0da9d92016-02-01 12:11:01 -0800325 package_dir: Path to the android-container package dir.
David Riley73f00d92016-02-16 18:54:20 -0800326 build_branch: branch of Android builds.
327 arc_bucket_url: URL of the target ARC build gs bucket.
David Rileyc0da9d92016-02-01 12:11:01 -0800328
329 Returns:
330 Full portage version atom (including rc's, etc) that was revved.
331 """
332 def IsTheNewEBuildRedundant(new_ebuild, stable_ebuild):
333 """Returns True if the new ebuild is redundant.
334
335 This is True if there if the current stable ebuild is the exact same copy
336 of the new one.
337 """
338 if not stable_ebuild:
339 return False
340
David Riley676f5402016-02-12 17:24:23 -0800341 if stable_candidate.version_no_rev == new_ebuild.version_no_rev:
David Rileyc0da9d92016-02-01 12:11:01 -0800342 return filecmp.cmp(
343 new_ebuild.ebuild_path, stable_ebuild.ebuild_path, shallow=False)
344
345 # Case where we have the last stable candidate with same version just rev.
David Riley676f5402016-02-12 17:24:23 -0800346 if stable_candidate and stable_candidate.version_no_rev == android_version:
David Rileyc0da9d92016-02-01 12:11:01 -0800347 new_ebuild_path = '%s-r%d.ebuild' % (
348 stable_candidate.ebuild_path_no_revision,
349 stable_candidate.current_revision + 1)
350 else:
351 pf = '%s-%s-r1' % (android_pn, android_version)
352 new_ebuild_path = os.path.join(package_dir, '%s.ebuild' % pf)
353
David Riley73f00d92016-02-16 18:54:20 -0800354 variables = {'BASE_URL': arc_bucket_url}
Hidehiko Abe12727dd2016-05-27 23:23:45 +0900355 for build, (target, _) in constants.ANDROID_BUILD_TARGETS.iteritems():
David Riley73f00d92016-02-16 18:54:20 -0800356 variables[build + '_TARGET'] = '%s-%s' % (build_branch, target)
David Rileyc0da9d92016-02-01 12:11:01 -0800357
358 portage_util.EBuild.MarkAsStable(
359 unstable_ebuild.ebuild_path, new_ebuild_path,
360 variables, make_stable=True)
361 new_ebuild = portage_util.EBuild(new_ebuild_path)
362
363 # Determine whether this is ebuild is redundant.
364 if IsTheNewEBuildRedundant(new_ebuild, stable_candidate):
365 msg = 'Previous ebuild with same version found and ebuild is redundant.'
366 logging.info(msg)
367 os.unlink(new_ebuild_path)
368 return None
369
370 if stable_candidate:
371 logging.PrintBuildbotLink('Android revisions',
372 GetAndroidRevisionListLink(build_branch,
373 stable_candidate,
374 new_ebuild))
375
376 git.RunGit(package_dir, ['add', new_ebuild_path])
377 if stable_candidate and not stable_candidate.IsSticky():
378 git.RunGit(package_dir, ['rm', stable_candidate.ebuild_path])
379
380 # Update ebuild manifest and git add it.
381 gen_manifest_cmd = ['ebuild', new_ebuild_path, 'manifest', '--force']
382 cros_build_lib.RunCommand(gen_manifest_cmd,
383 extra_env=None, print_cmd=True)
384 git.RunGit(package_dir, ['add', 'Manifest'])
385
386 portage_util.EBuild.CommitChange(
387 _GIT_COMMIT_MESSAGE % {'android_pn': android_pn,
388 'android_version': android_version},
389 package_dir)
390
391 return '%s-%s' % (new_ebuild.package, new_ebuild.version)
392
393
394def GetParser():
395 """Creates the argument parser."""
396 parser = commandline.ArgumentParser()
397 parser.add_argument('-b', '--boards')
398 parser.add_argument('--android_bucket_url',
David Riley73f00d92016-02-16 18:54:20 -0800399 default=constants.ANDROID_BUCKET_URL,
400 type='gs_path')
David Rileyc0da9d92016-02-01 12:11:01 -0800401 parser.add_argument('--android_build_branch',
402 default=constants.ANDROID_BUILD_BRANCH)
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900403 parser.add_argument('--android_gts_build_branch',
404 default=constants.ANDROID_GTS_BUILD_BRANCH)
David Riley73f00d92016-02-16 18:54:20 -0800405 parser.add_argument('--arc_bucket_url',
406 default=constants.ARC_BUCKET_URL,
407 type='gs_path')
David Rileyc0da9d92016-02-01 12:11:01 -0800408 parser.add_argument('-f', '--force_version',
409 help='Android build id to use')
410 parser.add_argument('-s', '--srcroot',
411 default=os.path.join(os.environ['HOME'], 'trunk', 'src'),
412 help='Path to the src directory')
413 parser.add_argument('-t', '--tracking_branch', default='cros/master',
414 help='Branch we are tracking changes against')
415 return parser
416
417
418def main(argv):
419 parser = GetParser()
420 options = parser.parse_args(argv)
421 options.Freeze()
422
423 overlay_dir = os.path.abspath(_OVERLAY_DIR % {'srcroot': options.srcroot})
424 android_package_dir = os.path.join(overlay_dir, constants.ANDROID_CP)
425 version_to_uprev = None
David Rileyc0da9d92016-02-01 12:11:01 -0800426
427 (unstable_ebuild, stable_ebuilds) = FindAndroidCandidates(android_package_dir)
David Riley73f00d92016-02-16 18:54:20 -0800428 acls = MakeAclDict(android_package_dir)
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900429 # Mirror artifacts, i.e., images and some sdk tools (e.g., adb, aapt).
430 version_to_uprev = MirrorArtifacts(options.android_bucket_url,
431 options.android_build_branch,
432 options.arc_bucket_url, acls,
433 constants.ANDROID_BUILD_TARGETS,
434 options.force_version)
435
436 # Mirror GTS.
437 MirrorArtifacts(options.android_bucket_url,
438 options.android_gts_build_branch,
439 options.arc_bucket_url, acls,
440 constants.ANDROID_GTS_BUILD_TARGETS)
David Riley73f00d92016-02-16 18:54:20 -0800441
David Rileyc0da9d92016-02-01 12:11:01 -0800442 stable_candidate = portage_util.BestEBuild(stable_ebuilds)
443
444 if stable_candidate:
David Riley676f5402016-02-12 17:24:23 -0800445 logging.info('Stable candidate found %s' % stable_candidate.version)
David Rileyc0da9d92016-02-01 12:11:01 -0800446 else:
447 logging.info('No stable candidate found.')
448
449 tracking_branch = 'remotes/m/%s' % os.path.basename(options.tracking_branch)
450 existing_branch = git.GetCurrentBranch(android_package_dir)
451 work_branch = cros_mark_as_stable.GitBranch(constants.STABLE_EBUILD_BRANCH,
452 tracking_branch,
453 android_package_dir)
454 work_branch.CreateBranch()
455
456 # In the case of uprevving overlays that have patches applied to them,
457 # include the patched changes in the stabilizing branch.
458 if existing_branch:
459 git.RunGit(overlay_dir, ['rebase', existing_branch])
460
461 android_version_atom = MarkAndroidEBuildAsStable(
462 stable_candidate, unstable_ebuild, constants.ANDROID_PN,
David Riley73f00d92016-02-16 18:54:20 -0800463 version_to_uprev, android_package_dir,
464 options.android_build_branch, options.arc_bucket_url)
David Rileyc0da9d92016-02-01 12:11:01 -0800465 if android_version_atom:
466 if options.boards:
467 cros_mark_as_stable.CleanStalePackages(options.srcroot,
468 options.boards.split(':'),
469 [android_version_atom])
470
471 # Explicit print to communicate to caller.
472 print('ANDROID_VERSION_ATOM=%s' % android_version_atom)