blob: b41196e45e0b4a982b9daa3fbca5f34e216b4594 [file] [log] [blame]
David Rileyc0da9d92016-02-01 12:11:01 -08001# Copyright 2016 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""This module uprevs Android for cbuildbot.
6
7After calling, it prints outs ANDROID_VERSION_ATOM=(version atom string). A
8caller could then use this atom with emerge to build the newly uprevved version
9of Android e.g.
10
11./cros_mark_android_as_stable
12Returns chromeos-base/android-container-2559197
13
14emerge-veyron_minnie-cheets =chromeos-base/android-container-2559197-r1
15"""
16
17from __future__ import print_function
18
19import filecmp
20import glob
21import os
22
23from chromite.cbuildbot import constants
24from chromite.lib import commandline
25from chromite.lib import cros_build_lib
26from chromite.lib import cros_logging as logging
27from chromite.lib import git
28from chromite.lib import gs
29from chromite.lib import portage_util
30from chromite.scripts import cros_mark_as_stable
31
32
33# Dir where all the action happens.
34_OVERLAY_DIR = '%(srcroot)s/private-overlays/project-cheets-private/'
35
36_GIT_COMMIT_MESSAGE = ('Marking latest for %(android_pn)s ebuild '
37 'with version %(android_version)s as stable.')
38
39# URLs that print lists of Android revisions between two build ids.
40_ANDROID_VERSION_URL = ('http://android-build-uber.corp.google.com/repo.html?'
41 'last_bid=%(old)s&bid=%(new)s&branch=%(branch)s')
42
43
44def IsBuildIdValid(bucket_url, build_branch, build_id):
45 """Checks that a specific build_id is valid.
46
47 Looks for that build_id for all builds. Confirms that the subpath can
48 be found and that the zip file is present in that subdirectory.
49
50 Args:
51 bucket_url: URL of Android build gs bucket
52 build_branch: branch of Android builds
53 build_id: A string. The Android build id number to check.
54
55 Returns:
56 Returns subpaths dictionary if build_id is valid.
57 None if the build_id is not valid.
58 """
59 gs_context = gs.GSContext()
60 subpaths_dict = {}
61 for build, target in constants.ANDROID_BUILD_TARGETS.iteritems():
62 build_dir = '%s-%s' % (build_branch, target)
63 build_id_path = os.path.join(bucket_url, build_dir, build_id)
64
65 # Find name of subpath.
66 try:
67 subpaths = gs_context.List(build_id_path)
68 except gs.GSNoSuchKey:
69 logging.warn(
70 'Directory [%s] does not contain any subpath, ignoring it.',
71 build_id_path)
72 return None
73 if len(subpaths) > 1:
74 logging.warn(
75 'Directory [%s] contains more than one subpath, ignoring it.',
76 build_id_path)
77 return None
78
79 subpath_dir = subpaths[0].url.rstrip('/')
80 subpath_name = os.path.basename(subpath_dir)
81
82 # Look for a zipfile ending in the build_id number.
83 try:
84 for zipfile in gs_context.List(subpath_dir):
85 if zipfile.url.endswith('-%s.zip' % (build_id)):
86 break
87 except gs.GSNoSuchKey:
88 logging.warn(
89 'Did not find a zipfile for build id [%s] in directory [%s].',
90 build_id, subpath_dir)
91 return None
92
93 # Record subpath for the build.
94 subpaths_dict[build] = subpath_name
95
96 # If we got here, it means we found an appropriate build for all platforms.
97 return subpaths_dict
98
99
100def GetLatestBuild(bucket_url, build_branch):
101 """Searches the gs bucket for the latest green build.
102
103 Args:
104 bucket_url: URL of Android build gs bucket
105 build_branch: branch of Android builds
106
107 Returns:
108 Tuple of (latest version string, subpaths dictionary)
109 If no latest build can be found, returns None, None
110 """
111 gs_context = gs.GSContext()
112 common_build_ids = None
113 # Find builds for each target.
114 for target in constants.ANDROID_BUILD_TARGETS.itervalues():
115 build_dir = '-'.join((build_branch, target))
116 base_path = os.path.join(bucket_url, build_dir)
117 build_ids = []
118 for gs_result in gs_context.List(base_path):
119 # Remove trailing slashes and get the base name, which is the build_id.
120 build_id = os.path.basename(gs_result.url.rstrip('/'))
121 if not build_id.isdigit():
122 logging.warn('Directory [%s] does not look like a valid build_id.',
123 gs_result.url)
124 continue
125 build_ids.append(build_id)
126
127 # Update current list of builds.
128 if common_build_ids is None:
129 # First run, populate it with the first platform.
130 common_build_ids = set(build_ids)
131 else:
132 # Already populated, find the ones that are common.
133 common_build_ids.intersection_update(build_ids)
134
135 if common_build_ids is None:
136 logging.warn('Did not find a build_id common to all platforms.')
137 return None, None
138
139 # Otherwise, find the most recent one that is valid.
140 for build_id in sorted(common_build_ids, key=int, reverse=True):
141 subpaths = IsBuildIdValid(bucket_url, build_branch, build_id)
142 if subpaths:
143 return build_id, subpaths
144
145 # If not found, no build_id is valid.
146 logging.warn('Did not find a build_id valid on all platforms.')
147 return None, None
148
149
150def FindAndroidCandidates(package_dir):
151 """Return a tuple of Android's unstable ebuild and stable ebuilds.
152
153 Args:
154 package_dir: The path to where the package ebuild is stored.
155
156 Returns:
157 Tuple [unstable_ebuild, stable_ebuilds].
158
159 Raises:
160 Exception: if no unstable ebuild exists for Android.
161 """
162 stable_ebuilds = []
163 unstable_ebuilds = []
164 for path in glob.glob(os.path.join(package_dir, '*.ebuild')):
165 ebuild = portage_util.EBuild(path)
166 if ebuild.version == '9999':
167 unstable_ebuilds.append(ebuild)
168 else:
169 stable_ebuilds.append(ebuild)
170
171 # Apply some sanity checks.
172 if not unstable_ebuilds:
173 raise Exception('Missing 9999 ebuild for %s' % package_dir)
174 if not stable_ebuilds:
175 logging.warning('Missing stable ebuild for %s' % package_dir)
176
177 return portage_util.BestEBuild(unstable_ebuilds), stable_ebuilds
178
179
180def GetAndroidRevisionListLink(build_branch, old_android, new_android):
181 """Returns a link to the list of revisions between two Android versions
182
183 Given two AndroidEBuilds, generate a link to a page that prints the
184 Android changes between those two revisions, inclusive.
185
186 Args:
187 build_branch: branch of Android builds
188 old_android: ebuild for the version to diff from
189 new_android: ebuild for the version to which to diff
190
191 Returns:
192 The desired URL.
193 """
194 return _ANDROID_VERSION_URL % {'branch': build_branch,
195 'old': old_android.version,
196 'new': new_android.version}
197
198
199def MarkAndroidEBuildAsStable(stable_candidate, unstable_ebuild, android_pn,
200 android_version, subpaths_dict,
201 package_dir, build_branch):
202 r"""Uprevs the Android ebuild.
203
204 This is the main function that uprevs from a stable candidate
205 to its new version.
206
207 Args:
208 stable_candidate: ebuild that corresponds to the stable ebuild we are
209 revving from. If None, builds the a new ebuild given the version
210 with revision set to 1.
211 unstable_ebuild: ebuild corresponding to the unstable ebuild for Android.
212 android_pn: package name.
213 android_version: The \d+ build id of Android.
214 subpaths_dict: Mapping of build to subpath
215 package_dir: Path to the android-container package dir.
216 build_branch: branch of Android builds
217
218 Returns:
219 Full portage version atom (including rc's, etc) that was revved.
220 """
221 def IsTheNewEBuildRedundant(new_ebuild, stable_ebuild):
222 """Returns True if the new ebuild is redundant.
223
224 This is True if there if the current stable ebuild is the exact same copy
225 of the new one.
226 """
227 if not stable_ebuild:
228 return False
229
230 if stable_candidate.version == new_ebuild.version:
231 return filecmp.cmp(
232 new_ebuild.ebuild_path, stable_ebuild.ebuild_path, shallow=False)
233
234 # Case where we have the last stable candidate with same version just rev.
235 if stable_candidate and stable_candidate.version == android_version:
236 new_ebuild_path = '%s-r%d.ebuild' % (
237 stable_candidate.ebuild_path_no_revision,
238 stable_candidate.current_revision + 1)
239 else:
240 pf = '%s-%s-r1' % (android_pn, android_version)
241 new_ebuild_path = os.path.join(package_dir, '%s.ebuild' % pf)
242
243 variables = {'ANDROID_BUILD_ID': android_version}
244 for build, subpath in subpaths_dict.iteritems():
245 variables[build + '_SUBPATH'] = subpath
246
247 portage_util.EBuild.MarkAsStable(
248 unstable_ebuild.ebuild_path, new_ebuild_path,
249 variables, make_stable=True)
250 new_ebuild = portage_util.EBuild(new_ebuild_path)
251
252 # Determine whether this is ebuild is redundant.
253 if IsTheNewEBuildRedundant(new_ebuild, stable_candidate):
254 msg = 'Previous ebuild with same version found and ebuild is redundant.'
255 logging.info(msg)
256 os.unlink(new_ebuild_path)
257 return None
258
259 if stable_candidate:
260 logging.PrintBuildbotLink('Android revisions',
261 GetAndroidRevisionListLink(build_branch,
262 stable_candidate,
263 new_ebuild))
264
265 git.RunGit(package_dir, ['add', new_ebuild_path])
266 if stable_candidate and not stable_candidate.IsSticky():
267 git.RunGit(package_dir, ['rm', stable_candidate.ebuild_path])
268
269 # Update ebuild manifest and git add it.
270 gen_manifest_cmd = ['ebuild', new_ebuild_path, 'manifest', '--force']
271 cros_build_lib.RunCommand(gen_manifest_cmd,
272 extra_env=None, print_cmd=True)
273 git.RunGit(package_dir, ['add', 'Manifest'])
274
275 portage_util.EBuild.CommitChange(
276 _GIT_COMMIT_MESSAGE % {'android_pn': android_pn,
277 'android_version': android_version},
278 package_dir)
279
280 return '%s-%s' % (new_ebuild.package, new_ebuild.version)
281
282
283def GetParser():
284 """Creates the argument parser."""
285 parser = commandline.ArgumentParser()
286 parser.add_argument('-b', '--boards')
287 parser.add_argument('--android_bucket_url',
288 default=constants.ANDROID_BUCKET_URL)
289 parser.add_argument('--android_build_branch',
290 default=constants.ANDROID_BUILD_BRANCH)
291 parser.add_argument('-f', '--force_version',
292 help='Android build id to use')
293 parser.add_argument('-s', '--srcroot',
294 default=os.path.join(os.environ['HOME'], 'trunk', 'src'),
295 help='Path to the src directory')
296 parser.add_argument('-t', '--tracking_branch', default='cros/master',
297 help='Branch we are tracking changes against')
298 return parser
299
300
301def main(argv):
302 parser = GetParser()
303 options = parser.parse_args(argv)
304 options.Freeze()
305
306 overlay_dir = os.path.abspath(_OVERLAY_DIR % {'srcroot': options.srcroot})
307 android_package_dir = os.path.join(overlay_dir, constants.ANDROID_CP)
308 version_to_uprev = None
309 subpaths = None
310
311 (unstable_ebuild, stable_ebuilds) = FindAndroidCandidates(android_package_dir)
312
313 if options.force_version:
314 version_to_uprev = options.force_version
315 subpaths = IsBuildIdValid(options.android_bucket_url,
316 options.android_build_branch, version_to_uprev)
317 if not subpaths:
318 logging.error('Requested build %s is not valid' % version_to_uprev)
319 else:
320 version_to_uprev, subpaths = GetLatestBuild(options.android_bucket_url,
321 options.android_build_branch)
322
323 stable_candidate = portage_util.BestEBuild(stable_ebuilds)
324
325 if stable_candidate:
326 logging.info('Stable candidate found %s' % stable_candidate)
327 else:
328 logging.info('No stable candidate found.')
329
330 tracking_branch = 'remotes/m/%s' % os.path.basename(options.tracking_branch)
331 existing_branch = git.GetCurrentBranch(android_package_dir)
332 work_branch = cros_mark_as_stable.GitBranch(constants.STABLE_EBUILD_BRANCH,
333 tracking_branch,
334 android_package_dir)
335 work_branch.CreateBranch()
336
337 # In the case of uprevving overlays that have patches applied to them,
338 # include the patched changes in the stabilizing branch.
339 if existing_branch:
340 git.RunGit(overlay_dir, ['rebase', existing_branch])
341
342 android_version_atom = MarkAndroidEBuildAsStable(
343 stable_candidate, unstable_ebuild, constants.ANDROID_PN,
344 version_to_uprev, subpaths, android_package_dir,
345 options.android_build_branch)
346 if android_version_atom:
347 if options.boards:
348 cros_mark_as_stable.CleanStalePackages(options.srcroot,
349 options.boards.split(':'),
350 [android_version_atom])
351
352 # Explicit print to communicate to caller.
353 print('ANDROID_VERSION_ATOM=%s' % android_version_atom)