scripts/cros_mark_chrome_as_stable.py

#!/usr/bin/python
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""This module uprevs Chrome for cbuildbot.

After calling, it prints outs CHROME_VERSION_ATOM=(version atom string).  A
caller could then use this atom with emerge to build the newly uprevved version
of Chrome e.g.

./cros_mark_chrome_as_stable tot
Returns chrome-base/chromeos-chrome-8.0.552.0_alpha_r1

emerge-x86-generic =chrome-base/chromeos-chrome-8.0.552.0_alpha_r1
"""

import base64
import distutils.version
import filecmp
import optparse
import os
import re
import sys
import urlparse

from chromite.cbuildbot import constants
from chromite.lib import cros_build_lib
from chromite.lib import git
from chromite.lib import gob_util
from chromite.lib import portage_util
from chromite.lib import timeout_util
from chromite.scripts import cros_mark_as_stable

# Helper regex's for finding ebuilds.
_CHROME_VERSION_REGEX = r'\d+\.\d+\.\d+\.\d+'
_NON_STICKY_REGEX = r'%s[(_rc.*)|(_alpha.*)]+' % _CHROME_VERSION_REGEX

# Dir where all the action happens.
_OVERLAY_DIR = '%(srcroot)s/third_party/chromiumos-overlay/'

_GIT_COMMIT_MESSAGE = ('Marking %(chrome_rev)s for %(chrome_pn)s ebuild '
                       'with version %(chrome_version)s as stable.')

# URLs that print lists of chrome revisions between two versions of the browser.
_CHROME_VERSION_URL = ('http://omahaproxy.appspot.com/changelog?'
                       'old_version=%(old)s&new_version=%(new)s')

# Only print links when we rev these types.
_REV_TYPES_FOR_LINKS = [constants.CHROME_REV_LATEST,
                        constants.CHROME_REV_STICKY]

# TODO(szager): This is inaccurate, but is it safe to change?  I have no idea.
_CHROME_SVN_TAG = 'CROS_SVN_COMMIT'


def _GetVersionContents(chrome_version_info):
  """Returns the current Chromium version, from the contents of a VERSION file.

  Args:
     chrome_version_info: The contents of a chromium VERSION file.
  """
  chrome_version_array = []
  for line in chrome_version_info.splitlines():
    chrome_version_array.append(line.rpartition('=')[2])

  return '.'.join(chrome_version_array)


def _GetSpecificVersionUrl(git_url, revision, time_to_wait=600):
  """Returns the Chromium version, from a repository URL and version.

  Args:
     git_url: Repository URL for chromium.
     revision: the git revision we want to use.
     time_to_wait: the minimum period before abandoning our wait for the
         desired revision to be present.
  """
  parsed_url = urlparse.urlparse(git_url)
  host = parsed_url[1]
  path = parsed_url[2].rstrip('/') + (
      '/+/%s/chrome/VERSION?format=text' % revision)

  # Allow for git repository replication lag with sleep/retry loop.
  def _fetch():
    fh = gob_util.FetchUrl(host, path, ignore_404=True)
    return fh.read() if fh else None

  def _wait_msg(_remaining_minutes):
    cros_build_lib.Info(
        'Repository does not yet have revision %s.  Sleeping...',
        revision)

  content = timeout_util.WaitForSuccess(
      retry_check=lambda x: not bool(x),
      func=_fetch,
      timeout=time_to_wait,
      period=30,
      side_effect_func=_wait_msg)
  return _GetVersionContents(base64.b64decode(content))


def _GetTipOfTrunkVersionFile(root):
  """Returns the current Chromium version, from a file in a checkout.

  Args:
     root: path to the root of the chromium checkout.
  """
  version_file = os.path.join(root, 'src', 'chrome', 'VERSION')
  chrome_version_info = cros_build_lib.RunCommand(
      ['cat', version_file],
      redirect_stdout=True,
      error_message='Could not read version file at %s.' % version_file).output

  return _GetVersionContents(chrome_version_info)


def CheckIfChromeRightForOS(deps_content):
  """Checks if DEPS is right for Chrome OS.

  This function checks for a variable called 'buildspec_platforms' to
  find out if its 'chromeos' or 'all'. If any of those values,
  then it chooses that DEPS.

  Args:
    deps_content: Content of release buildspec DEPS file.

  Returns:
    True if DEPS is the right Chrome for Chrome OS.
  """
  platforms_search = re.search(r'buildspec_platforms.*\s.*\s', deps_content)

  if platforms_search:
    platforms = platforms_search.group()
    if 'chromeos' in platforms or 'all' in platforms:
      return True

  return False


def GetLatestRelease(git_url, branch=None):
  """Gets the latest release version from the release tags in the repository.

  Args:
    git_url: URL of git repository.
    branch: If set, gets the latest release for branch, otherwise latest
      release.

  Returns:
    Latest version string.
  """
  # TODO(szager): This only works for public release buildspecs in the chromium
  # src repository.  Internal buildspecs are tracked differently.  At the time
  # of writing, I can't find any callers that use this method to scan for
  # internal buildspecs.  But there may be something lurking...

  parsed_url = urlparse.urlparse(git_url)
  path = parsed_url[2].rstrip('/') + '/+refs/tags?format=JSON'
  j = gob_util.FetchUrlJson(parsed_url[1], path, ignore_404=False)
  if branch:
    chrome_version_re = re.compile(r'^%s\.\d+.*' % branch)
  else:
    chrome_version_re = re.compile(r'^[0-9]+\..*')
  matching_versions = [key for key in j.keys() if chrome_version_re.match(key)]
  matching_versions.sort(key=distutils.version.LooseVersion)
  for chrome_version in reversed(matching_versions):
    path = parsed_url[2].rstrip() + (
        '/+/refs/tags/%s/DEPS?format=text' % chrome_version)
    fh = gob_util.FetchUrl(parsed_url[1], path, ignore_404=False)
    content = fh.read() if fh else None
    if content:
      deps_content = base64.b64decode(content)
      if CheckIfChromeRightForOS(deps_content):
        return chrome_version

  return None


def _GetStickyEBuild(stable_ebuilds):
  """Returns the sticky ebuild."""
  sticky_ebuilds = []
  non_sticky_re = re.compile(_NON_STICKY_REGEX)
  for ebuild in stable_ebuilds:
    if not non_sticky_re.match(ebuild.version):
      sticky_ebuilds.append(ebuild)

  if not sticky_ebuilds:
    raise Exception('No sticky ebuilds found')
  elif len(sticky_ebuilds) > 1:
    cros_build_lib.Warning('More than one sticky ebuild found')

  return portage_util.BestEBuild(sticky_ebuilds)


class ChromeEBuild(portage_util.EBuild):
  """Thin sub-class of EBuild that adds a chrome_version field."""
  chrome_version_re = re.compile(r'.*-(%s|9999).*' % (
      _CHROME_VERSION_REGEX))
  chrome_version = ''

  def __init__(self, path):
    portage_util.EBuild.__init__(self, path)
    re_match = self.chrome_version_re.match(self.ebuild_path_no_revision)
    if re_match:
      self.chrome_version = re_match.group(1)

  def __str__(self):
    return self.ebuild_path


def FindChromeCandidates(package_dir):
  """Return a tuple of chrome's unstable ebuild and stable ebuilds.

  Args:
    package_dir: The path to where the package ebuild is stored.

  Returns:
    Tuple [unstable_ebuild, stable_ebuilds].

  Raises:
    Exception: if no unstable ebuild exists for Chrome.
  """
  stable_ebuilds = []
  unstable_ebuilds = []
  for path in [
      os.path.join(package_dir, entry) for entry in os.listdir(package_dir)]:
    if path.endswith('.ebuild'):
      ebuild = ChromeEBuild(path)
      if not ebuild.chrome_version:
        cros_build_lib.Warning('Poorly formatted ebuild found at %s' % path)
      else:
        if '9999' in ebuild.version:
          unstable_ebuilds.append(ebuild)
        else:
          stable_ebuilds.append(ebuild)

  # Apply some sanity checks.
  if not unstable_ebuilds:
    raise Exception('Missing 9999 ebuild for %s' % package_dir)
  if not stable_ebuilds:
    cros_build_lib.Warning('Missing stable ebuild for %s' % package_dir)

  return portage_util.BestEBuild(unstable_ebuilds), stable_ebuilds


def FindChromeUprevCandidate(stable_ebuilds, chrome_rev, sticky_branch):
  """Finds the Chrome uprev candidate for the given chrome_rev.

  Using the pre-flight logic, this means the stable ebuild you are uprevving
  from.  The difference here is that the version could be different and in
  that case we want to find it to delete it.

  Args:
    stable_ebuilds: A list of stable ebuilds.
    chrome_rev: The chrome_rev designating which candidate to find.
    sticky_branch: The the branch that is currently sticky with Major/Minor
      components.  For example: 9.0.553. Can be None but not if chrome_rev
      is CHROME_REV_STICKY.

  Returns:
    The EBuild, otherwise None if none found.
  """
  candidates = []
  if chrome_rev in [constants.CHROME_REV_LOCAL, constants.CHROME_REV_TOT,
                    constants.CHROME_REV_SPEC]:
    # These are labelled alpha, for historic reasons,
    # not just for the fun of confusion.
    chrome_branch_re = re.compile(r'%s.*_alpha.*' % _CHROME_VERSION_REGEX)
    for ebuild in stable_ebuilds:
      if chrome_branch_re.search(ebuild.version):
        candidates.append(ebuild)

  elif chrome_rev == constants.CHROME_REV_STICKY:
    assert sticky_branch is not None
    chrome_branch_re = re.compile(r'%s\..*' % sticky_branch)
    for ebuild in stable_ebuilds:
      if chrome_branch_re.search(ebuild.version):
        candidates.append(ebuild)

  else:
    chrome_branch_re = re.compile(r'%s.*_rc.*' % _CHROME_VERSION_REGEX)
    for ebuild in stable_ebuilds:
      if chrome_branch_re.search(ebuild.version):
        candidates.append(ebuild)

  if candidates:
    return portage_util.BestEBuild(candidates)
  else:
    return None


def _AnnotateAndPrint(text, url):
  """Add buildbot trappings to print <a href='url'>text</a> in the waterfall.

  Args:
    text: Anchor text for the link
    url: the URL to which to link
  """
  print >> sys.stderr, '\n@@@STEP_LINK@%(text)s@%(url)s@@@' % { 'text': text,
                                                              'url': url }

def GetChromeRevisionLinkFromVersions(old_chrome_version, chrome_version):
  """Return appropriately formatted link to revision info, given versions

  Given two chrome version strings (e.g. 9.0.533.0), generate a link to a
  page that prints the Chromium revisions between those two versions.

  Args:
    old_chrome_version: version to diff from
    chrome_version: version to which to diff

  Returns:
    The desired URL.
  """
  return _CHROME_VERSION_URL % { 'old': old_chrome_version,
                                 'new': chrome_version }

def GetChromeRevisionListLink(old_chrome, new_chrome, chrome_rev):
  """Returns a link to the list of revisions between two Chromium versions

  Given two ChromeEBuilds and the kind of rev we're doing, generate a
  link to a page that prints the Chromium changes between those two
  revisions, inclusive.

  Args:
    old_chrome: ebuild for the version to diff from
    new_chrome: ebuild for the version to which to diff
    chrome_rev: one of constants.VALID_CHROME_REVISIONS

  Returns:
    The desired URL.
  """
  assert chrome_rev in _REV_TYPES_FOR_LINKS
  return GetChromeRevisionLinkFromVersions(old_chrome.chrome_version,
                                           new_chrome.chrome_version)


def MarkChromeEBuildAsStable(stable_candidate, unstable_ebuild, chrome_pn,
                             chrome_rev, chrome_version, commit, package_dir):
  r"""Uprevs the chrome ebuild specified by chrome_rev.

  This is the main function that uprevs the chrome_rev from a stable candidate
  to its new version.

  Args:
    stable_candidate: ebuild that corresponds to the stable ebuild we are
      revving from.  If None, builds the a new ebuild given the version
      and logic for chrome_rev type with revision set to 1.
    unstable_ebuild: ebuild corresponding to the unstable ebuild for chrome.
    chrome_pn: package name.
    chrome_rev: one of constants.VALID_CHROME_REVISIONS or LOCAL
      constants.CHROME_REV_SPEC -  Requires commit value.  Revs the ebuild for
        the specified version and uses the portage suffix of _alpha.
      constants.CHROME_REV_TOT -  Requires commit value.  Revs the ebuild for
        the TOT version and uses the portage suffix of _alpha.
      constants.CHROME_REV_LOCAL - Requires a chrome_root. Revs the ebuild for
        the local version and uses the portage suffix of _alpha.
      constants.CHROME_REV_LATEST - This uses the portage suffix of _rc as they
        are release candidates for the next sticky version.
      constants.CHROME_REV_STICKY -  Revs the sticky version.
    chrome_version: The \d.\d.\d.\d version of Chrome.
    commit: Used with constants.CHROME_REV_TOT.  The git revision of chrome.
    package_dir: Path to the chromeos-chrome package dir.

  Returns:
    Full portage version atom (including rc's, etc) that was revved.
  """
  def IsTheNewEBuildRedundant(new_ebuild, stable_ebuild):
    """Returns True if the new ebuild is redundant.

    This is True if there if the current stable ebuild is the exact same copy
    of the new one.
    """
    if not stable_ebuild:
      return False

    if stable_candidate.chrome_version == new_ebuild.chrome_version:
      return filecmp.cmp(
          new_ebuild.ebuild_path, stable_ebuild.ebuild_path, shallow=False)

  # Mark latest release and sticky branches as stable.
  mark_stable = chrome_rev not in [constants.CHROME_REV_TOT,
                                   constants.CHROME_REV_SPEC,
                                   constants.CHROME_REV_LOCAL]

  # Case where we have the last stable candidate with same version just rev.
  if stable_candidate and stable_candidate.chrome_version == chrome_version:
    new_ebuild_path = '%s-r%d.ebuild' % (
        stable_candidate.ebuild_path_no_revision,
        stable_candidate.current_revision + 1)
  else:
    suffix = 'rc' if mark_stable else 'alpha'
    pf = '%s-%s_%s-r1' % (chrome_pn, chrome_version, suffix)
    new_ebuild_path = os.path.join(package_dir, '%s.ebuild' % pf)

  chrome_variables = dict()
  if commit:
    chrome_variables[_CHROME_SVN_TAG] = commit

  portage_util.EBuild.MarkAsStable(
      unstable_ebuild.ebuild_path, new_ebuild_path,
      chrome_variables, make_stable=mark_stable)
  new_ebuild = ChromeEBuild(new_ebuild_path)

  # Determine whether this is ebuild is redundant.
  if IsTheNewEBuildRedundant(new_ebuild, stable_candidate):
    msg = 'Previous ebuild with same version found and ebuild is redundant.'
    cros_build_lib.Info(msg)
    os.unlink(new_ebuild_path)
    return None

  if stable_candidate and chrome_rev in _REV_TYPES_FOR_LINKS:
    _AnnotateAndPrint('Chromium revisions',
                      GetChromeRevisionListLink(stable_candidate,
                                                new_ebuild,
                                                chrome_rev))

  git.RunGit(package_dir, ['add', new_ebuild_path])
  if stable_candidate and not stable_candidate.IsSticky():
    git.RunGit(package_dir, ['rm', stable_candidate.ebuild_path])

  portage_util.EBuild.CommitChange(
      _GIT_COMMIT_MESSAGE % {'chrome_pn': chrome_pn,
                             'chrome_rev': chrome_rev,
                             'chrome_version': chrome_version},
      package_dir)

  return '%s-%s' % (new_ebuild.package, new_ebuild.version)


def main(_argv):
  usage_options = '|'.join(constants.VALID_CHROME_REVISIONS)
  usage = '%s OPTIONS [%s]' % (__file__, usage_options)
  parser = optparse.OptionParser(usage)
  parser.add_option('-b', '--boards', default=None)
  parser.add_option('-c', '--chrome_url',
                    default=constants.CHROMIUM_GOB_URL)
  parser.add_option('-f', '--force_version', default=None,
                    help='Chrome version or git revision hash to use')
  parser.add_option('-s', '--srcroot', default=os.path.join(os.environ['HOME'],
                                                            'trunk', 'src'),
                    help='Path to the src directory')
  parser.add_option('-t', '--tracking_branch', default='cros/master',
                    help='Branch we are tracking changes against')
  (options, args) = parser.parse_args()

  if len(args) != 1 or args[0] not in constants.VALID_CHROME_REVISIONS:
    parser.error('Commit requires arg set to one of %s.'
                 % constants.VALID_CHROME_REVISIONS)

  if options.force_version and args[0] not in (constants.CHROME_REV_SPEC,
                                               constants.CHROME_REV_LATEST):
    parser.error('--force_version is not compatible with the %r '
                 'option.' % (args[0],))

  overlay_dir = os.path.abspath(_OVERLAY_DIR % {'srcroot': options.srcroot})
  chrome_package_dir = os.path.join(overlay_dir, constants.CHROME_CP)
  chrome_rev = args[0]
  version_to_uprev = None
  commit_to_use = None
  sticky_branch = None

  (unstable_ebuild, stable_ebuilds) = FindChromeCandidates(chrome_package_dir)

  if chrome_rev == constants.CHROME_REV_LOCAL:
    if 'CHROME_ROOT' in os.environ:
      chrome_root = os.environ['CHROME_ROOT']
    else:
      chrome_root = os.path.join(os.environ['HOME'], 'chrome_root')

    version_to_uprev = _GetTipOfTrunkVersionFile(chrome_root)
    commit_to_use = 'Unknown'
    cros_build_lib.Info('Using local source, versioning is untrustworthy.')
  elif chrome_rev == constants.CHROME_REV_SPEC:
    if '.' in options.force_version:
      version_to_uprev = options.force_version
    else:
      commit_to_use = options.force_version
      if '@' in commit_to_use:
        commit_to_use = commit_to_use.rpartition('@')[2]
      version_to_uprev = _GetSpecificVersionUrl(options.chrome_url,
                                                commit_to_use)
  elif chrome_rev == constants.CHROME_REV_TOT:
    commit_to_use = gob_util.GetTipOfTrunkRevision(options.chrome_url)
    version_to_uprev = _GetSpecificVersionUrl(options.chrome_url,
                                              commit_to_use)
  elif chrome_rev == constants.CHROME_REV_LATEST:
    if options.force_version:
      if '.' not in options.force_version:
        parser.error('%s only accepts released Chrome versions, not SVN or '
                     'Git revisions.' % (chrome_rev,))
      version_to_uprev = options.force_version
    else:
      version_to_uprev = GetLatestRelease(options.chrome_url)
  else:
    sticky_ebuild = _GetStickyEBuild(stable_ebuilds)
    sticky_version = sticky_ebuild.chrome_version
    sticky_branch = sticky_version.rpartition('.')[0]
    version_to_uprev = GetLatestRelease(options.chrome_url, sticky_branch)

  stable_candidate = FindChromeUprevCandidate(stable_ebuilds, chrome_rev,
                                              sticky_branch)

  if stable_candidate:
    cros_build_lib.Info('Stable candidate found %s' % stable_candidate)
  else:
    cros_build_lib.Info('No stable candidate found.')

  tracking_branch = 'remotes/m/%s' % os.path.basename(options.tracking_branch)
  existing_branch = git.GetCurrentBranch(chrome_package_dir)
  work_branch = cros_mark_as_stable.GitBranch(constants.STABLE_EBUILD_BRANCH,
                                              tracking_branch,
                                              chrome_package_dir)
  work_branch.CreateBranch()

  # In the case of uprevving overlays that have patches applied to them,
  # include the patched changes in the stabilizing branch.
  if existing_branch:
    git.RunGit(chrome_package_dir, ['rebase', existing_branch])

  chrome_version_atom = MarkChromeEBuildAsStable(
      stable_candidate, unstable_ebuild, 'chromeos-chrome', chrome_rev,
      version_to_uprev, commit_to_use, chrome_package_dir)
  if chrome_version_atom:
    if options.boards:
      cros_mark_as_stable.CleanStalePackages(options.boards.split(':'),
                                             [chrome_version_atom])

    # If we did rev Chrome, now is a good time to uprev other packages.
    for other_ebuild in constants.OTHER_CHROME_PACKAGES:
      other_ebuild_name = os.path.basename(other_ebuild)
      other_package_dir = os.path.join(overlay_dir, other_ebuild)
      (other_unstable_ebuild, other_stable_ebuilds) = FindChromeCandidates(
          other_package_dir)
      other_stable_candidate = FindChromeUprevCandidate(other_stable_ebuilds,
                                                        chrome_rev,
                                                        sticky_branch)
      revved_atom = MarkChromeEBuildAsStable(other_stable_candidate,
                                             other_unstable_ebuild,
                                             other_ebuild_name,
                                             chrome_rev, version_to_uprev,
                                             commit_to_use, other_package_dir)
      if revved_atom and options.boards:
        cros_mark_as_stable.CleanStalePackages(options.boards.split(':'),
                                               [revved_atom])

  # Explicit print to communicate to caller.
  if chrome_version_atom:
    print 'CHROME_VERSION_ATOM=%s' % chrome_version_atom