scripts/cbuildbot.py

# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Main builder code for Chromium OS.

Used by Chromium OS buildbot configuration for all Chromium OS builds including
full and pre-flight-queue builds.
"""

import distutils.version  # pylint: disable=import-error,no-name-in-module
import glob
import json
import logging
import optparse  # pylint: disable=deprecated-module
import os
import pickle
import sys

from chromite.cbuildbot import builders
from chromite.cbuildbot import cbuildbot_alerts
from chromite.cbuildbot import cbuildbot_run
from chromite.cbuildbot import repository
from chromite.cbuildbot import topology
from chromite.cbuildbot.stages import completion_stages
from chromite.cbuildbot.stages import test_stages
from chromite.lib import builder_status_lib
from chromite.lib import cgroups
from chromite.lib import cidb
from chromite.lib import cleanup
from chromite.lib import commandline
from chromite.lib import config_lib
from chromite.lib import constants
from chromite.lib import cros_build_lib
from chromite.lib import failures_lib
from chromite.lib import git
from chromite.lib import gob_util
from chromite.lib import osutils
from chromite.lib import parallel
from chromite.lib import retry_stats
from chromite.lib import sudo
from chromite.lib import tee
from chromite.lib import timeout_util
from chromite.lib import ts_mon_config
from chromite.lib.buildstore import BuildStore


_DEFAULT_LOG_DIR = 'cbuildbot_logs'
_BUILDBOT_LOG_FILE = 'cbuildbot.log'
_DEFAULT_EXT_BUILDROOT = 'trybot'
_DEFAULT_INT_BUILDROOT = 'trybot-internal'
_BUILDBOT_REQUIRED_BINARIES = ('pbzip2',)
_API_VERSION_ATTR = 'api_version'
BOARD_DIM_LABEL = 'label-board'
MODEL_DIM_LABEL = 'label-model'
POOL_DIM_LABEL = 'label-pool'


def _BackupPreviousLog(log_file, backup_limit=25):
  """Rename previous log.

  Args:
    log_file: The absolute path to the previous log.
    backup_limit: Maximum number of old logs to keep.
  """
  if os.path.exists(log_file):
    old_logs = sorted(glob.glob(log_file + '.*'),
                      key=distutils.version.LooseVersion)

    if len(old_logs) >= backup_limit:
      os.remove(old_logs[0])

    last = 0
    if old_logs:
      last = int(old_logs.pop().rpartition('.')[2])

    os.rename(log_file, log_file + '.' + str(last + 1))


def _IsDistributedBuilder(options, chrome_rev, build_config):
  """Determines whether the builder should be a DistributedBuilder.

  Args:
    options: options passed on the commandline.
    chrome_rev: Chrome revision to build.
    build_config: Builder configuration dictionary.

  Returns:
    True if the builder should be a distributed_builder
  """
  if not options.buildbot:
    return False
  elif chrome_rev in (constants.CHROME_REV_TOT,
                      constants.CHROME_REV_LOCAL,
                      constants.CHROME_REV_SPEC):
    # We don't do distributed logic to TOT Chrome PFQ's, nor local
    # chrome roots (e.g. chrome try bots)
    # TODO(davidjames): Update any builders that rely on this logic to use
    # manifest_version=False instead.
    return False
  elif build_config['manifest_version']:
    return True

  return False


def _RunBuildStagesWrapper(options, site_config, build_config):
  """Helper function that wraps RunBuildStages()."""
  logging.info('cbuildbot was executed with args %s',
               cros_build_lib.CmdToStr(sys.argv))

  chrome_rev = build_config['chrome_rev']
  if options.chrome_rev:
    chrome_rev = options.chrome_rev
  if chrome_rev == constants.CHROME_REV_TOT:
    options.chrome_version = gob_util.GetTipOfTrunkRevision(
        constants.CHROMIUM_GOB_URL)
    options.chrome_rev = constants.CHROME_REV_SPEC

  # If it's likely we'll need to build Chrome, fetch the source.
  if build_config['sync_chrome'] is None:
    options.managed_chrome = (
        chrome_rev != constants.CHROME_REV_LOCAL and
        (not build_config['usepkg_build_packages'] or chrome_rev or
         build_config['profile']))
  else:
    options.managed_chrome = build_config['sync_chrome']

  chrome_root_mgr = None
  if options.managed_chrome:
    # Create a temp directory for syncing Chrome source.
    chrome_root_mgr = osutils.TempDir(prefix='chrome_root_')
    options.chrome_root = chrome_root_mgr.tempdir

  # We are done munging options values, so freeze options object now to avoid
  # further abuse of it.
  # TODO(mtennant): one by one identify each options value override and see if
  # it can be handled another way.  Try to push this freeze closer and closer
  # to the start of the script (e.g. in or after _PostParseCheck).
  options.Freeze()

  metadata_dump_dict = {
      # A detected default has been set before now if it wasn't explicit.
      'branch': options.branch,
  }
  if options.metadata_dump:
    with open(options.metadata_dump, 'r') as metadata_file:
      metadata_dump_dict = json.loads(metadata_file.read())

  with parallel.Manager() as manager:
    builder_run = cbuildbot_run.BuilderRun(
        options, site_config, build_config, manager)
    buildstore = BuildStore()
    if metadata_dump_dict:
      builder_run.attrs.metadata.UpdateWithDict(metadata_dump_dict)

    if builder_run.config.builder_class_name is None:
      # TODO: This should get relocated to chromeos_config.
      if _IsDistributedBuilder(options, chrome_rev, build_config):
        builder_cls_name = 'simple_builders.DistributedBuilder'
      else:
        builder_cls_name = 'simple_builders.SimpleBuilder'
      builder_cls = builders.GetBuilderClass(builder_cls_name)
      builder = builder_cls(builder_run, buildstore)
    else:
      builder = builders.Builder(builder_run, buildstore)

    try:
      if not builder.Run():
        sys.exit(1)
    finally:
      if chrome_root_mgr:
        chrome_root_mgr.Cleanup()


def _CheckChromeVersionOption(_option, _opt_str, value, parser):
  """Upgrade other options based on chrome_version being passed."""
  value = value.strip()

  if parser.values.chrome_rev is None and value:
    parser.values.chrome_rev = constants.CHROME_REV_SPEC

  parser.values.chrome_version = value


def _CheckChromeRootOption(_option, _opt_str, value, parser):
  """Validate and convert chrome_root to full-path form."""
  if parser.values.chrome_rev is None:
    parser.values.chrome_rev = constants.CHROME_REV_LOCAL

  parser.values.chrome_root = value


def FindCacheDir(_parser, _options):
  return None


class CustomGroup(optparse.OptionGroup):
  """Custom option group which supports arguments passed-through to trybot."""

  def add_remote_option(self, *args, **kwargs):
    """For arguments that are passed-through to remote trybot."""
    return optparse.OptionGroup.add_option(self, *args,
                                           remote_pass_through=True,
                                           **kwargs)


class CustomOption(commandline.FilteringOption):
  """Subclass FilteringOption class to implement pass-through and api."""

  def __init__(self, *args, **kwargs):
    # The remote_pass_through argument specifies whether we should directly
    # pass the argument (with its value) onto the remote trybot.
    self.pass_through = kwargs.pop('remote_pass_through', False)
    self.api_version = int(kwargs.pop('api', '0'))
    commandline.FilteringOption.__init__(self, *args, **kwargs)


class CustomParser(commandline.FilteringParser):
  """Custom option parser which supports arguments passed-through to trybot"""

  DEFAULT_OPTION_CLASS = CustomOption

  def add_remote_option(self, *args, **kwargs):
    """For arguments that are passed-through to remote trybot."""
    return self.add_option(*args, remote_pass_through=True, **kwargs)


def CreateParser():
  """Expose _CreateParser publicly."""
  # Name _CreateParser is needed for commandline library.
  return _CreateParser()


def _CreateParser():
  """Generate and return the parser with all the options."""
  # Parse options
  usage = 'usage: %prog [options] buildbot_config [buildbot_config ...]'
  parser = CustomParser(usage=usage, caching=FindCacheDir)

  # Main options
  parser.add_remote_option('-b', '--branch',
                           help='The manifest branch to test.  The branch to '
                                'check the buildroot out to.')
  parser.add_option('-r', '--buildroot', type='path', dest='buildroot',
                    help='Root directory where source is checked out to, and '
                         'where the build occurs. For external build configs, '
                         "defaults to 'trybot' directory at top level of your "
                         'repo-managed checkout.')
  parser.add_option('--workspace', type='path',
                    api=constants.REEXEC_API_WORKSPACE,
                    help='Root directory for a secondary checkout .')
  parser.add_option('--bootstrap-dir', type='path',
                    help='Bootstrapping cbuildbot may involve checking out '
                         'multiple copies of chromite. All these checkouts '
                         'will be contained in the directory specified here. '
                         'Default:%s' % osutils.GetGlobalTempDir())
  parser.add_remote_option('--android_rev', type='choice',
                           choices=constants.VALID_ANDROID_REVISIONS,
                           help=('Revision of Android to use, of type [%s]'
                                 % '|'.join(constants.VALID_ANDROID_REVISIONS)))
  parser.add_remote_option('--chrome_rev', type='choice',
                           choices=constants.VALID_CHROME_REVISIONS,
                           help=('Revision of Chrome to use, of type [%s]'
                                 % '|'.join(constants.VALID_CHROME_REVISIONS)))
  parser.add_remote_option('--profile',
                           help='Name of profile to sub-specify board variant.')
  # TODO(crbug.com/279618): Running GOMA is under development. Following
  # flags are added for development purpose due to repository dependency,
  # but not officially supported yet.
  parser.add_option('--goma_dir', type='path',
                    api=constants.REEXEC_API_GOMA,
                    help='Specify a directory containing goma. When this is '
                         'set, GOMA is used to build Chrome.')
  parser.add_option('--chromeos_goma_dir', type='path',
                    api=constants.REEXEC_API_CHROMEOS_GOMA_DIR,
                    help='Specify a directory containing goma for '
                         'build package.')
  # TODO(crbug.com/1359171): cleanup the flag.
  parser.add_option('--goma_client_json', type='path',
                    api=constants.REEXEC_API_GOMA,
                    help='Specify a service-account-goma-client.json path.')

  group = CustomGroup(
      parser,
      'Deprecated Options')

  parser.add_option('--local', action='store_true', default=False,
                    help='Deprecated. See cros tryjob.')
  parser.add_option('--remote', action='store_true', default=False,
                    help='Deprecated. See cros tryjob.')

  #
  # Patch selection options.
  #

  group = CustomGroup(
      parser,
      'Patch Options')

  group.add_remote_option('-g', '--gerrit-patches', action='split_extend',
                          type='string', default=[],
                          metavar="'Id1 *int_Id2...IdN'",
                          help='Space-separated list of short-form Gerrit '
                               "Change-Id's or change numbers to patch. "
                               "Please prepend '*' to internal Change-Id's")

  parser.add_argument_group(group)

  #
  # Remote trybot options.
  #

  group = CustomGroup(
      parser,
      'Options used to configure tryjob behavior.')
  group.add_remote_option('--hwtest', action='store_true', default=False,
                          help='Run the HWTest stage (tests on real hardware)')
  group.add_option('--hwtest_dut_dimensions', type='string',
                   action='split_extend', default=None,
                   help='Space-separated list of key:val Swarming bot '
                        'dimensions to run each builders SkylabHWTest '
                        'stages against (this overrides the configured '
                        'DUT dimensions for each test). Requires at least '
                        '"label-board", "label-model", and "label-pool".')
  group.add_remote_option('--channel', action='split_extend', dest='channels',
                          default=[],
                          help='Specify a channel for a payloads trybot. Can '
                               'be specified multiple times. No valid for '
                               'non-payloads configs.')

  parser.add_argument_group(group)

  #
  # Advanced options.
  #

  group = CustomGroup(
      parser,
      'Advanced Options',
      'Caution: use these options at your own risk.')

  group.add_remote_option('--bootstrap-args', action='append', default=[],
                          help='Args passed directly to the bootstrap re-exec '
                               'to skip verification by the bootstrap code')
  group.add_remote_option('--buildbot', action='store_true', dest='buildbot',
                          default=False,
                          help='This is running on a buildbot. '
                               'This can be used to make a build operate '
                               'like an official builder, e.g. generate '
                               'new version numbers and archive official '
                               'artifacts and such. This should only be '
                               'used if you are confident in what you are '
                               'doing, as it will make automated commits.')
  parser.add_remote_option('--repo-cache', type='path', dest='_repo_cache',
                           help='Present for backwards compatibility, ignored.')
  group.add_remote_option('--no-buildbot-tags', action='store_false',
                          dest='enable_buildbot_tags', default=True,
                          help='Suppress buildbot specific tags from log '
                               'output. This is used to hide recursive '
                               'cbuilbot runs on the waterfall.')
  group.add_remote_option('--buildnumber', type='int', default=0,
                          help='build number')
  group.add_option('--chrome_root', action='callback', type='path',
                   callback=_CheckChromeRootOption,
                   help='Local checkout of Chrome to use.')
  group.add_remote_option('--chrome_version', action='callback', type='string',
                          dest='chrome_version',
                          callback=_CheckChromeVersionOption,
                          help='Used with SPEC logic to force a particular '
                               'git revision of chrome rather than the '
                               'latest.')
  group.add_remote_option('--clobber', action='store_true', default=False,
                          help='Clears an old checkout before syncing')
  group.add_remote_option('--latest-toolchain', action='store_true',
                          default=False,
                          help='Use the latest toolchain.')
  parser.add_option('--log_dir', dest='log_dir', type='path',
                    help='Directory where logs are stored.')
  group.add_remote_option('--maxarchives', type='int',
                          dest='max_archive_builds', default=3,
                          help='Change the local saved build count limit.')
  parser.add_remote_option('--manifest-repo-url',
                           help='Overrides the default manifest repo url.')
  group.add_remote_option('--compilecheck', action='store_true', default=False,
                          help='Only verify compilation and unit tests.')
  group.add_remote_option('--noarchive', action='store_false', dest='archive',
                          default=True, help="Don't run archive stage.")
  group.add_remote_option('--nobootstrap', action='store_false',
                          dest='bootstrap', default=True,
                          help="Don't checkout and run from a standalone "
                               'chromite repo.')
  group.add_remote_option('--nobuild', action='store_false', dest='build',
                          default=True,
                          help="Don't actually build (for cbuildbot dev)")
  group.add_remote_option('--noclean', action='store_false', dest='clean',
                          default=True, help="Don't clean the buildroot")
  group.add_remote_option('--nocgroups', action='store_false', dest='cgroups',
                          default=True,
                          help='Disable cbuildbots usage of cgroups.')
  group.add_remote_option('--nochromesdk', action='store_false',
                          dest='chrome_sdk', default=True,
                          help="Don't run the ChromeSDK stage which builds "
                               'Chrome outside of the chroot.')
  group.add_remote_option('--noprebuilts', action='store_false',
                          dest='prebuilts', default=True,
                          help="Don't upload prebuilts.")
  group.add_remote_option('--nopatch', action='store_false',
                          dest='postsync_patch', default=True,
                          help="Don't run PatchChanges stage.  This does not "
                               'disable patching in of chromite patches '
                               'during BootstrapStage.')
  group.add_remote_option('--nopaygen', action='store_false',
                          dest='paygen', default=True,
                          help="Don't generate payloads.")
  group.add_remote_option('--noreexec', action='store_false',
                          dest='postsync_reexec', default=True,
                          help="Don't reexec into the buildroot after syncing.")
  group.add_remote_option('--nosdk', action='store_true', default=False,
                          help='Re-create the SDK from scratch.')
  group.add_remote_option('--nosync', action='store_false', dest='sync',
                          default=True, help="Don't sync before building.")
  group.add_remote_option('--notests', action='store_false', dest='tests',
                          default=True,
                          help='Override values from buildconfig, run no '
                               'tests, and build no autotest and artifacts.')
  group.add_remote_option('--novmtests', action='store_false', dest='vmtests',
                          default=True,
                          help='Override values from buildconfig, run no '
                               'vmtests.')
  group.add_remote_option('--noimagetests', action='store_false',
                          dest='image_test', default=True,
                          help='Override values from buildconfig and run no '
                               'image tests.')
  group.add_remote_option('--nouprev', action='store_false', dest='uprev',
                          default=True,
                          help='Override values from buildconfig and never '
                               'uprev.')
  group.add_option('--reference-repo',
                   help='Reuse git data stored in an existing repo '
                        'checkout. This can drastically reduce the network '
                        'time spent setting up the trybot checkout.  By '
                        "default, if this option isn't given but cbuildbot "
                        'is invoked from a repo checkout, cbuildbot will '
                        'use the repo root.')
  group.add_option('--resume', action='store_true', default=False,
                   help='Skip stages already successfully completed.')
  group.add_remote_option('--timeout', type='int', default=0,
                          help='Specify the maximum amount of time this job '
                               'can run for, at which point the build will be '
                               'aborted.  If set to zero, then there is no '
                               'timeout.')
  group.add_remote_option('--version', dest='force_version',
                          help='Used with manifest logic.  Forces use of this '
                               'version rather than create or get latest. '
                               'Examples: 4815.0.0-rc1, 4815.1.2')
  group.add_remote_option('--git-cache-dir', type='path',
                          api=constants.REEXEC_API_GIT_CACHE_DIR,
                          help='Specify the cache directory to store the '
                               'project caches populated by the git-cache '
                               'tool. Bootstrap the projects based on the git '
                               'cache files instead of fetching them directly '
                               'from the GoB servers.')
  group.add_remote_option('--chrome-preload-dir', type='path',
                          api=constants.REEXEC_API_CHROME_PRELOAD_DIR,
                          help='Specify a preloaded chrome source cache '
                               'directory populated by the git-cache tool. '
                               'Bootstrap chrome based on the cached files '
                               'instead of fetching them directly from the GoB '
                               'servers. When both this argument and '
                               '--git-cache-dir are provided this value will '
                               'be preferred for the chrome source cache.')
  group.add_remote_option('--source_cache', action='store_true', default=False,
                          help='Whether to utilize cache snapshot mounts.')
  group.add_remote_option('--debug-cidb', action='store_true', default=False,
                          help='Force Debug CIDB to be used.')
  # cbuildbot ChromeOS Findit options
  group.add_remote_option('--cbb_build_packages', action='split_extend',
                          dest='cbb_build_packages',
                          default=[],
                          help='Specify an explicit list of packages to build '
                               'for integration with Findit.')
  group.add_remote_option('--cbb_snapshot_revision', type='string',
                          dest='cbb_snapshot_revision', default=None,
                          help='Snapshot manifest revision to sync to '
                               'for building.')
  group.add_remote_option(
      '--no-publish-prebuilt-confs',
      dest='publish',
      action='store_false',
      default=True,
      help="Don't publish git commits to prebuilt.conf or sdk_version.conf")

  parser.add_argument_group(group)

  #
  # Internal options.
  #

  group = CustomGroup(
      parser,
      'Internal Chromium OS Build Team Options',
      'Caution: these are for meant for the Chromium OS build team only')

  group.add_remote_option('--archive-base', type='gs_path',
                          help='Base GS URL (gs://<bucket_name>/<path>) to '
                               'upload archive artifacts to')
  group.add_remote_option(
      '--cq-gerrit-query', dest='cq_gerrit_override',
      help='If given, this gerrit query will be used to find what patches to '
           "test, rather than the normal 'CommitQueue>=1 AND Verified=1 AND "
           "CodeReview=2' query it defaults to.  Use with care- note "
           'additionally this setting only has an effect if the buildbot '
           "target is a cq target, and we're in buildbot mode.")
  group.add_option('--pass-through', action='append', type='string',
                   dest='pass_through_args', default=[])
  group.add_option('--reexec-api-version', action='store_true',
                   dest='output_api_version', default=False,
                   help='Used for handling forwards/backwards compatibility '
                        'with --resume and --bootstrap')
  group.add_option('--remote-trybot', action='store_true', default=False,
                   help='Indicates this is running on a remote trybot machine')
  group.add_option('--buildbucket-id',
                   api=constants.REEXEC_API_GOMA, # Approximate.
                   help='The unique ID in buildbucket of current build '
                        'generated by buildbucket.')
  group.add_remote_option('--remote-patches', action='split_extend', default=[],
                          help='Patches uploaded by the trybot client when '
                               'run using the -p option')
  # Note the default here needs to be hardcoded to 3; that is the last version
  # that lacked this functionality.
  group.add_option('--remote-version', type='int', default=3,
                   help='Deprecated and ignored.')
  group.add_option('--sourceroot', type='path', default=constants.SOURCE_ROOT)
  group.add_remote_option('--test-bootstrap', action='store_true',
                          default=False,
                          help='Causes cbuildbot to bootstrap itself twice, '
                               'in the sequence A->B->C: A(unpatched) patches '
                               'and bootstraps B; B patches and bootstraps C')
  group.add_remote_option('--validation_pool',
                          help='Path to a pickled validation pool. Intended '
                               'for use only with the commit queue.')
  group.add_remote_option('--metadata_dump',
                          help='Path to a json dumped metadata file. This '
                               'will be used as the initial metadata.')
  group.add_remote_option('--master-build-id', type='int',
                          api=constants.REEXEC_API_MASTER_BUILD_ID,
                          help='cidb build id of the master build to this '
                               'slave build.')
  group.add_remote_option('--master-buildbucket-id',
                          api=constants.REEXEC_API_MASTER_BUILDBUCKET_ID,
                          help='buildbucket id of the master build to this '
                               'slave build.')
  # TODO(nxia): crbug.com/778838
  # cbuildbot doesn't use pickle files anymore, remove this.
  group.add_remote_option('--mock-slave-status',
                          metavar='MOCK_SLAVE_STATUS_PICKLE_FILE',
                          help='Override the result of the _FetchSlaveStatuses '
                               'method of MasterSlaveSyncCompletionStage, by '
                               'specifying a file with a pickle of the result '
                               'to be returned.')
  group.add_option('--previous-build-state', type='string', default='',
                   api=constants.REEXEC_API_PREVIOUS_BUILD_STATE,
                   help='A base64-encoded BuildSummary object describing the '
                        'previous build run on the same build machine.')

  parser.add_argument_group(group)

  #
  # Debug options
  #
  # Temporary hack; in place till --dry-run replaces --debug.
  # pylint: disable=protected-access
  group = parser.debug_group
  debug = [x for x in group.option_list if x._long_opts == ['--debug']][0]
  debug.help += '  Currently functions as --dry-run in addition.'
  debug.pass_through = True
  group.add_option('--notee', action='store_false', dest='tee', default=True,
                   help='Disable logging and internal tee process.  Primarily '
                        'used for debugging cbuildbot itself.')
  return parser


def _FinishParsing(options):
  """Perform some parsing tasks that need to take place after optparse.

  This function needs to be easily testable!  Keep it free of
  environment-dependent code.  Put more detailed usage validation in
  _PostParseCheck().

  Args:
    options: The options object returned by optparse
  """
  # Populate options.pass_through_args.
  accepted, _ = commandline.FilteringParser.FilterArgs(
      options.parsed_args, lambda x: x.opt_inst.pass_through)
  options.pass_through_args.extend(accepted)

  if options.local or options.remote:
    cros_build_lib.Die('Deprecated usage. Please use cros tryjob instead.')

  if not options.buildroot:
    cros_build_lib.Die('A buildroot is required to build.')

  if options.chrome_root:
    if options.chrome_rev != constants.CHROME_REV_LOCAL:
      cros_build_lib.Die('Chrome rev must be %s if chrome_root is set.' %
                         constants.CHROME_REV_LOCAL)
  elif options.chrome_rev == constants.CHROME_REV_LOCAL:
    cros_build_lib.Die('Chrome root must be set if chrome_rev is %s.' %
                       constants.CHROME_REV_LOCAL)

  if options.chrome_version:
    if options.chrome_rev != constants.CHROME_REV_SPEC:
      cros_build_lib.Die('Chrome rev must be %s if chrome_version is set.' %
                         constants.CHROME_REV_SPEC)
  elif options.chrome_rev == constants.CHROME_REV_SPEC:
    cros_build_lib.Die(
        'Chrome rev must not be %s if chrome_version is not set.'
        % constants.CHROME_REV_SPEC)

  patches = bool(options.gerrit_patches)

  # When running in release mode, make sure we are running with checked-in code.
  # We want checked-in cbuildbot/scripts to prevent errors, and we want to build
  # a release image with checked-in code for CrOS packages.
  if options.buildbot and patches and not options.debug:
    cros_build_lib.Die(
        'Cannot provide patches when running with --buildbot!')

  if options.buildbot and options.remote_trybot:
    cros_build_lib.Die(
        '--buildbot and --remote-trybot cannot be used together.')

  # Record whether --debug was set explicitly vs. it was inferred.
  options.debug_forced = options.debug
  # We force --debug to be set for builds that are not 'official'.
  options.debug = options.debug or not options.buildbot

  options.hwtest_dut_override = ParseHWTestDUTDims(
    options.hwtest_dut_dimensions)

def ParseHWTestDUTDims(dims):
  """Parse HWTest DUT dimensions into a valid HWTestDUTOverride object.

  Raises an error if any of board, model, or pool is missing.
  """
  if not dims:
    return None
  board = model = pool = None
  extra_dims = []
  for dim in dims:
    if dim.startswith(BOARD_DIM_LABEL):
      # Remove one extra character to account for the ":" or "=" symbol
      # separating the label from the dimension itself.
      board = dim[len(BOARD_DIM_LABEL)+1:]
    elif dim.startswith(MODEL_DIM_LABEL):
      model = dim[len(MODEL_DIM_LABEL)+1:]
    elif dim.startswith(POOL_DIM_LABEL):
      pool = dim[len(POOL_DIM_LABEL)+1:]
    else:
      extra_dims.append(dim)

  if not (board and model and pool):
    cros_build_lib.Die('HWTest DUT dimensions must include board, model, and '
                       'pool (given %s).' % dims)

  return test_stages.HWTestDUTOverride(board, model, pool, extra_dims)


# pylint: disable=unused-argument
def _PostParseCheck(parser, options, site_config):
  """Perform some usage validation after we've parsed the arguments

  Args:
    parser: Option parser that was used to parse arguments.
    options: The options returned by optparse.
    site_config: config_lib.SiteConfig containing all config info.
  """

  if not options.branch:
    options.branch = git.GetChromiteTrackingBranch()

  # Because the default cache dir depends on other options, FindCacheDir
  # always returns None, and we setup the default here.
  if options.cache_dir is None:
    # Note, options.sourceroot is set regardless of the path
    # actually existing.
    options.cache_dir = os.path.join(options.buildroot, '.cache')
    options.cache_dir = os.path.abspath(options.cache_dir)
    parser.ConfigureCacheDir(options.cache_dir)

  osutils.SafeMakedirsNonRoot(options.cache_dir)

  # Ensure that all args are legitimate config targets.
  if options.build_config_name not in site_config:
    cros_build_lib.Die('Unknown build config: "%s"' % options.build_config_name)

  build_config = site_config[options.build_config_name]
  is_payloads_build = build_config.build_type == constants.PAYLOADS_TYPE

  if options.channels and not is_payloads_build:
    cros_build_lib.Die('--channel must only be used with a payload config,'
                       ' not target (%s).' % options.build_config_name)

  if not options.channels and is_payloads_build:
    cros_build_lib.Die('payload configs (%s) require --channel to do anything'
                       ' useful.' % options.build_config_name)

  # If the build config explicitly forces the debug flag, set the debug flag
  # as if it was set from the command line.
  if build_config.debug:
    options.debug = True

  if not (config_lib.isTryjobConfig(build_config) or options.buildbot):
    cros_build_lib.Die(
        'Refusing to run non-tryjob config as a tryjob.\n'
        'Please "repo sync && cros tryjob --list %s" for alternatives.\n'
        'See go/cros-explicit-tryjob-build-configs-psa.',
        build_config.name)

  # The --version option is not compatible with an external target unless the
  # --buildbot option is specified.  More correctly, only "paladin versions"
  # will work with external targets, and those are only used with --buildbot.
  # If --buildbot is specified, then user should know what they are doing and
  # only specify a version that will work.  See crbug.com/311648.
  if (options.force_version and
      not (options.buildbot or build_config.internal)):
    cros_build_lib.Die('Cannot specify --version without --buildbot for an'
                       ' external target (%s).' % options.build_config_name)


def ParseCommandLine(parser, argv):
  """Completely parse the commandline arguments"""
  (options, args) = parser.parse_args(argv)

  # Handle the request for the reexec command line API version number.
  if options.output_api_version:
    print(constants.REEXEC_API_VERSION)
    sys.exit(0)

  # Record the configs targeted. Strip out null arguments.
  build_config_names = [x for x in args if x]
  if len(build_config_names) != 1:
    cros_build_lib.Die('Expected exactly one build config. Got: %r',
                       build_config_names)
  options.build_config_name = build_config_names[-1]

  _FinishParsing(options)
  return options


_ENVIRONMENT_PROD = 'prod'
_ENVIRONMENT_DEBUG = 'debug'
_ENVIRONMENT_STANDALONE = 'standalone'


def _GetRunEnvironment(options, build_config):
  """Determine whether this is a prod/debug/standalone run."""
  if options.debug_cidb:
    return _ENVIRONMENT_DEBUG

  # One of these arguments should always be set if running on a real builder.
  # If we aren't on a real builder, we are standalone.
  if not options.buildbot and not options.remote_trybot:
    return _ENVIRONMENT_STANDALONE

  if build_config['debug_cidb']:
    return _ENVIRONMENT_DEBUG

  return _ENVIRONMENT_PROD


def _SetupConnections(options, build_config):
  """Set up CIDB connections using the appropriate Setup call.

  Args:
    options: Command line options structure.
    build_config: Config object for this build.
  """
  # Outline:
  # 1) Based on options and build_config, decide whether we are a production
  # run, debug run, or standalone run.
  # 2) Set up cidb instance accordingly.
  # 3) Update topology info from cidb, so that any other service set up can use
  # topology.
  # 4) Set up any other services.
  run_type = _GetRunEnvironment(options, build_config)

  if run_type == _ENVIRONMENT_PROD:
    cidb.CIDBConnectionFactory.SetupProdCidb()
    context = ts_mon_config.SetupTsMonGlobalState('cbuildbot', indirect=True)
  elif run_type == _ENVIRONMENT_DEBUG:
    cidb.CIDBConnectionFactory.SetupDebugCidb()
    context = ts_mon_config.TrivialContextManager()
  else:
    cidb.CIDBConnectionFactory.SetupNoCidb()
    context = ts_mon_config.TrivialContextManager()

  topology.FetchTopology()
  return context


class _MockMethodWithReturnValue(object):
  """A method mocker which just returns the specific value."""
  def __init__(self, return_value):
    self.return_value = return_value

  def __call__(self, *args, **kwargs):
    return self.return_value


class _ObjectMethodPatcher(object):
  """A simplified mock.object.patch.

  It is a context manager that patches an object's method with specified
  return value.
  """
  def __init__(self, target, attr, return_value=None):
    """Constructor.

    Args:
      target: object to patch.
      attr: method name of the object to patch.
      return_value: the return value when calling target.attr
    """
    self.target = target
    self.attr = attr
    self.return_value = return_value
    self.original_attr = None
    self.new_attr = _MockMethodWithReturnValue(self.return_value)

  def __enter__(self):
    self.original_attr = self.target.__dict__[self.attr]
    setattr(self.target, self.attr, self.new_attr)

  def __exit__(self, *args):
    if self.target and self.original_attr:
      setattr(self.target, self.attr, self.original_attr)


# TODO(build): This function is too damn long.
def main(argv):
  # We get false positives with the options object.
  # pylint: disable=attribute-defined-outside-init

  # Turn on strict sudo checks.
  cros_build_lib.STRICT_SUDO = True

  # Set umask to 022 so files created by buildbot are readable.
  os.umask(0o22)

  parser = _CreateParser()
  options = ParseCommandLine(parser, argv)

  # Fetch our site_config now, because we need it to do anything else.
  site_config = config_lib.GetConfig()

  _PostParseCheck(parser, options, site_config)

  cros_build_lib.AssertOutsideChroot()

  if options.enable_buildbot_tags:
    cbuildbot_alerts.EnableBuildbotMarkers()

  if (options.buildbot and
      not options.debug and
      not cros_build_lib.HostIsCIBuilder()):
    # --buildbot can only be used on a real builder, unless it's debug.
    cros_build_lib.Die('This host is not a supported build machine.')

  # Only one config arg is allowed in this mode, which was confirmed earlier.
  build_config = site_config[options.build_config_name]

  # TODO: Re-enable this block when reference_repo support handles this
  #       properly. (see chromium:330775)
  # if options.reference_repo is None:
  #   repo_path = os.path.join(options.sourceroot, '.repo')
  #   # If we're being run from a repo checkout, reuse the repo's git pool to
  #   # cut down on sync time.
  #   if os.path.exists(repo_path):
  #     options.reference_repo = options.sourceroot

  if options.reference_repo:
    if not os.path.exists(options.reference_repo):
      parser.error('Reference path %s does not exist'
                   % (options.reference_repo,))
    elif not os.path.exists(os.path.join(options.reference_repo, '.repo')):
      parser.error('Reference path %s does not look to be the base of a '
                   'repo checkout; no .repo exists in the root.'
                   % (options.reference_repo,))

  if (options.buildbot or options.remote_trybot) and not options.resume:
    missing = osutils.FindMissingBinaries(_BUILDBOT_REQUIRED_BINARIES)
    if missing:
      parser.error('Option --buildbot/--remote-trybot requires the following '
                   "binaries which couldn't be found in $PATH: %s"
                   % (', '.join(missing)))

  if options.reference_repo:
    options.reference_repo = os.path.abspath(options.reference_repo)

  # Sanity check of buildroot- specifically that it's not pointing into the
  # midst of an existing repo since git-repo doesn't support nesting.
  if (not repository.IsARepoRoot(options.buildroot) and
      git.FindRepoDir(options.buildroot)):
    cros_build_lib.Die(
        'Configured buildroot %s is a subdir of an existing repo checkout.'
        % options.buildroot)

  if not options.log_dir:
    options.log_dir = os.path.join(options.buildroot, _DEFAULT_LOG_DIR)

  log_file = None
  if options.tee:
    log_file = os.path.join(options.log_dir, _BUILDBOT_LOG_FILE)
    osutils.SafeMakedirs(options.log_dir)
    _BackupPreviousLog(log_file)

  with cros_build_lib.ContextManagerStack() as stack:
    # Preserve chromite; we might be running from there!
    options.preserve_paths = set(['chromite'])
    if log_file is not None:
      # We don't want the critical section to try to clean up the tee process,
      # so we run Tee (forked off) outside of it. This prevents a deadlock
      # because the Tee process only exits when its pipe is closed, and the
      # critical section accidentally holds on to that file handle.
      stack.Add(tee.Tee, log_file)
      options.preserve_paths.add(_DEFAULT_LOG_DIR)

    critical_section = stack.Add(cleanup.EnforcedCleanupSection)
    stack.Add(sudo.SudoKeepAlive)

    if not options.resume:
      # If we're in resume mode, use our parents tempdir rather than
      # nesting another layer.
      stack.Add(osutils.TempDir, prefix='cbuildbot-tmp', set_global=True)
      logging.debug('Cbuildbot tempdir is %r.', os.environ.get('TMP'))

    if options.cgroups:
      stack.Add(cgroups.SimpleContainChildren, 'cbuildbot')

    # Mark everything between EnforcedCleanupSection and here as having to
    # be rolled back via the contextmanager cleanup handlers.  This
    # ensures that sudo bits cannot outlive cbuildbot, that anything
    # cgroups would kill gets killed, etc.
    stack.Add(critical_section.ForkWatchdog)

    if options.mock_slave_status is not None:
      with open(options.mock_slave_status, 'r') as f:
        mock_statuses = pickle.load(f)
        for key, value in mock_statuses.items():
          mock_statuses[key] = builder_status_lib.BuilderStatus(**value)
      stack.Add(_ObjectMethodPatcher,
                completion_stages.MasterSlaveSyncCompletionStage,
                '_FetchSlaveStatuses',
                return_value=mock_statuses)

    stack.Add(_SetupConnections, options, build_config)
    retry_stats.SetupStats()

    timeout_display_message = (
        'This build has reached the timeout deadline set by the master. '
        'Either this stage or a previous one took too long (see stage '
        'timing historical summary in ReportStage) or the build failed '
        'to start on time.')

    if options.timeout > 0:
      stack.Add(timeout_util.FatalTimeout, options.timeout,
                timeout_display_message)
    try:
      _RunBuildStagesWrapper(options, site_config, build_config)
    except failures_lib.ExitEarlyException as ex:
      # This build finished successfully. Do not re-raise ExitEarlyException.
      logging.info('One stage exited early: %s', ex)