scripts/cbuildbot.py

# Copyright 2012 The ChromiumOS Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Main builder code for Chromium OS.

Used by Chromium OS buildbot configuration for all Chromium OS builds including
full and pre-flight-queue builds.
"""

import distutils.version  # pylint: disable=import-error,no-name-in-module
import glob
import json
import logging
import optparse  # pylint: disable=deprecated-module
import os
import pickle
import sys

from chromite.cbuildbot import builders
from chromite.cbuildbot import cbuildbot_alerts
from chromite.cbuildbot import cbuildbot_run
from chromite.cbuildbot import repository
from chromite.cbuildbot import topology
from chromite.cbuildbot.stages import completion_stages
from chromite.cbuildbot.stages import test_stages
from chromite.lib import builder_status_lib
from chromite.lib import cgroups
from chromite.lib import cidb
from chromite.lib import cleanup
from chromite.lib import commandline
from chromite.lib import config_lib
from chromite.lib import constants
from chromite.lib import cros_build_lib
from chromite.lib import failures_lib
from chromite.lib import git
from chromite.lib import gob_util
from chromite.lib import osutils
from chromite.lib import parallel
from chromite.lib import retry_stats
from chromite.lib import sudo
from chromite.lib import tee
from chromite.lib import timeout_util
from chromite.lib import ts_mon_config
from chromite.lib.buildstore import BuildStore


_DEFAULT_LOG_DIR = "cbuildbot_logs"
_BUILDBOT_LOG_FILE = "cbuildbot.log"
_DEFAULT_EXT_BUILDROOT = "trybot"
_DEFAULT_INT_BUILDROOT = "trybot-internal"
_BUILDBOT_REQUIRED_BINARIES = ("pbzip2",)
_API_VERSION_ATTR = "api_version"
BOARD_DIM_LABEL = "label-board"
MODEL_DIM_LABEL = "label-model"
POOL_DIM_LABEL = "label-pool"


def _BackupPreviousLog(log_file, backup_limit=25):
    """Rename previous log.

    Args:
      log_file: The absolute path to the previous log.
      backup_limit: Maximum number of old logs to keep.
    """
    if os.path.exists(log_file):
        old_logs = sorted(
            glob.glob(log_file + ".*"), key=distutils.version.LooseVersion
        )

        if len(old_logs) >= backup_limit:
            os.remove(old_logs[0])

        last = 0
        if old_logs:
            last = int(old_logs.pop().rpartition(".")[2])

        os.rename(log_file, log_file + "." + str(last + 1))


def _IsDistributedBuilder(options, chrome_rev, build_config):
    """Determines whether the builder should be a DistributedBuilder.

    Args:
      options: options passed on the commandline.
      chrome_rev: Chrome revision to build.
      build_config: Builder configuration dictionary.

    Returns:
      True if the builder should be a distributed_builder
    """
    if not options.buildbot:
        return False
    elif chrome_rev in (
        constants.CHROME_REV_TOT,
        constants.CHROME_REV_LOCAL,
        constants.CHROME_REV_SPEC,
    ):
        # We don't do distributed logic to TOT Chrome PFQ's, nor local
        # chrome roots (e.g. chrome try bots)
        # TODO(davidjames): Update any builders that rely on this logic to use
        # manifest_version=False instead.
        return False
    elif build_config["manifest_version"]:
        return True

    return False


def _RunBuildStagesWrapper(options, site_config, build_config):
    """Helper function that wraps RunBuildStages()."""
    logging.info(
        "cbuildbot was executed with args %s", cros_build_lib.CmdToStr(sys.argv)
    )

    chrome_rev = build_config["chrome_rev"]
    if options.chrome_rev:
        chrome_rev = options.chrome_rev
    if chrome_rev == constants.CHROME_REV_TOT:
        options.chrome_version = gob_util.GetTipOfTrunkRevision(
            constants.CHROMIUM_GOB_URL
        )
        options.chrome_rev = constants.CHROME_REV_SPEC

    # If it's likely we'll need to build Chrome, fetch the source.
    if build_config["sync_chrome"] is None:
        options.managed_chrome = chrome_rev != constants.CHROME_REV_LOCAL and (
            not build_config["usepkg_build_packages"]
            or chrome_rev
            or build_config["profile"]
        )
    else:
        options.managed_chrome = build_config["sync_chrome"]

    chrome_root_mgr = None
    if options.managed_chrome:
        # Create a temp directory for syncing Chrome source.
        chrome_root_mgr = osutils.TempDir(prefix="chrome_root_")
        options.chrome_root = chrome_root_mgr.tempdir

    # We are done munging options values, so freeze options object now to avoid
    # further abuse of it.
    # TODO(mtennant): one by one identify each options value override and see if
    # it can be handled another way.  Try to push this freeze closer and closer
    # to the start of the script (e.g. in or after _PostParseCheck).
    options.Freeze()

    metadata_dump_dict = {
        # A detected default has been set before now if it wasn't explicit.
        "branch": options.branch,
    }
    if options.metadata_dump:
        with open(options.metadata_dump, "rb") as metadata_file:
            metadata_dump_dict = json.load(metadata_file)

    with parallel.Manager() as manager:
        builder_run = cbuildbot_run.BuilderRun(
            options, site_config, build_config, manager
        )
        buildstore = BuildStore()
        if metadata_dump_dict:
            builder_run.attrs.metadata.UpdateWithDict(metadata_dump_dict)

        if builder_run.config.builder_class_name is None:
            # TODO: This should get relocated to chromeos_config.
            if _IsDistributedBuilder(options, chrome_rev, build_config):
                builder_cls_name = "simple_builders.DistributedBuilder"
            else:
                builder_cls_name = "simple_builders.SimpleBuilder"
            builder_cls = builders.GetBuilderClass(builder_cls_name)
            builder = builder_cls(builder_run, buildstore)
        else:
            builder = builders.Builder(builder_run, buildstore)

        try:
            if not builder.Run():
                sys.exit(1)
        finally:
            if chrome_root_mgr:
                chrome_root_mgr.Cleanup()


def _CheckChromeVersionOption(_option, _opt_str, value, parser):
    """Upgrade other options based on chrome_version being passed."""
    value = value.strip()

    if parser.values.chrome_rev is None and value:
        parser.values.chrome_rev = constants.CHROME_REV_SPEC

    parser.values.chrome_version = value


def _CheckChromeRootOption(_option, _opt_str, value, parser):
    """Validate and convert chrome_root to full-path form."""
    if parser.values.chrome_rev is None:
        parser.values.chrome_rev = constants.CHROME_REV_LOCAL

    parser.values.chrome_root = value


def FindCacheDir(_parser, _options):
    return None


class CustomGroup(optparse.OptionGroup):
    """Custom option group which supports arguments passed-through to trybot."""

    def add_remote_option(self, *args, **kwargs):
        """For arguments that are passed-through to remote trybot."""
        return optparse.OptionGroup.add_option(
            self, *args, remote_pass_through=True, **kwargs
        )


class CustomOption(commandline.FilteringOption):
    """Subclass FilteringOption class to implement pass-through and api."""

    def __init__(self, *args, **kwargs):
        # The remote_pass_through argument specifies whether we should directly
        # pass the argument (with its value) onto the remote trybot.
        self.pass_through = kwargs.pop("remote_pass_through", False)
        self.api_version = int(kwargs.pop("api", "0"))
        commandline.FilteringOption.__init__(self, *args, **kwargs)


class CustomParser(commandline.FilteringParser):
    """Custom option parser which supports arguments passed-through to trybot"""

    DEFAULT_OPTION_CLASS = CustomOption

    def add_remote_option(self, *args, **kwargs):
        """For arguments that are passed-through to remote trybot."""
        return self.add_option(*args, remote_pass_through=True, **kwargs)


def CreateParser():
    """Expose _CreateParser publicly."""
    # Name _CreateParser is needed for commandline library.
    return _CreateParser()


def _CreateParser():
    """Generate and return the parser with all the options."""
    # Parse options
    usage = "usage: %prog [options] buildbot_config [buildbot_config ...]"
    parser = CustomParser(usage=usage, caching=FindCacheDir)

    # Main options
    parser.add_remote_option(
        "-b",
        "--branch",
        help="The manifest branch to test.  The branch to "
        "check the buildroot out to.",
    )
    parser.add_option(
        "-r",
        "--buildroot",
        type="path",
        dest="buildroot",
        help="Root directory where source is checked out to, and "
        "where the build occurs. For external build configs, "
        "defaults to 'trybot' directory at top level of your "
        "repo-managed checkout.",
    )
    parser.add_option(
        "--workspace",
        type="path",
        api=constants.REEXEC_API_WORKSPACE,
        help="Root directory for a secondary checkout .",
    )
    parser.add_option(
        "--bootstrap-dir",
        type="path",
        help="Bootstrapping cbuildbot may involve checking out "
        "multiple copies of chromite. All these checkouts "
        "will be contained in the directory specified here. "
        "Default:%s" % osutils.GetGlobalTempDir(),
    )
    parser.add_remote_option(
        "--android_rev",
        type="choice",
        choices=constants.VALID_ANDROID_REVISIONS,
        help=(
            "Revision of Android to use, of type [%s]"
            % "|".join(constants.VALID_ANDROID_REVISIONS)
        ),
    )
    parser.add_remote_option(
        "--chrome_rev",
        type="choice",
        choices=constants.VALID_CHROME_REVISIONS,
        help=(
            "Revision of Chrome to use, of type [%s]"
            % "|".join(constants.VALID_CHROME_REVISIONS)
        ),
    )
    parser.add_remote_option(
        "--profile", help="Name of profile to sub-specify board variant."
    )
    # TODO(crbug.com/279618): Running GOMA is under development. Following
    # flags are added for development purpose due to repository dependency,
    # but not officially supported yet.
    parser.add_option(
        "--goma_dir",
        type="path",
        api=constants.REEXEC_API_GOMA,
        help="Specify a directory containing goma. When this is "
        "set, GOMA is used to build Chrome.",
    )
    parser.add_option(
        "--chromeos_goma_dir",
        type="path",
        api=constants.REEXEC_API_CHROMEOS_GOMA_DIR,
        help="Specify a directory containing goma for " "build package.",
    )
    group = CustomGroup(parser, "Deprecated Options")

    parser.add_option(
        "--local",
        action="store_true",
        default=False,
        help="Deprecated. See cros tryjob.",
    )
    parser.add_option(
        "--remote",
        action="store_true",
        default=False,
        help="Deprecated. See cros tryjob.",
    )

    #
    # Patch selection options.
    #

    group = CustomGroup(parser, "Patch Options")

    group.add_remote_option(
        "-g",
        "--gerrit-patches",
        action="split_extend",
        type="string",
        default=[],
        metavar="'Id1 *int_Id2...IdN'",
        help="Space-separated list of short-form Gerrit "
        "Change-Id's or change numbers to patch. "
        "Please prepend '*' to internal Change-Id's",
    )

    parser.add_argument_group(group)

    #
    # Remote trybot options.
    #

    group = CustomGroup(parser, "Options used to configure tryjob behavior.")
    group.add_remote_option(
        "--hwtest",
        action="store_true",
        default=False,
        help="Run the HWTest stage (tests on real hardware)",
    )
    group.add_option(
        "--hwtest_dut_dimensions",
        type="string",
        action="split_extend",
        default=None,
        help="Space-separated list of key:val Swarming bot "
        "dimensions to run each builders SkylabHWTest "
        "stages against (this overrides the configured "
        "DUT dimensions for each test). Requires at least "
        '"label-board", "label-model", and "label-pool".',
    )
    group.add_remote_option(
        "--channel",
        action="split_extend",
        dest="channels",
        default=[],
        help="Specify a channel for a payloads trybot. Can "
        "be specified multiple times. No valid for "
        "non-payloads configs.",
    )

    parser.add_argument_group(group)

    #
    # Advanced options.
    #

    group = CustomGroup(
        parser,
        "Advanced Options",
        "Caution: use these options at your own risk.",
    )

    group.add_remote_option(
        "--bootstrap-args",
        action="append",
        default=[],
        help="Args passed directly to the bootstrap re-exec "
        "to skip verification by the bootstrap code",
    )
    group.add_remote_option(
        "--buildbot",
        action="store_true",
        dest="buildbot",
        default=False,
        help="This is running on a buildbot. "
        "This can be used to make a build operate "
        "like an official builder, e.g. generate "
        "new version numbers and archive official "
        "artifacts and such. This should only be "
        "used if you are confident in what you are "
        "doing, as it will make automated commits.",
    )
    parser.add_remote_option(
        "--repo-cache",
        type="path",
        dest="_repo_cache",
        help="Present for backwards compatibility, ignored.",
    )
    group.add_remote_option(
        "--no-buildbot-tags",
        action="store_false",
        dest="enable_buildbot_tags",
        default=True,
        help="Suppress buildbot specific tags from log "
        "output. This is used to hide recursive "
        "cbuilbot runs on the waterfall.",
    )
    group.add_remote_option(
        "--buildnumber", type="int", default=0, help="build number"
    )
    group.add_option(
        "--chrome_root",
        action="callback",
        type="path",
        callback=_CheckChromeRootOption,
        help="Local checkout of Chrome to use.",
    )
    group.add_remote_option(
        "--chrome_version",
        action="callback",
        type="string",
        dest="chrome_version",
        callback=_CheckChromeVersionOption,
        help="Used with SPEC logic to force a particular "
        "git revision of chrome rather than the "
        "latest.",
    )
    group.add_remote_option(
        "--clobber",
        action="store_true",
        default=False,
        help="Clears an old checkout before syncing",
    )
    group.add_remote_option(
        "--latest-toolchain",
        action="store_true",
        default=False,
        help="Use the latest toolchain.",
    )
    parser.add_option(
        "--log_dir",
        dest="log_dir",
        type="path",
        help="Directory where logs are stored.",
    )
    group.add_remote_option(
        "--maxarchives",
        type="int",
        dest="max_archive_builds",
        default=3,
        help="Change the local saved build count limit.",
    )
    parser.add_remote_option(
        "--manifest-repo-url", help="Overrides the default manifest repo url."
    )
    group.add_remote_option(
        "--compilecheck",
        action="store_true",
        default=False,
        help="Only verify compilation and unit tests.",
    )
    group.add_remote_option(
        "--noarchive",
        action="store_false",
        dest="archive",
        default=True,
        help="Don't run archive stage.",
    )
    group.add_remote_option(
        "--nobootstrap",
        action="store_false",
        dest="bootstrap",
        default=True,
        help="Don't checkout and run from a standalone " "chromite repo.",
    )
    group.add_remote_option(
        "--nobuild",
        action="store_false",
        dest="build",
        default=True,
        help="Don't actually build (for cbuildbot dev)",
    )
    group.add_remote_option(
        "--noclean",
        action="store_false",
        dest="clean",
        default=True,
        help="Don't clean the buildroot",
    )
    group.add_remote_option(
        "--nocgroups",
        action="store_false",
        dest="cgroups",
        default=True,
        help="Disable cbuildbots usage of cgroups.",
    )
    group.add_remote_option(
        "--nochromesdk",
        action="store_false",
        dest="chrome_sdk",
        default=True,
        help="Don't run the ChromeSDK stage which builds "
        "Chrome outside of the chroot.",
    )
    group.add_remote_option(
        "--noprebuilts",
        action="store_false",
        dest="prebuilts",
        default=True,
        help="Don't upload prebuilts.",
    )
    group.add_remote_option(
        "--nopatch",
        action="store_false",
        dest="postsync_patch",
        default=True,
        help="Don't run PatchChanges stage.  This does not "
        "disable patching in of chromite patches "
        "during BootstrapStage.",
    )
    group.add_remote_option(
        "--nopaygen",
        action="store_false",
        dest="paygen",
        default=True,
        help="Don't generate payloads.",
    )
    group.add_remote_option(
        "--noreexec",
        action="store_false",
        dest="postsync_reexec",
        default=True,
        help="Don't reexec into the buildroot after syncing.",
    )
    group.add_remote_option(
        "--nosdk",
        action="store_true",
        default=False,
        help="Re-create the SDK from scratch.",
    )
    group.add_remote_option(
        "--nosync",
        action="store_false",
        dest="sync",
        default=True,
        help="Don't sync before building.",
    )
    group.add_remote_option(
        "--notests",
        action="store_false",
        dest="tests",
        default=True,
        help="Override values from buildconfig, run no "
        "tests, and build no autotest and artifacts.",
    )
    group.add_remote_option(
        "--novmtests",
        action="store_false",
        dest="vmtests",
        default=True,
        help="Override values from buildconfig, run no " "vmtests.",
    )
    group.add_remote_option(
        "--noimagetests",
        action="store_false",
        dest="image_test",
        default=True,
        help="Override values from buildconfig and run no " "image tests.",
    )
    group.add_remote_option(
        "--nouprev",
        action="store_false",
        dest="uprev",
        default=True,
        help="Override values from buildconfig and never " "uprev.",
    )
    group.add_option(
        "--reference-repo",
        help="Reuse git data stored in an existing repo "
        "checkout. This can drastically reduce the network "
        "time spent setting up the trybot checkout.  By "
        "default, if this option isn't given but cbuildbot "
        "is invoked from a repo checkout, cbuildbot will "
        "use the repo root.",
    )
    group.add_option(
        "--resume",
        action="store_true",
        default=False,
        help="Skip stages already successfully completed.",
    )
    group.add_remote_option(
        "--timeout",
        type="int",
        default=0,
        help="Specify the maximum amount of time this job "
        "can run for, at which point the build will be "
        "aborted.  If set to zero, then there is no "
        "timeout.",
    )
    group.add_remote_option(
        "--version",
        dest="force_version",
        help="Used with manifest logic.  Forces use of this "
        "version rather than create or get latest. "
        "Examples: 4815.0.0-rc1, 4815.1.2",
    )
    group.add_remote_option(
        "--git-cache-dir",
        type="path",
        api=constants.REEXEC_API_GIT_CACHE_DIR,
        help="Specify the cache directory to store the "
        "project caches populated by the git-cache "
        "tool. Bootstrap the projects based on the git "
        "cache files instead of fetching them directly "
        "from the GoB servers.",
    )
    group.add_remote_option(
        "--chrome-preload-dir",
        type="path",
        api=constants.REEXEC_API_CHROME_PRELOAD_DIR,
        help="Specify a preloaded chrome source cache "
        "directory populated by the git-cache tool. "
        "Bootstrap chrome based on the cached files "
        "instead of fetching them directly from the GoB "
        "servers. When both this argument and "
        "--git-cache-dir are provided this value will "
        "be preferred for the chrome source cache.",
    )
    group.add_remote_option(
        "--source_cache",
        action="store_true",
        default=False,
        help="Whether to utilize cache snapshot mounts.",
    )
    group.add_remote_option(
        "--debug-cidb",
        action="store_true",
        default=False,
        help="Force Debug CIDB to be used.",
    )
    # cbuildbot ChromeOS Findit options
    group.add_remote_option(
        "--cbb_build_packages",
        action="split_extend",
        dest="cbb_build_packages",
        default=[],
        help="Specify an explicit list of packages to build "
        "for integration with Findit.",
    )
    group.add_remote_option(
        "--cbb_snapshot_revision",
        type="string",
        dest="cbb_snapshot_revision",
        default=None,
        help="Snapshot manifest revision to sync to " "for building.",
    )
    group.add_remote_option(
        "--no-publish-prebuilt-confs",
        dest="publish",
        action="store_false",
        default=True,
        help="Don't publish git commits to prebuilt.conf or sdk_version.conf",
    )

    parser.add_argument_group(group)

    #
    # Internal options.
    #

    group = CustomGroup(
        parser,
        "Internal Chromium OS Build Team Options",
        "Caution: these are for meant for the Chromium OS build team only",
    )

    group.add_remote_option(
        "--archive-base",
        type="gs_path",
        help="Base GS URL (gs://<bucket_name>/<path>) to "
        "upload archive artifacts to",
    )
    group.add_remote_option(
        "--cq-gerrit-query",
        dest="cq_gerrit_override",
        help="If given, this gerrit query will be used to find what patches to "
        "test, rather than the normal 'CommitQueue>=1 AND Verified=1 AND "
        "CodeReview=2' query it defaults to.  Use with care- note "
        "additionally this setting only has an effect if the buildbot "
        "target is a cq target, and we're in buildbot mode.",
    )
    group.add_option(
        "--pass-through",
        action="append",
        type="string",
        dest="pass_through_args",
        default=[],
    )
    group.add_option(
        "--reexec-api-version",
        action="store_true",
        dest="output_api_version",
        default=False,
        help="Used for handling forwards/backwards compatibility "
        "with --resume and --bootstrap",
    )
    group.add_option(
        "--remote-trybot",
        action="store_true",
        default=False,
        help="Indicates this is running on a remote trybot machine",
    )
    group.add_option(
        "--buildbucket-id",
        api=constants.REEXEC_API_GOMA,  # Approximate.
        help="The unique ID in buildbucket of current build "
        "generated by buildbucket.",
    )
    group.add_remote_option(
        "--remote-patches",
        action="split_extend",
        default=[],
        help="Patches uploaded by the trybot client when "
        "run using the -p option",
    )
    # Note the default here needs to be hardcoded to 3; that is the last version
    # that lacked this functionality.
    group.add_option(
        "--remote-version",
        type="int",
        default=3,
        help="Deprecated and ignored.",
    )
    group.add_option("--sourceroot", type="path", default=constants.SOURCE_ROOT)
    group.add_remote_option(
        "--test-bootstrap",
        action="store_true",
        default=False,
        help="Causes cbuildbot to bootstrap itself twice, "
        "in the sequence A->B->C: A(unpatched) patches "
        "and bootstraps B; B patches and bootstraps C",
    )
    group.add_remote_option(
        "--validation_pool",
        help="Path to a pickled validation pool. Intended "
        "for use only with the commit queue.",
    )
    group.add_remote_option(
        "--metadata_dump",
        help="Path to a json dumped metadata file. This "
        "will be used as the initial metadata.",
    )
    group.add_remote_option(
        "--master-build-id",
        type="int",
        api=constants.REEXEC_API_MASTER_BUILD_ID,
        help="cidb build id of the master build to this " "slave build.",
    )
    group.add_remote_option(
        "--master-buildbucket-id",
        api=constants.REEXEC_API_MASTER_BUILDBUCKET_ID,
        help="buildbucket id of the master build to this " "slave build.",
    )
    # TODO(nxia): crbug.com/778838
    # cbuildbot doesn't use pickle files anymore, remove this.
    group.add_remote_option(
        "--mock-slave-status",
        metavar="MOCK_SLAVE_STATUS_PICKLE_FILE",
        help="Override the result of the _FetchSlaveStatuses "
        "method of MasterSlaveSyncCompletionStage, by "
        "specifying a file with a pickle of the result "
        "to be returned.",
    )
    group.add_option(
        "--previous-build-state",
        type="string",
        default="",
        api=constants.REEXEC_API_PREVIOUS_BUILD_STATE,
        help="A base64-encoded BuildSummary object describing the "
        "previous build run on the same build machine.",
    )

    parser.add_argument_group(group)

    #
    # Debug options
    #
    # Temporary hack; in place till --dry-run replaces --debug.
    # pylint: disable=protected-access
    group = parser.debug_group
    debug = [x for x in group.option_list if x._long_opts == ["--debug"]][0]
    debug.help += "  Currently functions as --dry-run in addition."
    debug.pass_through = True
    group.add_option(
        "--notee",
        action="store_false",
        dest="tee",
        default=True,
        help="Disable logging and internal tee process.  Primarily "
        "used for debugging cbuildbot itself.",
    )
    return parser


def _FinishParsing(options):
    """Perform some parsing tasks that need to take place after optparse.

    This function needs to be easily testable!  Keep it free of
    environment-dependent code.  Put more detailed usage validation in
    _PostParseCheck().

    Args:
      options: The options object returned by optparse
    """
    # Populate options.pass_through_args.
    accepted, _ = commandline.FilteringParser.FilterArgs(
        options.parsed_args, lambda x: x.opt_inst.pass_through
    )
    options.pass_through_args.extend(accepted)

    if options.local or options.remote:
        cros_build_lib.Die("Deprecated usage. Please use cros tryjob instead.")

    if not options.buildroot:
        cros_build_lib.Die("A buildroot is required to build.")

    if options.chrome_root:
        if options.chrome_rev != constants.CHROME_REV_LOCAL:
            cros_build_lib.Die(
                "Chrome rev must be %s if chrome_root is set."
                % constants.CHROME_REV_LOCAL
            )
    elif options.chrome_rev == constants.CHROME_REV_LOCAL:
        cros_build_lib.Die(
            "Chrome root must be set if chrome_rev is %s."
            % constants.CHROME_REV_LOCAL
        )

    if options.chrome_version:
        if options.chrome_rev != constants.CHROME_REV_SPEC:
            cros_build_lib.Die(
                "Chrome rev must be %s if chrome_version is set."
                % constants.CHROME_REV_SPEC
            )
    elif options.chrome_rev == constants.CHROME_REV_SPEC:
        cros_build_lib.Die(
            "Chrome rev must not be %s if chrome_version is not set."
            % constants.CHROME_REV_SPEC
        )

    patches = bool(options.gerrit_patches)

    # When running in release mode, make sure we are running with checked-in code.
    # We want checked-in cbuildbot/scripts to prevent errors, and we want to build
    # a release image with checked-in code for CrOS packages.
    if options.buildbot and patches and not options.debug:
        cros_build_lib.Die(
            "Cannot provide patches when running with --buildbot!"
        )

    if options.buildbot and options.remote_trybot:
        cros_build_lib.Die(
            "--buildbot and --remote-trybot cannot be used together."
        )

    # Record whether --debug was set explicitly vs. it was inferred.
    options.debug_forced = options.debug
    # We force --debug to be set for builds that are not 'official'.
    options.debug = options.debug or not options.buildbot

    options.hwtest_dut_override = ParseHWTestDUTDims(
        options.hwtest_dut_dimensions
    )


def ParseHWTestDUTDims(dims):
    """Parse HWTest DUT dimensions into a valid HWTestDUTOverride object.

    Raises an error if any of board, model, or pool is missing.
    """
    if not dims:
        return None
    board = model = pool = None
    extra_dims = []
    for dim in dims:
        if dim.startswith(BOARD_DIM_LABEL):
            # Remove one extra character to account for the ":" or "=" symbol
            # separating the label from the dimension itself.
            board = dim[len(BOARD_DIM_LABEL) + 1 :]
        elif dim.startswith(MODEL_DIM_LABEL):
            model = dim[len(MODEL_DIM_LABEL) + 1 :]
        elif dim.startswith(POOL_DIM_LABEL):
            pool = dim[len(POOL_DIM_LABEL) + 1 :]
        else:
            extra_dims.append(dim)

    if not (board and model and pool):
        cros_build_lib.Die(
            "HWTest DUT dimensions must include board, model, and "
            "pool (given %s)." % dims
        )

    return test_stages.HWTestDUTOverride(board, model, pool, extra_dims)


# pylint: disable=unused-argument
def _PostParseCheck(parser, options, site_config):
    """Perform some usage validation after we've parsed the arguments

    Args:
      parser: Option parser that was used to parse arguments.
      options: The options returned by optparse.
      site_config: config_lib.SiteConfig containing all config info.
    """

    if not options.branch:
        options.branch = git.GetChromiteTrackingBranch()

    # Because the default cache dir depends on other options, FindCacheDir
    # always returns None, and we setup the default here.
    if options.cache_dir is None:
        # Note, options.sourceroot is set regardless of the path
        # actually existing.
        options.cache_dir = os.path.join(options.buildroot, ".cache")
        options.cache_dir = os.path.abspath(options.cache_dir)
        parser.ConfigureCacheDir(options.cache_dir)

    osutils.SafeMakedirsNonRoot(options.cache_dir)

    # Ensure that all args are legitimate config targets.
    if options.build_config_name not in site_config:
        cros_build_lib.Die(
            'Unknown build config: "%s"' % options.build_config_name
        )

    build_config = site_config[options.build_config_name]
    is_payloads_build = build_config.build_type == constants.PAYLOADS_TYPE

    if options.channels and not is_payloads_build:
        cros_build_lib.Die(
            "--channel must only be used with a payload config,"
            " not target (%s)." % options.build_config_name
        )

    if not options.channels and is_payloads_build:
        cros_build_lib.Die(
            "payload configs (%s) require --channel to do anything"
            " useful." % options.build_config_name
        )

    # If the build config explicitly forces the debug flag, set the debug flag
    # as if it was set from the command line.
    if build_config.debug:
        options.debug = True

    if not (config_lib.isTryjobConfig(build_config) or options.buildbot):
        cros_build_lib.Die(
            "Refusing to run non-tryjob config as a tryjob.\n"
            'Please "repo sync && cros tryjob --list %s" for alternatives.\n'
            "See go/cros-explicit-tryjob-build-configs-psa.",
            build_config.name,
        )

    # The --version option is not compatible with an external target unless the
    # --buildbot option is specified.  More correctly, only "paladin versions"
    # will work with external targets, and those are only used with --buildbot.
    # If --buildbot is specified, then user should know what they are doing and
    # only specify a version that will work.  See crbug.com/311648.
    if options.force_version and not (
        options.buildbot or build_config.internal
    ):
        cros_build_lib.Die(
            "Cannot specify --version without --buildbot for an"
            " external target (%s)." % options.build_config_name
        )


def ParseCommandLine(parser, argv):
    """Completely parse the commandline arguments"""
    (options, args) = parser.parse_args(argv)

    # Handle the request for the reexec command line API version number.
    if options.output_api_version:
        print(constants.REEXEC_API_VERSION)
        sys.exit(0)

    # Record the configs targeted. Strip out null arguments.
    build_config_names = [x for x in args if x]
    if len(build_config_names) != 1:
        cros_build_lib.Die(
            "Expected exactly one build config. Got: %r", build_config_names
        )
    options.build_config_name = build_config_names[-1]

    _FinishParsing(options)
    return options


_ENVIRONMENT_PROD = "prod"
_ENVIRONMENT_DEBUG = "debug"
_ENVIRONMENT_STANDALONE = "standalone"


def _GetRunEnvironment(options, build_config):
    """Determine whether this is a prod/debug/standalone run."""
    if options.debug_cidb:
        return _ENVIRONMENT_DEBUG

    # One of these arguments should always be set if running on a real builder.
    # If we aren't on a real builder, we are standalone.
    if not options.buildbot and not options.remote_trybot:
        return _ENVIRONMENT_STANDALONE

    if build_config["debug_cidb"]:
        return _ENVIRONMENT_DEBUG

    return _ENVIRONMENT_PROD


def _SetupConnections(options, build_config):
    """Set up CIDB connections using the appropriate Setup call.

    Args:
      options: Command line options structure.
      build_config: Config object for this build.
    """
    # Outline:
    # 1) Based on options and build_config, decide whether we are a production
    # run, debug run, or standalone run.
    # 2) Set up cidb instance accordingly.
    # 3) Update topology info from cidb, so that any other service set up can use
    # topology.
    # 4) Set up any other services.
    run_type = _GetRunEnvironment(options, build_config)

    if run_type == _ENVIRONMENT_PROD:
        cidb.CIDBConnectionFactory.SetupProdCidb()
        context = ts_mon_config.SetupTsMonGlobalState(
            "cbuildbot", indirect=True
        )
    elif run_type == _ENVIRONMENT_DEBUG:
        cidb.CIDBConnectionFactory.SetupDebugCidb()
        context = ts_mon_config.TrivialContextManager()
    else:
        cidb.CIDBConnectionFactory.SetupNoCidb()
        context = ts_mon_config.TrivialContextManager()

    topology.FetchTopology()
    return context


class _MockMethodWithReturnValue(object):
    """A method mocker which just returns the specific value."""

    def __init__(self, return_value):
        self.return_value = return_value

    def __call__(self, *args, **kwargs):
        return self.return_value


class _ObjectMethodPatcher(object):
    """A simplified mock.object.patch.

    It is a context manager that patches an object's method with specified
    return value.
    """

    def __init__(self, target, attr, return_value=None):
        """Constructor.

        Args:
          target: object to patch.
          attr: method name of the object to patch.
          return_value: the return value when calling target.attr
        """
        self.target = target
        self.attr = attr
        self.return_value = return_value
        self.original_attr = None
        self.new_attr = _MockMethodWithReturnValue(self.return_value)

    def __enter__(self):
        self.original_attr = self.target.__dict__[self.attr]
        setattr(self.target, self.attr, self.new_attr)

    def __exit__(self, *args):
        if self.target and self.original_attr:
            setattr(self.target, self.attr, self.original_attr)


# TODO(build): This function is too damn long.
def main(argv):
    # We get false positives with the options object.
    # pylint: disable=attribute-defined-outside-init

    # Turn on strict sudo checks.
    cros_build_lib.STRICT_SUDO = True

    # Set umask to 022 so files created by buildbot are readable.
    os.umask(0o22)

    parser = _CreateParser()
    options = ParseCommandLine(parser, argv)

    # Fetch our site_config now, because we need it to do anything else.
    site_config = config_lib.GetConfig()

    _PostParseCheck(parser, options, site_config)

    cros_build_lib.AssertOutsideChroot()

    if options.enable_buildbot_tags:
        cbuildbot_alerts.EnableBuildbotMarkers()

    if (
        options.buildbot
        and not options.debug
        and not cros_build_lib.HostIsCIBuilder()
    ):
        # --buildbot can only be used on a real builder, unless it's debug.
        cros_build_lib.Die("This host is not a supported build machine.")

    # Only one config arg is allowed in this mode, which was confirmed earlier.
    build_config = site_config[options.build_config_name]

    # TODO: Re-enable this block when reference_repo support handles this
    #       properly. (see chromium:330775)
    # if options.reference_repo is None:
    #   repo_path = os.path.join(options.sourceroot, '.repo')
    #   # If we're being run from a repo checkout, reuse the repo's git pool to
    #   # cut down on sync time.
    #   if os.path.exists(repo_path):
    #     options.reference_repo = options.sourceroot

    if options.reference_repo:
        if not os.path.exists(options.reference_repo):
            parser.error(
                "Reference path %s does not exist" % (options.reference_repo,)
            )
        elif not os.path.exists(os.path.join(options.reference_repo, ".repo")):
            parser.error(
                "Reference path %s does not look to be the base of a "
                "repo checkout; no .repo exists in the root."
                % (options.reference_repo,)
            )

    if (options.buildbot or options.remote_trybot) and not options.resume:
        missing = osutils.FindMissingBinaries(_BUILDBOT_REQUIRED_BINARIES)
        if missing:
            parser.error(
                "Option --buildbot/--remote-trybot requires the following "
                "binaries which couldn't be found in $PATH: %s"
                % (", ".join(missing))
            )

    if options.reference_repo:
        options.reference_repo = os.path.abspath(options.reference_repo)

    # Sanity check of buildroot- specifically that it's not pointing into the
    # midst of an existing repo since git-repo doesn't support nesting.
    if not repository.IsARepoRoot(options.buildroot) and git.FindRepoDir(
        options.buildroot
    ):
        cros_build_lib.Die(
            "Configured buildroot %s is a subdir of an existing repo checkout."
            % options.buildroot
        )

    if not options.log_dir:
        options.log_dir = os.path.join(options.buildroot, _DEFAULT_LOG_DIR)

    log_file = None
    if options.tee:
        log_file = os.path.join(options.log_dir, _BUILDBOT_LOG_FILE)
        osutils.SafeMakedirs(options.log_dir)
        _BackupPreviousLog(log_file)

    with cros_build_lib.ContextManagerStack() as stack:
        # Preserve chromite; we might be running from there!
        options.preserve_paths = set(["chromite"])
        if log_file is not None:
            # We don't want the critical section to try to clean up the tee process,
            # so we run Tee (forked off) outside of it. This prevents a deadlock
            # because the Tee process only exits when its pipe is closed, and the
            # critical section accidentally holds on to that file handle.
            stack.Add(tee.Tee, log_file)
            options.preserve_paths.add(_DEFAULT_LOG_DIR)

        critical_section = stack.Add(cleanup.EnforcedCleanupSection)
        stack.Add(sudo.SudoKeepAlive)

        if not options.resume:
            # If we're in resume mode, use our parents tempdir rather than
            # nesting another layer.
            stack.Add(osutils.TempDir, prefix="cbb", set_global=True)
            logging.debug("Cbuildbot tempdir is %r.", os.environ.get("TMP"))

        if options.cgroups:
            stack.Add(cgroups.SimpleContainChildren, "cbuildbot")

        # Mark everything between EnforcedCleanupSection and here as having to
        # be rolled back via the contextmanager cleanup handlers.  This
        # ensures that sudo bits cannot outlive cbuildbot, that anything
        # cgroups would kill gets killed, etc.
        stack.Add(critical_section.ForkWatchdog)

        if options.mock_slave_status is not None:
            with open(options.mock_slave_status, "rb") as f:
                mock_statuses = pickle.load(f)
                for key, value in mock_statuses.items():
                    mock_statuses[key] = builder_status_lib.BuilderStatus(
                        **value
                    )
            stack.Add(
                _ObjectMethodPatcher,
                completion_stages.MasterSlaveSyncCompletionStage,
                "_FetchSlaveStatuses",
                return_value=mock_statuses,
            )

        stack.Add(_SetupConnections, options, build_config)
        retry_stats.SetupStats()

        timeout_display_message = (
            "This build has reached the timeout deadline set by the master. "
            "Either this stage or a previous one took too long (see stage "
            "timing historical summary in ReportStage) or the build failed "
            "to start on time."
        )

        if options.timeout > 0:
            stack.Add(
                timeout_util.FatalTimeout,
                options.timeout,
                timeout_display_message,
            )
        try:
            _RunBuildStagesWrapper(options, site_config, build_config)
        except failures_lib.ExitEarlyException as ex:
            # This build finished successfully. Do not re-raise ExitEarlyException.
            logging.info("One stage exited early: %s", ex)