cbuildbot: Add ChromeSDK stage.

Adds a ChromeSDK stage that:

1. Generates artifacts required for chrome-sdk for the board being
built.
2. Runs chrome-sdk against the newly generated artifacts, and builds
Chrome with it (using ninja and goma).
3. Runs deploy_chrome on the artifacts with --staging-only.

BUG=chromium:242998
TEST=Local, trybots

Change-Id: I6789fd1e842cbc72d0a72f9fdf1a8a90ea591a0b
Reviewed-on: https://gerrit.chromium.org/gerrit/63844
Commit-Queue: David James <davidjames@chromium.org>
Reviewed-by: David James <davidjames@chromium.org>
Tested-by: David James <davidjames@chromium.org>
diff --git a/buildbot/cbuildbot_commands.py b/buildbot/cbuildbot_commands.py
index aaae09e..6dc6f80 100644
--- a/buildbot/cbuildbot_commands.py
+++ b/buildbot/cbuildbot_commands.py
@@ -1745,15 +1745,55 @@
 
 
 class ChromeSDK(object):
+  """Wrapper for the 'cros chrome-sdk' command."""
 
   DEFAULT_TARGETS = ('chrome', 'chrome_sandbox', 'nacl_helper',)
+  DEFAULT_JOBS = 24
+  DEFAULT_JOBS_GOMA = 500
 
-  @staticmethod
-  def Run(cwd, board, cmd):
-    cmd = ['cros', 'chrome-sdk', '--board', board, '--'] + cmd
-    cros_build_lib.RunCommand(cmd, cwd=cwd)
+  def __init__(self, cwd, board, extra_args=None, chrome_src=None, goma=False,
+               debug_log=True, cache_dir=None):
+    """Initialization.
 
-  @classmethod
-  def Ninja(cls, cwd, board, jobs=500, targets=DEFAULT_TARGETS):
-    cmd = ['ninja', '-j', str(jobs)] + list(targets)
-    cls.Run(cwd, board, cmd)
+    Args:
+      cwd: Where to invoke 'cros chrome-sdk'.
+      board: The board to run chrome-sdk for.
+      extra_args: Extra args to pass in on the command line.
+      chrome_src: Path to pass in with --chrome-src.
+      debug_log: If set, run with debug log-level.
+    """
+    self.cwd = cwd
+    self.board = board
+    self.extra_args = extra_args or []
+    if chrome_src:
+      self.extra_args += ['--chrome-src', chrome_src]
+    self.goma = goma
+    if not self.goma:
+      self.extra_args.append('--nogoma')
+    self.debug_log = debug_log
+    self.cache_dir = cache_dir
+
+  def Run(self, cmd, extra_args=None):
+    """Run a command inside the chrome-sdk context."""
+    cros_cmd = ['cros']
+    if self.debug_log:
+      cros_cmd += ['--log-level', 'debug']
+    if self.cache_dir:
+      cros_cmd += ['--cache-dir', self.cache_dir]
+    cros_cmd += ['chrome-sdk', '--board', self.board] + self.extra_args
+    cros_cmd += (extra_args or []) + ['--'] + cmd
+    cros_build_lib.RunCommand(cros_cmd, cwd=self.cwd)
+
+  def Ninja(self, jobs=None, debug=False, targets=DEFAULT_TARGETS):
+    """Run 'ninja' inside a chrome-sdk context.
+
+    Args:
+      jobs: The number of -j jobs to run.
+      debug: Whether to do a Debug build (defaults to Release).
+      targets: The targets to compile.
+    """
+    if jobs is None:
+      jobs = self.DEFAULT_JOBS_GOMA if self.goma else self.DEFAULT_JOBS
+    flavor = 'Debug' if debug else 'Release'
+    cmd = ['ninja', '-C', 'out_%s/%s' % (self.board, flavor) , '-j', str(jobs)]
+    self.Run(cmd + list(targets))
diff --git a/buildbot/cbuildbot_commands_unittest.py b/buildbot/cbuildbot_commands_unittest.py
index eac8420..80cb3d3 100755
--- a/buildbot/cbuildbot_commands_unittest.py
+++ b/buildbot/cbuildbot_commands_unittest.py
@@ -127,17 +127,32 @@
 class ChromeSDKTest(cros_build_lib_unittest.RunCommandTempDirTestCase):
   """Basic tests for ChromeSDK commands with RunCommand mocked out."""
   BOARD = 'daisy_foo'
+  EXTRA_ARGS = ('--monkey', 'banana')
+  EXTRA_ARGS2 = ('--donkey', 'kong')
+  CHROME_SRC = 'chrome_src'
   CMD = ['bar', 'baz']
   CWD = 'fooey'
 
+  def setUp(self):
+    self.inst = commands.ChromeSDK(self.CWD, self.BOARD)
+
   def testRunCommand(self):
     """Test that running a command is possible."""
-    commands.ChromeSDK.Run(self.CWD, self.BOARD, self.CMD)
+    self.inst.Run(self.CMD)
     self.assertCommandContains([self.BOARD] + self.CMD, cwd=self.CWD)
 
+  def testRunCommandKwargs(self):
+    """Exercise optional arguments."""
+    custom_inst = commands.ChromeSDK(
+        self.CWD, self.BOARD, extra_args=list(self.EXTRA_ARGS),
+        chrome_src=self.CHROME_SRC, debug_log=True)
+    custom_inst.Run(self.CMD, list(self.EXTRA_ARGS2))
+    self.assertCommandContains(['debug', self.BOARD] + list(self.EXTRA_ARGS) +
+                               list(self.EXTRA_ARGS2) + self.CMD, cwd=self.CWD)
+
   def testNinja(self):
     """Test that running ninja is possible."""
-    commands.ChromeSDK.Ninja(self.CWD, self.BOARD)
+    self.inst.Ninja(self.BOARD)
     self.assertCommandContains([self.BOARD], cwd=self.CWD)
 
 
diff --git a/buildbot/cbuildbot_config.py b/buildbot/cbuildbot_config.py
index 8a02129..7443a5d 100755
--- a/buildbot/cbuildbot_config.py
+++ b/buildbot/cbuildbot_config.py
@@ -461,6 +461,13 @@
 # Enable rootfs verification on the image.
   rootfs_verification=True,
 
+# Runs through the chrome-sdk workflow.
+  chrome_sdk=False,
+
+# If chrome_sdk is set to True, this determines whether we use goma to build
+# chrome.
+  chrome_sdk_goma=False,
+
 # =============================================================================
 )
 
@@ -1366,6 +1373,7 @@
   trybot_list=True,
   hwqual=True,
   description="Release Builds (canary) (internal)",
+  chrome_sdk=True,
 )
 
 ### Master release config.
diff --git a/buildbot/cbuildbot_stages.py b/buildbot/cbuildbot_stages.py
index 2d22602..4f9d47a 100644
--- a/buildbot/cbuildbot_stages.py
+++ b/buildbot/cbuildbot_stages.py
@@ -226,7 +226,7 @@
     duration = '%s' % (current_time - start_time,)
 
     sdk_verinfo = cros_build_lib.LoadKeyValueFile(
-        os.path.join(constants.SOURCE_ROOT, constants.SDK_VERSION_FILE),
+        os.path.join(self._build_root, constants.SDK_VERSION_FILE),
         ignore_missing=True)
     verinfo = self.archive_stage.GetVersionInfo()
     metadata = {
@@ -260,7 +260,8 @@
         'toolchain-url': sdk_verinfo.get('TC_PATH', '<unknown>'),
     }
     if len(config['boards']) == 1:
-      toolchains = toolchain.GetToolchainsForBoard(config['boards'][0])
+      toolchains = toolchain.GetToolchainsForBoard(config['boards'][0],
+                                                   buildroot=self._build_root)
       metadata['toolchain-tuple'] = (
           toolchain.FilterToolchains(toolchains, 'default', True).keys() +
           toolchain.FilterToolchains(toolchains, 'default', False).keys())
@@ -293,7 +294,7 @@
 
     return metadata
 
-  def UploadMetadata(self, stage=None, **kwargs):
+  def UploadMetadata(self, stage=None, upload_queue=None, **kwargs):
     """Create a JSON of various metadata describing this build."""
     metadata = self.GetMetadata(stage=stage, **kwargs)
     filename = constants.METADATA_JSON
@@ -303,7 +304,11 @@
 
     # Stages may run in parallel, so we have to do atomic updates on this.
     osutils.WriteFile(metadata_json, json.dumps(metadata), atomic=True)
-    self.UploadArtifact(os.path.basename(metadata_json), archive=False)
+
+    if upload_queue is not None:
+      upload_queue.put([filename])
+    else:
+      self.UploadArtifact(filename, archive=False)
 
 
 class CleanUpStage(bs.BuilderStage):
@@ -346,6 +351,11 @@
         self._options.log_dir, '*.%s' % HWTestStage.PERF_RESULTS_EXTENSION)):
       os.remove(result)
 
+  def _DeleteChromeBuildOutput(self):
+    chrome_src = os.path.join(self._options.chrome_root, 'src')
+    for out_dir in glob.glob(os.path.join(chrome_src, 'out_*')):
+      osutils.RmDir(out_dir)
+
   def PerformStage(self):
     if (not (self._options.buildbot or self._options.remote_trybot)
         and self._options.clobber):
@@ -382,6 +392,8 @@
                functools.partial(commands.WipeOldOutput, self._build_root),
                self._DeleteArchivedTrybotImages,
                self._DeleteArchivedPerfResults]
+      if self._options.chrome_root:
+        tasks.append(self._DeleteChromeBuildOutput)
       if self._build_config['chroot_replace'] and self._options.build:
         tasks.append(self._DeleteChroot)
       else:
@@ -1916,6 +1928,99 @@
                    extra_env=self._env)
 
 
+class ChromeSDKStage(ArchivingStage):
+  """Run through the simple chrome workflow."""
+
+  def __init__(self, *args, **kwargs):
+    super(ChromeSDKStage, self).__init__(*args, **kwargs)
+    self._upload_queue = multiprocessing.Queue()
+    self._pkg_dir = os.path.join(
+        self._build_root, constants.DEFAULT_CHROOT_DIR,
+        'build', self._current_board, 'var', 'db', 'pkg')
+    self.chrome_src = os.path.join(self._options.chrome_root, 'src')
+    self.out_board_dir = os.path.join(
+        self.chrome_src, 'out_%s' % self._current_board)
+
+  def _BuildAndArchiveChromeSysroot(self):
+    """Generate and upload sysroot for building Chrome."""
+    assert self.archive_path.startswith(self._build_root)
+    extra_env = {}
+    if self._build_config['useflags']:
+      extra_env['USE'] = ' '.join(self._build_config['useflags'])
+    in_chroot_path = git.ReinterpretPathForChroot(self.archive_path)
+    cmd = ['cros_generate_sysroot', '--out-dir', in_chroot_path, '--board',
+           self._current_board, '--package', constants.CHROME_CP]
+    cros_build_lib.RunCommand(cmd, cwd=self._build_root, enter_chroot=True,
+                              extra_env=extra_env)
+    self._upload_queue.put([constants.CHROME_SYSROOT_TAR])
+
+  def _ArchiveChromeEbuildEnv(self):
+    """Generate and upload Chrome ebuild environment."""
+    chrome_dir = self.archive_stage.SingleMatchGlob(
+        os.path.join(self._pkg_dir, constants.CHROME_CP) + '-*')
+    env_bzip = os.path.join(chrome_dir, 'environment.bz2')
+    with osutils.TempDir(prefix='chrome-sdk-stage') as tempdir:
+      # Convert from bzip2 to tar format.
+      bzip2 = cros_build_lib.FindCompressor(cros_build_lib.COMP_BZIP2)
+      cros_build_lib.RunCommand(
+          [bzip2, '-d', env_bzip, '-c'],
+          log_stdout_to_file=os.path.join(tempdir, constants.CHROME_ENV_FILE))
+      env_tar = os.path.join(self.archive_path, constants.CHROME_ENV_TAR)
+      cros_build_lib.CreateTarball(env_tar, tempdir)
+      self._upload_queue.put([os.path.basename(env_tar)])
+
+  def _VerifyChromeDeployed(self, tempdir):
+    """Check to make sure deploy_chrome ran correctly."""
+    if not os.path.exists(os.path.join(tempdir, 'chrome')):
+      raise AssertionError('deploy_chrome did not run successfully!')
+
+  def _VerifySDKEnvironment(self):
+    """Make sure the SDK environment is set up properly."""
+    # If the environment wasn't set up, then the output directory wouldn't be
+    # created after 'gclient runhooks'.
+    # TODO: Make this check actually look at the environment.
+    if not os.path.exists(self.out_board_dir):
+      raise AssertionError('%s not created!' % self.out_board_dir)
+
+  def _BuildChrome(self, sdk_cmd):
+    """Use the generated SDK to build Chrome."""
+    # Validate fetching of the SDK and setting everything up.
+    sdk_cmd.Run(['true'])
+    # Actually build chromium.
+    sdk_cmd.Run(['gclient', 'runhooks'])
+    self._VerifySDKEnvironment()
+    sdk_cmd.Ninja()
+
+  def _TestDeploy(self, sdk_cmd):
+    """Test SDK deployment."""
+    with osutils.TempDir(prefix='chrome-sdk-stage') as tempdir:
+      # Use the TOT deploy_chrome.
+      script_path = os.path.join(
+          self._build_root, constants.CHROMITE_BIN_SUBDIR, 'deploy_chrome')
+      sdk_cmd.Run([script_path, '--build-dir',
+                   os.path.join(self.out_board_dir, 'Release'),
+                   '--staging-only', '--staging-dir', tempdir])
+      self._VerifyChromeDeployed(tempdir)
+
+  def PerformStage(self):
+    upload_metadata = functools.partial(
+        self.UploadMetadata, upload_queue=self._upload_queue)
+    steps = [self._BuildAndArchiveChromeSysroot, self._ArchiveChromeEbuildEnv,
+             upload_metadata]
+    with self.ArtifactUploader(self._upload_queue, archive=False):
+      parallel.RunParallelSteps(steps)
+
+      with osutils.TempDir(prefix='chrome-sdk-cache') as tempdir:
+        cache_dir = os.path.join(tempdir, 'cache')
+        extra_args = ['--cwd', self.chrome_src, '--sdk-path', self.archive_path]
+        sdk_cmd = commands.ChromeSDK(
+            self._build_root, self._current_board, chrome_src=self.chrome_src,
+            goma=self._build_config['chrome_sdk_goma'],
+            extra_args=extra_args, cache_dir=cache_dir)
+        self._BuildChrome(sdk_cmd)
+        self._TestDeploy(sdk_cmd)
+
+
 class BuildImageStage(BuildPackagesStage):
   """Build standard Chromium OS images."""
 
@@ -2383,10 +2488,6 @@
     self._upload_queue = multiprocessing.Queue()
     self._upload_symbols_queue = multiprocessing.Queue()
 
-    self._pkg_dir = os.path.join(
-        self._build_root, constants.DEFAULT_CHROOT_DIR,
-        'build', self._current_board, 'var', 'db', 'pkg')
-
     # Setup the archive path. This is used by other stages.
     self._SetupArchivePath()
 
@@ -2456,10 +2557,10 @@
       osutils.SafeUnlink(os.path.join(self.archive_path,
                                       commands.UPLOADED_LIST_FILENAME))
 
-    os.makedirs(self.archive_path)
+    osutils.SafeMakedirs(self.archive_path)
 
   @staticmethod
-  def _SingleMatchGlob(path_pattern):
+  def SingleMatchGlob(path_pattern):
     """Returns the last match (after sort) if multiple found."""
     files = glob.glob(path_pattern)
     files.sort()
@@ -2479,40 +2580,12 @@
     pkg_dir = os.path.join(
         self._build_root, constants.DEFAULT_CHROOT_DIR, 'build',
         self._current_board, 'stripped-packages')
-    chrome_tarball = self._SingleMatchGlob(
+    chrome_tarball = self.SingleMatchGlob(
         os.path.join(pkg_dir, constants.CHROME_CP) + '-*')
     filename = os.path.basename(chrome_tarball)
     os.link(chrome_tarball, os.path.join(self.archive_path, filename))
     self._upload_queue.put([filename])
 
-  def BuildAndArchiveChromeSysroot(self):
-    """Generate and upload sysroot for building Chrome."""
-    assert self.archive_path.startswith(self._build_root)
-    extra_env = {}
-    if self._build_config['useflags']:
-      extra_env['USE'] = ' '.join(self._build_config['useflags'])
-    in_chroot_path = git.ReinterpretPathForChroot(self.archive_path)
-    cmd = ['cros_generate_sysroot', '--out-dir', in_chroot_path, '--board',
-           self._current_board, '--package', constants.CHROME_CP]
-    cros_build_lib.RunCommand(cmd, cwd=self._build_root, enter_chroot=True,
-                              extra_env=extra_env)
-    self._upload_queue.put([constants.CHROME_SYSROOT_TAR])
-
-  def ArchiveChromeEbuildEnv(self):
-    """Generate and upload Chrome ebuild environment."""
-    chrome_dir = self._SingleMatchGlob(
-        os.path.join(self._pkg_dir, constants.CHROME_CP) + '-*')
-    env_bzip = os.path.join(chrome_dir, 'environment.bz2')
-    with osutils.TempDir() as tempdir:
-      # Convert from bzip2 to tar format.
-      bzip2 = cros_build_lib.FindCompressor(cros_build_lib.COMP_BZIP2)
-      cros_build_lib.RunCommand(
-          [bzip2, '-d', env_bzip, '-c'],
-          log_stdout_to_file=os.path.join(tempdir, constants.CHROME_ENV_FILE))
-      env_tar = os.path.join(self.archive_path, constants.CHROME_ENV_TAR)
-      cros_build_lib.CreateTarball(env_tar, tempdir)
-      self._upload_queue.put([os.path.basename(env_tar)])
-
   def BuildAndArchiveDeltaSysroot(self):
     """Generate and upload delta sysroot for initial build_packages."""
     extra_env = {}
@@ -2558,8 +2631,6 @@
     #          \- ArchiveHWQual
     #       \- PushImage (blocks on BuildAndArchiveAllImages)
     #    \- ArchiveStrippedChrome
-    #    \- BuildAndArchiveChromeSysroot
-    #    \- ArchiveChromeEbuildEnv
     #    \- ArchiveImageScripts
 
     def ArchiveDebugSymbols():
@@ -2736,9 +2807,7 @@
       # Run archiving steps in parallel.
       steps = [ArchiveReleaseArtifacts]
       if config['images']:
-        steps.extend(
-            [self.ArchiveStrippedChrome, self.BuildAndArchiveChromeSysroot,
-             self.ArchiveChromeEbuildEnv, ArchiveImageScripts])
+        steps.extend([self.ArchiveStrippedChrome, ArchiveImageScripts])
       if config['create_delta_sysroot']:
         steps.append(self.BuildAndArchiveDeltaSysroot)
 
diff --git a/buildbot/cbuildbot_stages_unittest.py b/buildbot/cbuildbot_stages_unittest.py
index b5ea75e..65e1007 100755
--- a/buildbot/cbuildbot_stages_unittest.py
+++ b/buildbot/cbuildbot_stages_unittest.py
@@ -61,6 +61,7 @@
     self.bot_id = 'x86-generic-paladin'
     self.build_config = copy.deepcopy(config.config[self.bot_id])
     self.build_root = os.path.join(self.tempdir, self.BUILDROOT)
+    osutils.SafeMakedirs(os.path.join(self.build_root, '.repo'))
     self._boards = self.build_config['boards']
     self._current_board = self._boards[0] if self._boards else None
 
@@ -481,7 +482,7 @@
 
     # Prepare a fake chroot.
     self.fake_chroot = os.path.join(self.build_root, 'chroot/build/amd64-host')
-    os.makedirs(self.fake_chroot)
+    osutils.SafeMakedirs(self.fake_chroot)
     osutils.Touch(os.path.join(self.fake_chroot, 'file'))
     for package, v in self.fake_packages:
       cpv = portage_utilities.SplitCPV('%s-%s' % (package, v))
@@ -1020,7 +1021,8 @@
 
     # The buildtools manifest doesn't have any overlays. In this case, we can't
     # find any toolchains.
-    overlays = portage_utilities.FindOverlays(constants.BOTH_OVERLAYS, None)
+    overlays = portage_utilities.FindOverlays(
+        constants.BOTH_OVERLAYS, board=None, buildroot=self.build_root)
     overlay_tuples = ['i686-pc-linux-gnu', 'arm-none-eabi']
     self.assertEquals(json_data['toolchain-tuple'],
                       overlay_tuples if overlays else [])
@@ -1070,24 +1072,6 @@
     # pylint: disable=E1101
     self.assertEquals(commands.PushImages.call_count, 0)
 
-  def testChromeEnvironment(self):
-    """Test that the Chrome environment is built."""
-    # Create the chrome environment compressed file.
-    stage = self.ConstructStage()
-    chrome_env_dir = os.path.join(
-        stage._pkg_dir, constants.CHROME_CP + '-25.3643.0_rc1')
-    env_file = os.path.join(chrome_env_dir, 'environment')
-    osutils.Touch(env_file, makedirs=True)
-
-    cros_build_lib.RunCommand(['bzip2', env_file])
-
-    # Run the code.
-    stage.ArchiveChromeEbuildEnv()
-
-    env_tar = stage._upload_queue.get()[0]
-    env_tar = os.path.join(stage.archive_path, env_tar)
-    self.assertTrue(os.path.exists(env_tar))
-    cros_test_lib.VerifyTarball(env_tar, ['./', 'environment'])
 
   def ConstructStageForArchiveStep(self):
     """Stage construction for archive steps."""
@@ -1825,6 +1809,53 @@
     self.runTrybotTest(launching=2, waiting=1, failed=1, runs=3)
 
 
+class ChromeSDKStageTest(AbstractStageTest, cros_test_lib.LoggingTestCase):
+  """Verify stage that creates the chrome-sdk and builds chrome with it."""
+
+  def setUp(self):
+    self.build_config = copy.deepcopy(config.config['link-paladin'])
+    self.StartPatcher(ArchiveStageMock())
+    self.StartPatcher(parallel_unittest.ParallelMock())
+    self.options.chrome_root = '/tmp/non-existent'
+
+  def ConstructStage(self):
+    archive_stage = stages.ArchiveStage(self.options, self.build_config,
+                                        self._current_board, None)
+    return stages.ChromeSDKStage(self.options, self.build_config,
+                                 self._current_board, archive_stage)
+
+  def testIt(self):
+    """A simple run-through test."""
+    rc_mock = self.StartPatcher(cros_build_lib_unittest.RunCommandMock())
+    rc_mock.SetDefaultCmdResult()
+    self.PatchObject(stages.ChromeSDKStage, '_ArchiveChromeEbuildEnv',
+                     autospec=True)
+    self.PatchObject(stages.ChromeSDKStage, '_VerifyChromeDeployed',
+                     autospec=True)
+    self.PatchObject(stages.ChromeSDKStage, '_VerifySDKEnvironment',
+                     autospec=True)
+    self.RunStage()
+
+  def testChromeEnvironment(self):
+    """Test that the Chrome environment is built."""
+    # Create the chrome environment compressed file.
+    stage = self.ConstructStage()
+    chrome_env_dir = os.path.join(
+        stage._pkg_dir, constants.CHROME_CP + '-25.3643.0_rc1')
+    env_file = os.path.join(chrome_env_dir, 'environment')
+    osutils.Touch(env_file, makedirs=True)
+
+    cros_build_lib.RunCommand(['bzip2', env_file])
+
+    # Run the code.
+    stage._ArchiveChromeEbuildEnv()
+
+    env_tar_base = stage._upload_queue.get()[0]
+    env_tar = os.path.join(stage.archive_path, env_tar_base)
+    self.assertTrue(os.path.exists(env_tar))
+    cros_test_lib.VerifyTarball(env_tar, ['./', 'environment'])
+
+
 class BranchUtilStageTest(AbstractStageTest, cros_test_lib.LoggingTestCase):
   """Tests for branch creation/deletion."""
 
diff --git a/buildbot/constants.py b/buildbot/constants.py
index 3b1d29f..d875c88 100644
--- a/buildbot/constants.py
+++ b/buildbot/constants.py
@@ -349,6 +349,7 @@
 BASE_IMAGE_BIN = '%s.bin' % BASE_IMAGE_NAME
 IMAGE_SCRIPTS_NAME = 'image_scripts'
 IMAGE_SCRIPTS_TAR = '%s.tar.xz' % IMAGE_SCRIPTS_NAME
+
 METADATA_JSON = 'metadata.json'
 METADATA_STAGE_JSON = 'metadata_%(stage)s.json'
 DELTA_SYSROOT_TAR = 'delta_sysroot.tar.xz'
diff --git a/cros/commands/cros_chrome_sdk.py b/cros/commands/cros_chrome_sdk.py
index 53574fe..f277f20 100644
--- a/cros/commands/cros_chrome_sdk.py
+++ b/cros/commands/cros_chrome_sdk.py
@@ -853,6 +853,13 @@
           # BASH_ENV, and ignores the --rcfile flag.
           extra_env = {'BASH_ENV': rcfile}
 
+        # Bash behaves differently when it detects that it's being launched by
+        # sshd - it ignores the BASH_ENV variable.  So prevent ssh-related
+        # environment variables from being passed through.
+        os.environ.pop('SSH_CLIENT', None)
+        os.environ.pop('SSH_CONNECTION', None)
+        os.environ.pop('SSH_TTY', None)
+
         cmd_result = cros_build_lib.RunCommand(
             bash_cmd, print_cmd=False, debug_level=logging.CRITICAL,
             error_code_ok=True, extra_env=extra_env, cwd=self.options.cwd)
diff --git a/lib/gs.py b/lib/gs.py
index 79198ab..3403213 100644
--- a/lib/gs.py
+++ b/lib/gs.py
@@ -219,6 +219,8 @@
     if not path.startswith(BASE_GS_URL):
       # gsutil doesn't support cat-ting a local path, so just run 'cat' in that
       # case.
+      kwargs.pop('retries', None)
+      kwargs.pop('headers', None)
       return cros_build_lib.RunCommand(['cat', path], **kwargs)
     return self._DoCommand(['cat', path], **kwargs)
 
@@ -343,6 +345,11 @@
   def LS(self, path, **kwargs):
     """Does a directory listing of the given gs path."""
     kwargs['redirect_stdout'] = True
+    if not path.startswith(BASE_GS_URL):
+      # gsutil doesn't support listing a local path, so just run 'ls'.
+      kwargs.pop('retries', None)
+      kwargs.pop('headers', None)
+      return cros_build_lib.RunCommand(['ls', path], **kwargs)
     return self._DoCommand(['ls', '--', path], **kwargs)
 
   def SetACL(self, upload_url, acl=None):
diff --git a/lib/toolchain.py b/lib/toolchain.py
index f4aba2c..679a125 100644
--- a/lib/toolchain.py
+++ b/lib/toolchain.py
@@ -80,13 +80,14 @@
   return targets
 
 
-def GetToolchainsForBoard(board):
+def GetToolchainsForBoard(board, buildroot=constants.SOURCE_ROOT):
   """Get a list of toolchain tuples for a given board name
 
   returns the list of toolchain tuples for the given board
   """
   overlays = portage_utilities.FindOverlays(
-      constants.BOTH_OVERLAYS, None if board in ('all', 'sdk') else board)
+      constants.BOTH_OVERLAYS, None if board in ('all', 'sdk') else board,
+      buildroot=buildroot)
   toolchains = GetTuplesForOverlays(overlays)
   if board == 'sdk':
     toolchains = FilterToolchains(toolchains, 'sdk', True)
diff --git a/scripts/cbuildbot.py b/scripts/cbuildbot.py
index 8c9f384..b558217 100644
--- a/scripts/cbuildbot.py
+++ b/scripts/cbuildbot.py
@@ -390,11 +390,14 @@
                      config=config)
       self._RunStage(stages.UnitTestStage, board, config=config)
       return
-    stage_list = [[stages.VMTestStage, board, archive_stage],
-                  [stages.SignerTestStage, board, archive_stage],
-                  [stages.UnitTestStage, board],
-                  [stages.UploadPrebuiltsStage, board, archive_stage],
-                  [stages.DevInstallerPrebuiltsStage, board, archive_stage]]
+    stage_list = []
+    if self.options.chrome_sdk and self.build_config['chrome_sdk']:
+      stage_list.append([stages.ChromeSDKStage, board, archive_stage])
+    stage_list += [[stages.VMTestStage, board, archive_stage],
+                   [stages.SignerTestStage, board, archive_stage],
+                   [stages.UnitTestStage, board],
+                   [stages.UploadPrebuiltsStage, board, archive_stage],
+                   [stages.DevInstallerPrebuiltsStage, board, archive_stage]]
 
     # We can not run hw tests without archiving the payloads.
     if self.options.archive:
@@ -987,6 +990,10 @@
   group.add_remote_option('--nocgroups', action='store_false', dest='cgroups',
                           default=True,
                           help='Disable cbuildbots usage of cgroups.')
+  group.add_remote_option('--nochromesdk', action='store_false',
+                          dest='chrome_sdk', default=True,
+                          help="Don't run the ChromeSDK stage which builds "
+                               "Chrome outside of the chroot.")
   group.add_remote_option('--noprebuilts', action='store_false',
                           dest='prebuilts', default=True,
                           help="Don't upload prebuilts.")