Move all chromite bin/* content into scripts/
The purpose of this change is to have all chromite scripts run through
a common wrapper. Via this design, all scripts are accessible in python
namespace by default, and it gives us a single point to inject in
the common boilerplate for scripts (signal handling in particular).
Additionally, since PATH w/in the chroot no longer points into a python
namespace, we can expose only what we wish to be accessible- unittests
for example no longer are exposed. We no longer have parallel_emerge and
parallel_emerge.py; just parallel_emerge.
The filtering of what is/isn't exposed was done via checking each/every
script name in the tree to see what consumed it. Certain cases, we
can't yet clean it fully- for chrome-set_ver.py the chrome ebuild
hardcodes the .py (fixed via I9f2d7a14). For cros_sdk.py, we
have to expose that (along with __init__.py) to allow depot_tools
chromite_wrapper to continue working. Once that is fixed/deployed,
__init__.py and cros_sdk.py will be removed, and bin/ will no longer
be a valid python namespace.
Since these scripts are now invoked via a wrapper, all sys.path
modification has been removed- it's no longer necessary.
Finally, permissions were cleaned up. Things that don't need +x no
longer have it.
For reviewing, by its nature this has to be a semi large change. It's
in the reviewers interest to first look at scripts/wrapper.py, then
start looking through the rest.
For reference, the steps taken to generate this change are roughly thus:
1) Move all bin/ content into scripts as proper modules.
2) Update all pathways to access chromite.scripts.
3) Add a generic wrapper script that loads the script and executes it
(scripts.wrapper).
4) For each bin/ entry that is desired/required to be accessible,
add a symlink to bin/<the-name> pointing at ../scripts/wrapper.py.
Exempting where required (code made use of it), we no longer expose
the .py part of the name. For example, you invoke check_gdata_token,
not check_gdata_token.py; the consumer doesn't care about the implementation,
the .py was exposed purely because we had unittests in the same directory.
BUG=chromium-os:26916
TEST=run each unittest in chromite.scripts
TEST=for x in chromite/scripts/*; do [ -L $x ] && \
$x --help &> /dev/null || echo $x; done # basic check of import machinery.
# Note that a few will fail because the script will bail out w/ need to be
# ran as root, for example. Need to run that from the chroot also.
TEST=cros_sdk --replace
TEST=cbuildbot x86-generic-full
Change-Id: I527ef6dd61e95b3fa3c89b61c6cbaf9022332d29
Reviewed-on: https://gerrit.chromium.org/gerrit/17104
Commit-Ready: Brian Harring <ferringb@chromium.org>
Reviewed-by: Brian Harring <ferringb@chromium.org>
Tested-by: Brian Harring <ferringb@chromium.org>
diff --git a/scripts/cros_sdk.py b/scripts/cros_sdk.py
new file mode 100644
index 0000000..66db447
--- /dev/null
+++ b/scripts/cros_sdk.py
@@ -0,0 +1,359 @@
+#!/usr/bin/env python
+# Copyright (c) 2011-2012 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""This script fetches and prepares an SDK chroot.
+"""
+
+import optparse
+import os
+import sys
+import urlparse
+
+from chromite.buildbot import constants
+from chromite.lib import cros_build_lib
+from chromite.lib import sudo
+from chromite.lib import locking
+
+cros_build_lib.STRICT_SUDO = True
+
+
+DEFAULT_URL = 'https://commondatastorage.googleapis.com/chromiumos-sdk/'
+SRC_ROOT = os.path.realpath(constants.SOURCE_ROOT)
+SDK_DIR = os.path.join(SRC_ROOT, 'sdks')
+OVERLAY_DIR = os.path.join(SRC_ROOT, 'src/third_party/chromiumos-overlay')
+SDK_VERSION_FILE = os.path.join(OVERLAY_DIR,
+ 'chromeos/binhost/host/sdk_version.conf')
+
+# TODO(zbehan): Remove the dependency on these, reimplement them in python
+MAKE_CHROOT = [os.path.join(SRC_ROOT, 'src/scripts/sdk_lib/make_chroot.sh')]
+ENTER_CHROOT = [os.path.join(SRC_ROOT, 'src/scripts/sdk_lib/enter_chroot.sh')]
+
+# We need these tools to run. Very common tools (tar,..) are ommited.
+NEEDED_TOOLS = ['curl']
+
+def GetHostArch():
+ """Returns a string for the host architecture"""
+ out = cros_build_lib.RunCommand(['uname', '-m'],
+ redirect_stdout=True, print_cmd=False).output
+ return out.rstrip('\n')
+
+def CheckPrerequisites(needed_tools):
+ """Verifies that the required tools are present on the system.
+
+ This is especially important as this script is intended to run
+ outside the chroot.
+
+ Arguments:
+ needed_tools: an array of string specified binaries to look for.
+
+ Returns:
+ True if all needed tools were found.
+ """
+ for tool in needed_tools:
+ cmd = ['which', tool]
+ try:
+ cros_build_lib.RunCommand(cmd, print_cmd=False, redirect_stdout=True,
+ combine_stdout_stderr=True)
+ except cros_build_lib.RunCommandError:
+ print 'The tool \'' + tool + '\' not found.'
+ print 'Please install the appropriate package in your host.'
+ print 'Example(ubuntu):'
+ print ' sudo apt-get install <packagename>'
+ return False
+ return True
+
+def GetLatestVersion():
+ """Extracts latest version from chromiumos-overlay."""
+ sdk_file = open(SDK_VERSION_FILE)
+ buf = sdk_file.readline().rstrip('\n').split('=')
+ if buf[0] != 'SDK_LATEST_VERSION':
+ raise Exception('Malformed version file')
+ return buf[1].strip('"')
+
+
+def GetArchStageTarball(tarballArch, version):
+ """Returns the URL for a given arch/version"""
+ D = { 'x86_64': 'cros-sdk-' }
+ try:
+ return DEFAULT_URL + D[tarballArch] + version + '.tbz2'
+ except KeyError:
+ sys.exit('Unsupported arch: %s' % (tarballArch,))
+
+
+def FetchRemoteTarball(url):
+ """Fetches a tarball given by url, and place it in sdk/."""
+ tarball_dest = os.path.join(SDK_DIR,
+ os.path.basename(urlparse.urlparse(url).path))
+
+ print 'Downloading sdk: "%s"' % url
+ cmd = ['curl', '-f', '--retry', '5', '-L', '-y', '30',
+ '--output', tarball_dest]
+
+ if not url.startswith('file://') and os.path.exists(tarball_dest):
+ # Only resume for remote URLs. If the file is local, there's no
+ # real speedup, and using the same filename for different files
+ # locally will cause issues.
+ cmd.extend(['-C', '-'])
+
+ # Additionally, certain versions of curl incorrectly fail if
+ # told to resume a file that is fully downloaded, thus do a
+ # check on our own.
+ # see:
+ # https://sourceforge.net/tracker/?func=detail&atid=100976&aid=3482927&group_id=976
+ result = cros_build_lib.RunCommand(['curl', '-I', url],
+ redirect_stdout=True, print_cmd=False)
+ for x in result.output.splitlines():
+ if x.lower().startswith("content-length:"):
+ length = int(x.split(":", 1)[-1].strip())
+ if length == os.path.getsize(tarball_dest):
+ # Fully fetched; bypass invoking curl, since it can screw up handling
+ # of this (>=7.21.4 and up).
+ return tarball_dest
+ break
+
+ cmd.append(url)
+
+ cros_build_lib.RunCommand(cmd)
+ return tarball_dest
+
+
+def BootstrapChroot(chroot_path, stage_url, replace):
+ """Builds a new chroot from source"""
+ cmd = MAKE_CHROOT + ['--chroot', chroot_path,
+ '--nousepkg']
+
+ stage = None
+ if stage_url:
+ stage = FetchRemoteTarball(stage_url)
+
+ if stage:
+ cmd.extend(['--stage3_path', stage])
+
+ if replace:
+ cmd.append('--replace')
+
+ try:
+ cros_build_lib.RunCommand(cmd, print_cmd=False)
+ except cros_build_lib.RunCommandError:
+ print 'Running %r failed!' % cmd
+ sys.exit(1)
+
+
+def CreateChroot(sdk_url, sdk_version, chroot_path, replace):
+ """Creates a new chroot from a given SDK"""
+ if not os.path.exists(SDK_DIR):
+ cros_build_lib.RunCommand(['mkdir', '-p', SDK_DIR], print_cmd=False)
+
+ # Based on selections, fetch the tarball
+ if sdk_url:
+ url = sdk_url
+ else:
+ arch = GetHostArch()
+ if sdk_version:
+ url = GetArchStageTarball(arch, sdk_version)
+ else:
+ url = GetArchStageTarball(arch)
+
+ sdk = FetchRemoteTarball(url)
+
+ # TODO(zbehan): Unpack and install
+ # For now, we simply call make_chroot on the prebuilt chromeos-sdk.
+ # make_chroot provides a variety of hacks to make the chroot useable.
+ # These should all be eliminated/minimised, after which, we can change
+ # this to just unpacking the sdk.
+ cmd = MAKE_CHROOT + ['--stage3_path', sdk,
+ '--chroot', chroot_path]
+
+ if replace:
+ cmd.append('--replace')
+
+ try:
+ cros_build_lib.RunCommand(cmd, print_cmd=False)
+ except cros_build_lib.RunCommandError:
+ print 'Running %r failed!' % cmd
+ sys.exit(1)
+
+
+def DeleteChroot(chroot_path):
+ """Deletes an existing chroot"""
+ cmd = MAKE_CHROOT + ['--chroot', chroot_path,
+ '--delete']
+ try:
+ cros_build_lib.RunCommand(cmd, print_cmd=False)
+ except cros_build_lib.RunCommandError:
+ print 'Running %r failed!' % cmd
+ sys.exit(1)
+
+
+def _CreateLockFile(path):
+ """Create a lockfile via sudo that is writable by current user."""
+ cros_build_lib.SudoRunCommand(['touch', path], print_cmd=False)
+ cros_build_lib.SudoRunCommand(['chown', str(os.getuid()), path],
+ print_cmd=False)
+ cros_build_lib.SudoRunCommand(['chmod', '644', path], print_cmd=False)
+
+
+def EnterChroot(chroot_path, chrome_root, chrome_root_mount, additional_args):
+ """Enters an existing SDK chroot"""
+ cmd = ENTER_CHROOT + ['--chroot', chroot_path]
+ if chrome_root:
+ cmd.extend(['--chrome_root', chrome_root])
+ if chrome_root_mount:
+ cmd.extend(['--chrome_root_mount', chrome_root_mount])
+ if len(additional_args) > 0:
+ cmd.append('--')
+ cmd.extend(additional_args)
+ try:
+ cros_build_lib.RunCommand(cmd, print_cmd=False)
+ except cros_build_lib.RunCommandError:
+ print 'Running %r failed!' % cmd
+ sys.exit(1)
+
+
+def main(argv=None):
+ # TODO(ferringb): make argv required once depot_tools is fixed.
+ if argv is None:
+ argv = sys.argv[1:]
+ usage="""usage: %prog [options] [VAR1=val1 .. VARn=valn -- <args>]
+
+This script manages a local CrOS SDK chroot. Depending on the flags,
+it can download, build or enter a chroot.
+
+Action taken is the following:
+--enter (default) .. Installs and enters a chroot
+--download .. Just download a chroot (enter if combined with --enter)
+--bootstrap .. Builds a chroot from source (enter if --enter)
+--delete .. Removes a chroot
+"""
+ sdk_latest_version = GetLatestVersion()
+ parser = optparse.OptionParser(usage)
+ # Actions:
+ parser.add_option('', '--bootstrap',
+ action='store_true', dest='bootstrap', default=False,
+ help=('Build a new SDK chroot from source'))
+ parser.add_option('', '--delete',
+ action='store_true', dest='delete', default=False,
+ help=('Delete the current SDK chroot'))
+ parser.add_option('', '--download',
+ action='store_true', dest='download', default=False,
+ help=('Download and install a prebuilt SDK'))
+ parser.add_option('', '--enter',
+ action='store_true', dest='enter', default=False,
+ help=('Enter the SDK chroot, possibly (re)create first'))
+
+ # Global options:
+ parser.add_option('', '--chroot',
+ dest='chroot', default=constants.DEFAULT_CHROOT_DIR,
+ help=('SDK chroot dir name [%s]' %
+ constants.DEFAULT_CHROOT_DIR))
+
+ # Additional options:
+ parser.add_option('', '--chrome_root',
+ dest='chrome_root', default='',
+ help=('Mount this chrome root into the SDK chroot'))
+ parser.add_option('', '--chrome_root_mount',
+ dest='chrome_root_mount', default='',
+ help=('Mount chrome into this path inside SDK chroot'))
+ parser.add_option('-r', '--replace',
+ action='store_true', dest='replace', default=False,
+ help=('Replace an existing SDK chroot'))
+ parser.add_option('-u', '--url',
+ dest='sdk_url', default='',
+ help=('''Use sdk tarball located at this url.
+ Use file:// for local files.'''))
+ parser.add_option('-v', '--version',
+ dest='sdk_version', default='',
+ help=('Use this sdk version [%s]' % sdk_latest_version))
+ (options, remaining_arguments) = parser.parse_args(argv)
+
+ # Some sanity checks first, before we ask for sudo credentials.
+ if cros_build_lib.IsInsideChroot():
+ print "This needs to be ran outside the chroot"
+ sys.exit(1)
+
+ if not CheckPrerequisites(NEEDED_TOOLS):
+ sys.exit(1)
+
+ # Default action is --enter, if no other is selected.
+ if not (options.bootstrap or options.download or options.delete):
+ options.enter = True
+
+ # Only --enter can process additional args as passthrough commands.
+ # Warn and exit for least surprise.
+ if len(remaining_arguments) > 0 and not options.enter:
+ print "Additional arguments not permitted, unless running with --enter"
+ parser.print_help()
+ sys.exit(1)
+
+ # Some actions can be combined, as they merely modify how is the chroot
+ # going to be made. The only option that hates all others is --delete.
+ if options.delete and \
+ (options.enter or options.download or options.bootstrap):
+ print "--delete cannot be combined with --enter, --download or --bootstrap"
+ parser.print_help()
+ sys.exit(1)
+ # NOTE: --delete is a true hater, it doesn't like other options either, but
+ # those will hardly lead to confusion. Nobody can expect to pass --version to
+ # delete and actually change something.
+
+ if options.bootstrap and options.download:
+ print "Either --bootstrap or --download, not both"
+ sys.exit(1)
+
+ # Bootstrap will start off from a non-selectable stage3 tarball. Attempts to
+ # select sdk by version are confusing. Warn and exit. We can still specify a
+ # tarball by path or URL though.
+ if options.bootstrap and options.sdk_version:
+ print "Cannot use --version when bootstrapping"
+ parser.print_help()
+ sys.exit(1)
+
+ chroot_path = os.path.join(SRC_ROOT, options.chroot)
+ chroot_path = os.path.abspath(chroot_path)
+ chroot_path = os.path.normpath(chroot_path)
+
+ if not options.sdk_version:
+ sdk_version = sdk_latest_version
+ else:
+ sdk_version = options.sdk_version
+
+ if options.delete and not os.path.exists(chroot_path):
+ print "Not doing anything. The chroot you want to remove doesn't exist."
+ sys.exit(0)
+
+ lock_path = os.path.dirname(chroot_path)
+ lock_path = os.path.join(lock_path,
+ '.%s_lock' % os.path.basename(chroot_path))
+ with sudo.SudoKeepAlive():
+ _CreateLockFile(lock_path)
+ with locking.FileLock(lock_path, 'chroot lock') as lock:
+
+ if options.delete:
+ lock.write_lock()
+ DeleteChroot(chroot_path)
+ sys.exit(0)
+
+ # Print a suggestion for replacement, but not if running just --enter.
+ if os.path.exists(chroot_path) and not options.replace and \
+ (options.bootstrap or options.download):
+ print "Chroot already exists. Run with --replace to re-create."
+
+ # Chroot doesn't exist or asked to replace.
+ if not os.path.exists(chroot_path) or options.replace:
+ lock.write_lock()
+ if options.bootstrap:
+ BootstrapChroot(chroot_path, options.sdk_url,
+ options.replace)
+ else:
+ CreateChroot(options.sdk_url, sdk_version,
+ chroot_path, options.replace)
+ if options.enter:
+ lock.read_lock()
+ EnterChroot(chroot_path, options.chrome_root,
+ options.chrome_root_mount, remaining_arguments)
+
+
+if __name__ == '__main__':
+ main(sys.argv[1:])