devserver: general cleanup
* Unified logging API via module/class-local _Log() calls.
* Renamed modules: devserver_util --> common_util (this is the common
util module, and it saves a lot of characters in many places);
downloadable_artifact --> build_artifact (we don't have any
non-downloadable artifacts); buildutil.py --> build_util.py (for
uniformity)
* Reorganized Artifact (now BuildArtifact) class hierarchy to properly
reflect the inheritance chains.
BUG=None
TEST=Unit tests run successfully
Change-Id: I91f3bd7afe78cc1e2be10b64640515402fb6184b
Reviewed-on: https://gerrit.chromium.org/gerrit/33755
Reviewed-by: Chris Sosa <sosa@chromium.org>
Commit-Ready: Gilad Arnold <garnold@chromium.org>
Tested-by: Gilad Arnold <garnold@chromium.org>
diff --git a/build_artifact.py b/build_artifact.py
new file mode 100644
index 0000000..e7c42e4
--- /dev/null
+++ b/build_artifact.py
@@ -0,0 +1,207 @@
+# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Module containing classes that wrap artifact downloads."""
+
+import cherrypy
+import os
+import re
+import shutil
+import subprocess
+
+import gsutil_util
+import log_util
+
+
+# Names of artifacts we care about.
+DEBUG_SYMBOLS = 'debug.tgz'
+STATEFUL_UPDATE = 'stateful.tgz'
+TEST_IMAGE = 'chromiumos_test_image.bin'
+ROOT_UPDATE = 'update.gz'
+AUTOTEST_PACKAGE = 'autotest.tar'
+AUTOTEST_ZIPPED_PACKAGE = 'autotest.tar.bz2'
+TEST_SUITES_PACKAGE = 'test_suites.tar.bz2'
+IMAGE_ARCHIVE = 'image.zip'
+
+
+class ArtifactDownloadError(Exception):
+ """Error used to signify an issue processing an artifact."""
+ pass
+
+
+class BuildArtifact(log_util.Loggable):
+ """Wrapper around an artifact to download from gsutil.
+
+ The purpose of this class is to download objects from Google Storage
+ and install them to a local directory. There are two main functions, one to
+ download/prepare the artifacts in to a temporary staging area and the second
+ to stage it into its final destination.
+ """
+ def __init__(self, gs_path, tmp_staging_dir, install_path, synchronous=False):
+ """Args:
+ gs_path: Path to artifact in google storage.
+ tmp_staging_dir: Temporary working directory maintained by caller.
+ install_path: Final destination of artifact.
+ synchronous: If True, artifact must be downloaded in the foreground.
+ """
+ self._gs_path = gs_path
+ self._tmp_staging_dir = tmp_staging_dir
+ self._tmp_stage_path = os.path.join(tmp_staging_dir,
+ os.path.basename(self._gs_path))
+ self._synchronous = synchronous
+ self._install_path = install_path
+
+ if not os.path.isdir(self._tmp_staging_dir):
+ os.makedirs(self._tmp_staging_dir)
+
+ if not os.path.isdir(os.path.dirname(self._install_path)):
+ os.makedirs(os.path.dirname(self._install_path))
+
+ def Download(self):
+ """Stages the artifact from google storage to a local staging directory."""
+ gsutil_util.DownloadFromGS(self._gs_path, self._tmp_stage_path)
+
+ def Synchronous(self):
+ """Returns False if this artifact can be downloaded in the background."""
+ return self._synchronous
+
+ def Stage(self):
+ """Moves the artifact from the tmp staging directory to the final path."""
+ shutil.move(self._tmp_stage_path, self._install_path)
+
+ def __str__(self):
+ """String representation for the download."""
+ return '->'.join([self._gs_path, self._tmp_staging_dir, self._install_path])
+
+
+class AUTestPayloadBuildArtifact(BuildArtifact):
+ """Wrapper for AUTest delta payloads which need additional setup."""
+ def Stage(self):
+ super(AUTestPayloadBuildArtifact, self).Stage()
+
+ payload_dir = os.path.dirname(self._install_path)
+ # Setup necessary symlinks for updating.
+ os.symlink(os.path.join(os.pardir, os.pardir, TEST_IMAGE),
+ os.path.join(payload_dir, TEST_IMAGE))
+ os.symlink(os.path.join(os.pardir, os.pardir, STATEFUL_UPDATE),
+ os.path.join(payload_dir, STATEFUL_UPDATE))
+
+
+class TarballBuildArtifact(BuildArtifact):
+ """Wrapper around an artifact to download from gsutil which is a tarball."""
+
+ def _ExtractTarball(self, exclude=None):
+ """Detects whether the tarball is compressed or not based on the file
+ extension and extracts the tarball into the install_path with optional
+ exclude path."""
+
+ exclude_str = '--exclude=%s' % exclude if exclude else ''
+ tarball = os.path.basename(self._tmp_stage_path)
+
+ if re.search('.tar.bz2$', tarball):
+ compress_str = '--use-compress-prog=pbzip2'
+ else:
+ compress_str = ''
+
+ cmd = 'tar xf %s %s %s --directory=%s' % (
+ self._tmp_stage_path, exclude_str, compress_str, self._install_path)
+ msg = 'An error occurred when attempting to untar %s' % self._tmp_stage_path
+
+ try:
+ subprocess.check_call(cmd, shell=True)
+ except subprocess.CalledProcessError, e:
+ raise ArtifactDownloadError('%s %s' % (msg, e))
+
+ def Stage(self):
+ """Changes directory into the install path and untars the tarball."""
+ if not os.path.isdir(self._install_path):
+ os.makedirs(self._install_path)
+
+ self._ExtractTarball()
+
+
+class AutotestTarballBuildArtifact(TarballBuildArtifact):
+ """Wrapper around the autotest tarball to download from gsutil."""
+
+ def Stage(self):
+ """Untars the autotest tarball into the install path excluding test suites.
+ """
+ if not os.path.isdir(self._install_path):
+ os.makedirs(self._install_path)
+
+ self._ExtractTarball(exclude='autotest/test_suites')
+ autotest_dir = os.path.join(self._install_path, 'autotest')
+ autotest_pkgs_dir = os.path.join(autotest_dir, 'packages')
+ if not os.path.exists(autotest_pkgs_dir):
+ os.makedirs(autotest_pkgs_dir)
+
+ if not os.path.exists(os.path.join(autotest_pkgs_dir, 'packages.checksum')):
+ cmd = 'autotest/utils/packager.py upload --repository=%s --all' % (
+ autotest_pkgs_dir)
+ msg = 'Failed to create autotest packages!'
+ try:
+ subprocess.check_call(cmd, cwd=self._tmp_staging_dir,
+ shell=True)
+ except subprocess.CalledProcessError, e:
+ raise ArtifactDownloadError('%s %s' % (msg, e))
+ else:
+ self._LOG('Using pre-generated packages from autotest')
+
+ # TODO(scottz): Remove after we have moved away from the old test_scheduler
+ # code.
+ cmd = 'cp %s/* %s' % (autotest_pkgs_dir, autotest_dir)
+ subprocess.check_call(cmd, shell=True)
+
+
+class DebugTarballBuildArtifact(TarballBuildArtifact):
+ """Wrapper around the debug symbols tarball to download from gsutil."""
+
+ def _ExtractTarball(self):
+ """Extracts debug/breakpad from the tarball into the install_path."""
+ cmd = 'tar xzf %s --directory=%s debug/breakpad' % (
+ self._tmp_stage_path, self._install_path)
+ msg = 'An error occurred when attempting to untar %s' % self._tmp_stage_path
+ try:
+ subprocess.check_call(cmd, shell=True)
+ except subprocess.CalledProcessError, e:
+ raise ArtifactDownloadError('%s %s' % (msg, e))
+
+
+class ZipfileBuildArtifact(BuildArtifact):
+ """A downloadable artifact that is a zipfile.
+
+ This class defines an extra public method for setting the list of files to be
+ extracted upon staging. Staging amounts to unzipping the desired files to the
+ install path.
+
+ """
+
+ def __init__(self, gs_path, tmp_staging_dir, install_path, synchronous=False,
+ unzip_file_list=None):
+ super(ZipfileBuildArtifact, self).__init__(
+ gs_path, tmp_staging_dir, install_path, synchronous)
+ self._unzip_file_list = unzip_file_list
+
+ def _Unzip(self):
+ """Unzip files into the install path."""
+
+ cmd = 'unzip -o %s -d %s%s' % (
+ self._tmp_stage_path,
+ os.path.join(self._install_path),
+ (' ' + ' '.join(self._unzip_file_list)
+ if self._unzip_file_list else ''))
+ self._Log('unzip command: %s' % cmd)
+ msg = 'An error occurred when attempting to unzip %s' % self._tmp_stage_path
+
+ try:
+ subprocess.check_call(cmd, shell=True)
+ except subprocess.CalledProcessError, e:
+ raise ArtifactDownloadError('%s %s' % (msg, e))
+
+ def Stage(self):
+ """Unzip files into the install path."""
+ if not os.path.isdir(self._install_path):
+ os.makedirs(self._install_path)
+
+ self._Unzip()