Update devserver to support downloader other than from Google Storage
Main changes:
1. Restructure artifact wrappers to support both CrOS and Android artifacts.
2. Support different downloaders in devserver.py.
3. Add LaunchControlDownloader class, the functions are to be implemented.
BUG=chromium:512668
TEST=run_unittests, devserver_integration_test.py, guado_moblab (au and dummy)
cros flash and cros stage to guado moblab
Change-Id: Ia350b00a2a5ceaeff6d922600dc84c8fc7295ef9
Reviewed-on: https://chromium-review.googlesource.com/301992
Commit-Ready: Dan Shi <dshi@chromium.org>
Tested-by: Dan Shi <dshi@chromium.org>
Reviewed-by: Dan Shi <dshi@chromium.org>
diff --git a/downloader.py b/downloader.py
index 2a319d6..554ce33 100755
--- a/downloader.py
+++ b/downloader.py
@@ -1,14 +1,24 @@
+#!/usr/bin/python2
+#
# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+"""Downloaders used to download artifacts and files from a given source."""
+
+from __future__ import print_function
+
import collections
+import glob
import os
+import re
+import shutil
import threading
from datetime import datetime
import build_artifact
import common_util
+import gsutil_util
import log_util
@@ -23,7 +33,8 @@
def __init__(self, exceptions):
"""Initialize a DownloaderException instance with a list of exceptions.
- @param exceptions: Exceptions raised when downloading artifacts.
+ Args:
+ exceptions: Exceptions raised when downloading artifacts.
"""
message = 'Exceptions were raised when downloading artifacts.'
Exception.__init__(self, message)
@@ -39,92 +50,40 @@
class Downloader(log_util.Loggable):
"""Downloader of images to the devsever.
+ This is the base class for different types of downloaders, including
+ GoogleStorageDownloader, LocalDownloader and LaunchControlDownloader.
+
Given a URL to a build on the archive server:
- Caches that build and the given artifacts onto the devserver.
- May also initiate caching of related artifacts in the background.
Private class members:
- archive_url: a URL where to download build artifacts from.
static_dir: local filesystem directory to store all artifacts.
build_dir: the local filesystem directory to store artifacts for the given
- build defined by the archive_url.
+ build based on the remote source.
+
+ Public methods must be overridden:
+ Wait: Verifies the local artifact exists and returns the appropriate names.
+ Fetch: Downloads artifact from given source to a local directory.
+ DescribeSource: Gets the source of the download, e.g., a url to GS.
"""
# This filename must be kept in sync with clean_staged_images.py
_TIMESTAMP_FILENAME = 'staged.timestamp'
- def __init__(self, static_dir, archive_url):
+ def __init__(self, static_dir, build_dir, build):
super(Downloader, self).__init__()
- self._archive_url = archive_url
self._static_dir = static_dir
- self._build_dir = Downloader.GetBuildDir(static_dir, archive_url)
+ self._build_dir = build_dir
+ self._build = build
- @staticmethod
- def ParseUrl(path_or_url):
- """Parses |path_or_url| into build relative path and the shorter build name.
+ def GetBuildDir(self):
+ """Returns the path to where the artifacts will be staged."""
+ return self._build_dir
- Args:
- path_or_url: a local path or URL at which build artifacts are archived.
-
- Returns:
- A tuple of (build relative path, short build name)
- """
- if path_or_url.startswith('gs://'):
- return Downloader.ParseGSUrl(path_or_url)
- return Downloader.ParseLocalPath(path_or_url)
-
- @staticmethod
- def ParseGSUrl(archive_url):
- """Parses |path_or_url| into build relative path and the shorter build name.
-
- Parses archive_url into rel_path and build e.g.
- gs://chromeos-image-archive/{rel_path}/{build}.
-
- Args:
- archive_url: a URL at which build artifacts are archived.
-
- Returns:
- A tuple of (build relative path, short build name)
- """
- # The archive_url is of the form gs://server/[some_path/target]/...]/build
- # This function discards 'gs://server/' and extracts the [some_path/target]
- # as rel_path and the build as build.
- sub_url = archive_url.partition('://')[2]
- split_sub_url = sub_url.split('/')
- rel_path = '/'.join(split_sub_url[1:-1])
- build = split_sub_url[-1]
- return rel_path, build
-
- @staticmethod
- def ParseLocalPath(local_path):
- """Parses local_path into rel_path and build.
-
- Parses a local path into rel_path and build e.g.
- /{path to static dir}/{rel_path}/{build}.
-
- Args:
- local_path: a local path that the build artifacts are stored. Must be a
- subpath of the static directory.
-
- Returns:
- A tuple of (build relative path, short build name)
- """
- rel_path = os.path.basename(os.path.dirname(local_path))
- build = os.path.basename(local_path)
- return rel_path, build
-
- @staticmethod
- def GetBuildDir(static_dir, archive_url):
- """Returns the path to where the artifacts will be staged.
-
- Args:
- static_dir: The base static dir that will be used.
- archive_url: The gs path to the archive url.
- """
- # Parse archive_url into rel_path (contains the build target) and
- # build e.g. gs://chromeos-image-archive/{rel_path}/{build}.
- rel_path, build = Downloader.ParseUrl(archive_url)
- return os.path.join(static_dir, rel_path, build)
+ def GetBuild(self):
+ """Returns the path to where the artifacts will be staged."""
+ return self._build
@staticmethod
def TouchTimestampForStaged(directory_path):
@@ -141,8 +100,8 @@
is the only file in that directory. The build could be non-existing, and
the directory should be removed.
- @param directory_path: directory used to stage the image.
-
+ Args:
+ directory_path: directory used to stage the image.
"""
file_name = os.path.join(directory_path, Downloader._TIMESTAMP_FILENAME)
if os.path.exists(file_name) and len(os.listdir(directory_path)) == 1:
@@ -171,9 +130,9 @@
'not a directory.' % (self._archive_url, self._build_dir))
ls_format = collections.namedtuple(
- 'ls', ['name', 'accessed', 'modified', 'size'])
+ 'ls', ['name', 'accessed', 'modified', 'size'])
output_format = ('Name: %(name)s Accessed: %(accessed)s '
- 'Modified: %(modified)s Size: %(size)s bytes.\n')
+ 'Modified: %(modified)s Size: %(size)s bytes.\n')
build_dir_info = 'Listing contents of :%s \n' % self._build_dir
for file_name in os.listdir(self._build_dir):
@@ -186,18 +145,15 @@
build_dir_info += output_format % ls_info._asdict()
return build_dir_info
- def Download(self, artifacts, files, async=False):
+ def Download(self, factory, async=False):
"""Downloads and caches the |artifacts|.
- Downloads and caches the |artifacts|. Returns once these
- are present on the devserver. A call to this will attempt to cache
- non-specified artifacts in the background following the principle of
- spatial locality.
+ Downloads and caches the |artifacts|. Returns once these are present on the
+ devserver. A call to this will attempt to cache non-specified artifacts in
+ the background following the principle of spatial locality.
Args:
- artifacts: A list of artifact names that correspond to
- artifacts defined in artifact_info.py to stage.
- files: A list of filenames to stage from an archive_url.
+ factory: The artifact factory.
async: If True, return without waiting for download to complete.
Raises:
@@ -211,10 +167,6 @@
Downloader.TouchTimestampForStaged(self._build_dir)
# Create factory to create build_artifacts from artifact names.
- build = self.ParseUrl(self._archive_url)[1]
- factory = build_artifact.ArtifactFactory(
- self._build_dir, self._archive_url, artifacts, files,
- build)
background_artifacts = factory.OptionalArtifacts()
if background_artifacts:
self._DownloadArtifactsInBackground(background_artifacts)
@@ -228,20 +180,21 @@
else:
self._DownloadArtifactsSerially(required_artifacts, no_wait=True)
- def IsStaged(self, artifacts, files):
+ def IsStaged(self, factory):
"""Check if all artifacts have been downloaded.
- artifacts: A list of artifact names that correspond to
- artifacts defined in artifact_info.py to stage.
- files: A list of filenames to stage from an archive_url.
- @returns: True if all artifacts are staged.
- @raise exception: that was raised by any artifact when calling Process.
+ Args:
+ factory: An instance of BaseArtifactFactory to be used to check if desired
+ artifacts or files are staged.
+
+ Returns:
+ True if all artifacts are staged.
+
+ Raises:
+ DownloaderException: A wrapper for exceptions raised by any artifact when
+ calling Process.
"""
- # Create factory to create build_artifacts from artifact names.
- build = self.ParseUrl(self._archive_url)[1]
- factory = build_artifact.ArtifactFactory(
- self._build_dir, self._archive_url, artifacts, files, build)
required_artifacts = factory.RequiredArtifacts()
exceptions = [artifact.GetException() for artifact in required_artifacts if
artifact.GetException()]
@@ -266,7 +219,7 @@
"""
try:
for artifact in artifacts:
- artifact.Process(no_wait)
+ artifact.Process(self, no_wait)
except build_artifact.ArtifactDownloadError:
Downloader._TryRemoveStageDir(self._build_dir)
raise
@@ -284,3 +237,196 @@
thread = threading.Thread(target=self._DownloadArtifactsSerially,
args=(artifacts, False))
thread.start()
+
+ def Wait(self, name, is_regex_name, timeout):
+ """Waits for artifact to exist and returns the appropriate names.
+
+ Args:
+ name: Name to look at.
+ is_regex_name: True if the name is a regex pattern.
+ timeout: How long to wait for the artifact to become available.
+
+ Returns:
+ A list of names that match.
+ """
+ raise NotImplementedError()
+
+ def Fetch(self, remote_name, local_path):
+ """Downloads artifact from given source to a local directory.
+
+ Args:
+ remote_name: Remote name of the file to fetch.
+ local_path: Local path to the folder to store fetched file.
+
+ Returns:
+ The path to fetched file.
+ """
+ raise NotImplementedError()
+
+ def DescribeSource(self):
+ """Gets the source of the download, e.g., a url to GS."""
+ raise NotImplementedError()
+
+
+class GoogleStorageDownloader(Downloader):
+ """Downloader of images to the devserver from Google Storage.
+
+ Given a URL to a build on the archive server:
+ - Caches that build and the given artifacts onto the devserver.
+ - May also initiate caching of related artifacts in the background.
+
+ This is intended to be used with ChromeOS.
+
+ Private class members:
+ archive_url: Google Storage URL to download build artifacts from.
+ """
+
+ def __init__(self, static_dir, archive_url):
+ # The archive_url is of the form gs://server/[some_path/target]/...]/build
+ # This function discards 'gs://server/' and extracts the [some_path/target]
+ # as rel_path and the build as build.
+ sub_url = archive_url.partition('://')[2]
+ split_sub_url = sub_url.split('/')
+ rel_path = '/'.join(split_sub_url[1:-1])
+ build = split_sub_url[-1]
+ build_dir = os.path.join(static_dir, rel_path, build)
+
+ super(GoogleStorageDownloader, self).__init__(static_dir, build_dir, build)
+
+ self._archive_url = archive_url
+
+ def Wait(self, name, is_regex_name, timeout):
+ """Waits for artifact to exist and returns the appropriate names.
+
+ Args:
+ name: Name to look at.
+ is_regex_name: True if the name is a regex pattern.
+ timeout: How long to wait for the artifact to become available.
+
+ Returns:
+ A list of names that match.
+
+ Raises:
+ ArtifactDownloadError: An error occurred when obtaining artifact.
+ """
+ names = gsutil_util.GetGSNamesWithWait(
+ name, self._archive_url, str(self), timeout=timeout,
+ is_regex_pattern=is_regex_name)
+ if not names:
+ raise build_artifact.ArtifactDownloadError(
+ 'Could not find %s in Google Storage at %s' %
+ (name, self._archive_url))
+ return names
+
+ def Fetch(self, remote_name, local_path):
+ """Downloads artifact from Google Storage to a local directory."""
+ install_path = os.path.join(local_path, remote_name)
+ gs_path = '/'.join([self._archive_url, remote_name])
+ gsutil_util.DownloadFromGS(gs_path, local_path)
+ return install_path
+
+ def DescribeSource(self):
+ return self._archive_url
+
+
+class LocalDownloader(Downloader):
+ """Downloader of images to the devserver from local storage.
+
+ Given a local path:
+ - Caches that build and the given artifacts onto the devserver.
+ - May also initiate caching of related artifacts in the background.
+
+ Private class members:
+ archive_params: parameters for where to download build artifacts from.
+ """
+
+ def __init__(self, static_dir, source_path):
+ # The local path is of the form /{path to static dir}/{rel_path}/{build}.
+ # local_path must be a subpath of the static directory.
+ self.source_path = source_path
+ rel_path = os.path.basename(os.path.dirname(source_path))
+ build = os.path.basename(source_path)
+ build_dir = os.path.join(static_dir, rel_path, build)
+
+ super(LocalDownloader, self).__init__(static_dir, build_dir, build)
+
+ def Wait(self, name, is_regex_name, timeout):
+ """Verifies the local artifact exists and returns the appropriate names.
+
+ Args:
+ name: Name to look at.
+ is_regex_name: True if the name is a regex pattern.
+ timeout: How long to wait for the artifact to become available.
+
+ Returns:
+ A list of names that match.
+
+ Raises:
+ ArtifactDownloadError: An error occurred when obtaining artifact.
+ """
+ local_path = os.path.join(self.source_path, name)
+ if is_regex_name:
+ filter_re = re.compile(name)
+ for filename in os.listdir(self.source_path):
+ if filter_re.match(filename):
+ return [filename]
+ else:
+ glob_search = glob.glob(local_path)
+ if glob_search and len(glob_search) == 1:
+ return [os.path.basename(glob_search[0])]
+ raise build_artifact.ArtifactDownloadError('Artifact not found.')
+
+ def Fetch(self, remote_name, local_path):
+ """Downloads artifact from Google Storage to a local directory."""
+ install_path = os.path.join(local_path, remote_name)
+ # It's a local path so just copy it into the staged directory.
+ shutil.copyfile(os.path.join(self.source_path, remote_name),
+ install_path)
+ return install_path
+
+ def DescribeSource(self):
+ return self.source_path
+
+
+class LaunchControlDownloader(Downloader):
+ """Downloader of images to the devserver from launch control."""
+
+ def __init__(self, static_dir, build_id, target):
+ """Initialize LaunchControlDownloader.
+
+ Args:
+ static_dir: Root directory to store the build.
+ build_id: Build id of the Android build, e.g., 2155602.
+ target: Target of the Android build, e.g., shamu-userdebug.
+ """
+ build = '%s/%s' % (target, build_id)
+ build_dir = os.path.join(static_dir, '', build)
+
+ self.build_id = build_id
+ self.target = target
+
+ super(LaunchControlDownloader, self).__init__(static_dir, build_dir, build)
+
+ def Wait(self, name, is_regex_name, timeout):
+ """Verifies the local artifact exists and returns the appropriate names.
+
+ Args:
+ name: Name to look at.
+ is_regex_name: True if the name is a regex pattern.
+ timeout: How long to wait for the artifact to become available.
+
+ Returns:
+ A list of names that match.
+
+ Raises:
+ ArtifactDownloadError: An error occurred when obtaining artifact.
+ """
+ raise NotImplementedError()
+
+ def Fetch(self, remote_name, local_path):
+ """Downloads artifact from LaunchControl to a local directory."""
+ install_path = os.path.join(local_path, remote_name)
+ return install_path
+
+ def DescribeSource(self):
+ raise NotImplementedError()