Re-land "Split up artifacts to download some in the foreground/background.""
This refactors the devserver to place artifacts into their own classes and
enables us to download certain artifacts in the foreground and some in the
background. I also add wait_for_status, an RPC you can call using the same
archive_url to determine whether background artifacts have finished
downloading (a blocking call).
My next change is to modify the dynamic_suite code to do:
Download()
Process Duts
Wait_For_Status
Run tests.
BUG=chromium-os:27285
TEST=Ran all unittests + pylint -- added a whole bunch of tests.
Ran downloader from localhost?downloader and saw artifacts staged correctly.
Change-Id: I14c888dcc08cf4df2719915fe2fe88d52fbcdc62
Reviewed-on: https://gerrit.chromium.org/gerrit/19244
Tested-by: Chris Sosa <sosa@chromium.org>
Reviewed-by: Scott Zawalski <scottz@chromium.org>
Commit-Ready: Chris Sosa <sosa@chromium.org>
diff --git a/devserver.py b/devserver.py
index 35c48e6..9249314 100755
--- a/devserver.py
+++ b/devserver.py
@@ -14,6 +14,7 @@
import autoupdate
import devserver_util
+import downloader
CACHED_ENTRIES = 12
@@ -23,6 +24,11 @@
updater = None
+def DevServerError(Exception):
+ """Exception class used by this module."""
+ pass
+
+
def _LeadingWhiteSpaceCount(string):
"""Count the amount of leading whitespace in a string.
@@ -52,7 +58,7 @@
for line in func.__doc__.splitlines():
leading_space = _LeadingWhiteSpaceCount(line)
if leading_space > 0:
- line = ' '*leading_space + line
+ line = ' ' * leading_space + line
html_doc.append('<BR>%s' % line)
@@ -190,7 +196,7 @@
def __init__(self):
self._builder = None
- self._downloader = None
+ self._downloader_dict = {}
@cherrypy.expose
def build(self, board, pkg, **kwargs):
@@ -204,6 +210,11 @@
def download(self, **kwargs):
"""Downloads and archives full/delta payloads from Google Storage.
+ This methods downloads artifacts. It may download artifacts in the
+ background in which case a caller should call wait_for_status to get
+ the status of the background artifact downloads. They should use the same
+ args passed to download.
+
Args:
archive_url: Google Storage URL for the build.
@@ -211,10 +222,36 @@
'http://myhost/download?archive_url=gs://chromeos-image-archive/'
'x86-generic/R17-1208.0.0-a1-b338'
"""
- import downloader
- if self._downloader is None:
- self._downloader = downloader.Downloader(updater.static_dir)
- return self._downloader.Download(kwargs['archive_url'])
+ downloader_instance = downloader.Downloader(updater.static_dir)
+ archive_url = kwargs.get('archive_url')
+ if not archive_url:
+ raise DevServerError("Didn't specify the archive_url in request")
+
+ return_obj = downloader_instance.Download(archive_url)
+ self._downloader_dict[archive_url] = downloader_instance
+ return return_obj
+
+ @cherrypy.expose
+ def wait_for_status(self, **kwargs):
+ """Waits for background artifacts to be downloaded from Google Storage.
+
+ Args:
+ archive_url: Google Storage URL for the build.
+
+ Example URL:
+ 'http://myhost/wait_for_status?archive_url=gs://chromeos-image-archive/'
+ 'x86-generic/R17-1208.0.0-a1-b338'
+ """
+ archive_url = kwargs.get('archive_url')
+ if not archive_url:
+ raise DevServerError("Didn't specify the archive_url in request")
+
+ downloader_instance = self._downloader_dict.get(archive_url)
+ if downloader_instance:
+ self._downloader_dict[archive_url] = None
+ return downloader_instance.GetStatusOfBackgroundDownloads()
+ else:
+ raise DevServerError('No download for the given archive_url found.')
@cherrypy.expose
def latestbuild(self, **params):
@@ -361,7 +398,7 @@
# to devserver_dir to keep these unbroken. For now.
archive_dir = options.archive_dir
if not os.path.isabs(archive_dir):
- archive_dir = os.path.realpath(os.path.join(devserver_dir,archive_dir))
+ archive_dir = os.path.realpath(os.path.join(devserver_dir, archive_dir))
_PrepareToServeUpdatesOnly(archive_dir, static_dir)
static_dir = os.path.realpath(archive_dir)
serve_only = True