Revert "Revert "GS Cache: using vpython""
This reverts commit 30c6bbd0a05990b744de7ba9f501973172d612a7.
The original CL got reverted due to a bug in chromite. Now it has been
fixed in crosreview/1050876. Thus this CL can re-merged.
The original review for the CL is crosreview/1038646.
BUG=chromium:824580
TEST=None
Change-Id: I6112af5d9d71205dcadebe4223a60c3a2526a779
Reviewed-on: https://chromium-review.googlesource.com/1056158
Commit-Ready: Congbin Guo <guocb@chromium.org>
Tested-by: Congbin Guo <guocb@chromium.org>
Reviewed-by: Congbin Guo <guocb@chromium.org>
diff --git a/gs_cache/.vpython b/gs_cache/.vpython
new file mode 100644
index 0000000..95ff3f5
--- /dev/null
+++ b/gs_cache/.vpython
@@ -0,0 +1,66 @@
+python_version: "2.7"
+
+wheel: <
+ name: "infra/python/wheels/attrs-py2_py3"
+ version: "version:17.4.0"
+>
+
+wheel: <
+ name: "infra/python/wheels/backports_functools_lru_cache-py2_py3"
+ version: "version:1.5"
+>
+
+wheel: <
+ name: "infra/python/wheels/cheroot-py2_py3"
+ version: "version:6.2.4"
+>
+
+wheel: <
+ name: "infra/python/wheels/cherrypy-py2_py3"
+ version: "version:14.2.0"
+>
+
+wheel: <
+ name: "infra/python/wheels/funcsigs-py2_py3"
+ version: "version:1.0.2"
+>
+
+wheel: <
+ name: "infra/python/wheels/more-itertools-py2_py3"
+ version: "version:4.1.0"
+>
+
+wheel: <
+ name: "infra/python/wheels/pluggy-py2_py3"
+ version: "version:0.6.0"
+>
+
+wheel: <
+ name: "infra/python/wheels/portend-py2_py3"
+ version: "version:2.2"
+>
+
+wheel: <
+ name: "infra/python/wheels/py-py2_py3"
+ version: "version:1.5.3"
+>
+
+wheel: <
+ name: "infra/python/wheels/pytest-py2_py3"
+ version: "version:3.5.0"
+>
+
+wheel: <
+ name: "infra/python/wheels/pytz-py2_py3"
+ version: "version:2018.4"
+>
+
+wheel: <
+ name: "infra/python/wheels/six-py2_py3"
+ version: "version:1.11.0"
+>
+
+wheel: <
+ name: "infra/python/wheels/tempora-py2_py3"
+ version: "version:1.11"
+>
diff --git a/gs_cache/chromite b/gs_cache/chromite
new file mode 120000
index 0000000..d780b88
--- /dev/null
+++ b/gs_cache/chromite
@@ -0,0 +1 @@
+../../../../chromite
\ No newline at end of file
diff --git a/gs_cache/gs_archive_server.py b/gs_cache/gs_archive_server.py
new file mode 100644
index 0000000..958904f
--- /dev/null
+++ b/gs_cache/gs_archive_server.py
@@ -0,0 +1,131 @@
+# Copyright 2018 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""The backend service of Google Storage Cache Server.
+
+Run `./bin/gs_archive_server` to start the server. After started, it listens on
+a TCP port and/or an Unix domain socket. The latter performs better when work
+with a local hosted reverse proxy server, e.g. Nginx.
+
+The server accepts below requests:
+ - GET /download/<bucket>/path/to/file: download the file from google storage
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import argparse
+import os
+import sys
+
+import cherrypy
+
+from chromite.lib import cros_logging as logging
+from chromite.lib import gs
+
+# some http status codes
+_HTTP_UNAUTHORIZED = 401
+_HTTP_NOT_FOUND = 404
+_HTTP_SERVICE_UNAVAILABLE = 503
+
+_logger = logging.getLogger(__file__)
+
+
+def _log(*args, **kwargs):
+ """A wrapper function of logging.debug/info, etc."""
+ level = kwargs.pop('level', logging.DEBUG)
+ _logger.log(level, extra=cherrypy.request.headers, *args, **kwargs)
+
+
+class GSArchiveServer(object):
+ """The backend of Google Storage Cache server."""
+
+ def __init__(self):
+ self._gsutil = gs.GSContext()
+
+ @cherrypy.expose
+ def download(self, *args):
+ """Download a file from Google Storage.
+
+ For example: GET /download/bucket/path/to/file. This downloads the file
+ gs://bucket/path/to/file.
+
+ Args:
+ *args: All parts of the GS file path without gs:// prefix.
+
+ Returns:
+ The stream of downloaded file.
+ """
+ path = 'gs://%s' % '/'.join(args)
+
+ _log('Downloading %s', path, level=logging.INFO)
+ try:
+ stat = self._gsutil.Stat(path)
+ content = self._gsutil.StreamingCat(path)
+ except gs.GSNoSuchKey as err:
+ raise cherrypy.HTTPError(_HTTP_NOT_FOUND, err.message)
+ except gs.GSCommandError as err:
+ if "You aren't authorized to read" in err.result.error:
+ status = _HTTP_UNAUTHORIZED
+ else:
+ status = _HTTP_SERVICE_UNAVAILABLE
+ raise cherrypy.HTTPError(status, '%s: %s' % (err.message,
+ err.result.error))
+
+ cherrypy.response.headers.update({
+ 'Content-Type': stat.content_type,
+ 'Accept-Ranges': 'bytes',
+ 'Content-Length': stat.content_length,
+ })
+ _log('Download complete.')
+
+ return content
+
+ # pylint:disable=protected-access
+ download._cp_config = {'response.stream': True}
+
+
+def parse_args(argv):
+ """Parse arguments."""
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('-s', '--socket', help='Unix domain socket to bind')
+ parser.add_argument('-p', '--port', type=int, default=8080,
+ help='Port number to listen, default: %(default)s.')
+ return parser.parse_args(argv)
+
+
+def setup_logger():
+ """Setup logger."""
+ formatter = logging.Formatter(
+ '%(module)s:%(asctime)-15s [%(Remote-Addr)s:%(thread)d] %(levelname)s:'
+ ' %(message)s')
+ handler = logging.StreamHandler(sys.stdout)
+ handler.setFormatter(formatter)
+ _logger.setLevel(logging.DEBUG)
+ _logger.addHandler(handler)
+
+
+def main(argv):
+ """Main function."""
+ args = parse_args(argv)
+ setup_logger()
+
+ if args.socket:
+ # in order to allow group user writing to domain socket, the directory
+ # should have GID bit set, i.e. g+s
+ os.umask(0002)
+
+ # pylint:disable=protected-access
+ domain_server = cherrypy._cpserver.Server()
+ domain_server.socket_file = args.socket
+ domain_server.subscribe()
+
+ cherrypy.config.update({'server.socket_port': args.port,
+ 'server.socket_host': '127.0.0.1'})
+ cherrypy.quickstart(GSArchiveServer())
+
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv[1:]))
diff --git a/gs_cache/gs_archive_server_test.py b/gs_cache/gs_archive_server_test.py
new file mode 100644
index 0000000..c6047f5
--- /dev/null
+++ b/gs_cache/gs_archive_server_test.py
@@ -0,0 +1,63 @@
+# Copyright 2018 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""Tests for gs_archive_server."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import unittest
+
+import cherrypy
+from cherrypy.test import helper
+
+import gs_archive_server
+from chromite.lib import gs
+
+_DIR = '/gs_archive_server_test'
+# some REAL files and info on Google Storage
+_TEST_DATA = {
+ 'a_plain_file': {
+ 'path': '%s/README.md' % _DIR,
+ 'mime': 'application/octet-stream',
+ 'size': 139
+ },
+}
+
+
+def access_to_gs():
+ """Skip some tests if we cannot access google storage."""
+ return gs.GSContext()._TestGSLs() # pylint:disable=protected-access
+
+
+@unittest.skipUnless(access_to_gs(), 'Have no access to google storage')
+class UnmockedGSArchiveServerTest(helper.CPWebCase):
+ """Some integration tests using cherrypy test framework."""
+ @staticmethod
+ def setup_server():
+ """An API used by cherrypy to setup test environment."""
+ cherrypy.tree.mount(gs_archive_server.GSArchiveServer())
+
+ def test_download_a_file(self):
+ """Test normal files downloading."""
+ tested_file = _TEST_DATA['a_plain_file']
+ self.getPage('/download%(path)s' % tested_file)
+ self.assertStatus(200)
+ self.assertHeader('Content-Type', tested_file['mime'])
+ self.assertEquals(len(self.body), tested_file['size'])
+
+ def test_download_a_non_existing_file(self):
+ """Test downloading non-existing files."""
+ self.getPage('/download/chromeos-images-archive/existing/file')
+ self.assertStatus(404)
+
+ def test_download_against_unauthorized_bucket(self):
+ """Test downloading from unauthorized bucket."""
+ self.getPage('/download/another_bucket/file')
+ self.assertStatus(401)
+
+
+if __name__ == "__main__":
+ unittest.main()