dev: Let devserver use gsutil functions in chromite.
Together with CL:378576, this CL is to make platform/dev to use GS-related
functions in chromite. This could help reduce the maintain cost and
inconsistency errors caused by different versions of gsutil.
BUG=chromium:640609
TEST=Ran unittest.
Test 2 functions with local devserver: trigger_download &
get_latest_build_in_gs (including xbuddy/latest, xbuddy/latest-official,
xbuddy/latest-{dev|beta|stable}, xbuddy/latest-R53).
Change-Id: I27f97ebebb76ea6b6b5ef9cc4d5b81c524f6968c
Reviewed-on: https://chromium-review.googlesource.com/381693
Commit-Ready: Xixuan Wu <xixuan@chromium.org>
Tested-by: Xixuan Wu <xixuan@chromium.org>
Reviewed-by: Don Garrett <dgarrett@chromium.org>
diff --git a/xbuddy.py b/xbuddy.py
index d0bd8e3..b0fa5f5 100644
--- a/xbuddy.py
+++ b/xbuddy.py
@@ -9,6 +9,7 @@
import cherrypy
import ConfigParser
import datetime
+import distutils.version
import operator
import os
import re
@@ -22,9 +23,16 @@
import common_util
import devserver_constants
import downloader
-import gsutil_util
import log_util
+# Make sure that chromite is available to import.
+import setup_chromite # pylint: disable=unused-import
+
+try:
+ from chromite.lib import gs
+except ImportError as e:
+ gs = None
+
# Module-local log function.
def _Log(message, *args):
return log_util.LogWithTag('XBUDDY', message, *args)
@@ -205,6 +213,8 @@
else:
self.images_dir = os.path.join(self.GetSourceRoot(), 'src/build/images')
+ self._ctx = gs.GSContext() if gs else None
+
common_util.MkDirP(self._timestamp_folder)
@classmethod
@@ -339,15 +349,57 @@
{'image_dir': image_dir,
'board': board,
'suffix': suffix})
- cmd = 'gsutil cat %s' % latest_addr
- msg = 'Failed to find build at %s' % latest_addr
# Full release + version is in the LATEST file.
- version = gsutil_util.GSUtilRun(cmd, msg)
+ version = self._ctx.Cat(latest_addr)
return devserver_constants.IMAGE_DIR % {'board':board,
'suffix':suffix,
'version':version}
+ def _LS(self, path, list_subdirectory=False):
+ """Does a directory listing of the given gs path.
+
+ Args:
+ path: directory location on google storage to check.
+ list_subdirectory: whether to only list subdirectory for |path|.
+
+ Returns:
+ A list of paths that matched |path|.
+ """
+ if list_subdirectory:
+ return self._ctx.DoCommand(
+ ['ls', '-d', '--', path]).output.splitlines()
+ else:
+ return self._ctx.LS(path)
+
+ def _GetLatestVersionFromGsDir(self, path, list_subdirectory=False,
+ with_release=True):
+ """Returns most recent version number found in a google storage directory.
+
+ This lists out the contents of the given GS bucket or regex to GS buckets,
+ and tries to grab the newest version found in the directory names.
+
+ Args:
+ path: directory location on google storage to check.
+ list_subdirectory: whether to only list subdirectory for |path|.
+ with_release: whether versions include a release milestone (e.g. R12).
+
+ Returns:
+ The most recent version number found.
+ """
+ list_result = self._LS(path, list_subdirectory=list_subdirectory)
+ dir_names = [os.path.basename(p.rstrip('/')) for p in list_result]
+ try:
+ filter_re = re.compile(devserver_constants.VERSION_RE if with_release
+ else devserver_constants.VERSION)
+ versions = filter(filter_re.match, dir_names)
+ latest_version = max(versions, key=distutils.version.LooseVersion)
+ except ValueError:
+ raise gs.GSContextException(
+ 'Failed to find most recent builds at %s' % path)
+
+ return latest_version
+
def _LookupChannel(self, board, suffix, channel='stable',
image_dir=None):
"""Check the channel folder for the version number of interest."""
@@ -355,12 +407,12 @@
_Log("Checking channel '%s' for latest '%s' image", channel, board)
# Due to historical reasons, gs://chromeos-releases uses
# daisy-spring as opposed to the board name daisy_spring. Convert
- # the board name for the lookup.
+ # he board name for the lookup.
channel_dir = devserver_constants.GS_CHANNEL_DIR % {
'channel':channel,
'board':re.sub('_', '-', board)}
- latest_version = gsutil_util.GetLatestVersionFromGSDir(
- channel_dir, with_release=False)
+ latest_version = self._GetLatestVersionFromGsDir(channel_dir,
+ with_release=False)
# Figure out release number from the version number.
image_url = devserver_constants.IMAGE_DIR % {
@@ -371,7 +423,8 @@
gs_url = os.path.join(image_dir, image_url)
# There should only be one match on cros-image-archive.
- full_version = gsutil_util.GetLatestVersionFromGSDir(gs_url)
+ full_version = self._GetLatestVersionFromGsDir(gs_url,
+ list_subdirectory=True)
return devserver_constants.IMAGE_DIR % {'board': board,
'suffix': suffix,
@@ -387,7 +440,8 @@
image_dir = os.path.join(devserver_constants.GS_IMAGE_DIR, image_url)
# Grab the newest version of the ones matched.
- full_version = gsutil_util.GetLatestVersionFromGSDir(image_dir)
+ full_version = self._GetLatestVersionFromGsDir(image_dir,
+ list_subdirectory=True)
return devserver_constants.IMAGE_DIR % {'board': board,
'suffix': suffix,
'version': full_version}
@@ -408,11 +462,11 @@
# is better than with a default suffix added i.e. x86-generic/blah is
# more valuable than x86-generic-release/blah.
for build_id in build_id_as_is, build_id_suffix:
- cmd = 'gsutil ls %s/%s' % (devserver_constants.GS_IMAGE_DIR, build_id)
try:
- version = gsutil_util.GSUtilRun(cmd, None)
+ version = self._ctx.LS(
+ '%s/%s' % (devserver_constants.GS_IMAGE_DIR, build_id))
return build_id
- except gsutil_util.GSUtilError:
+ except (gs.GSCommandError, gs.GSContextException, gs.GSNoSuchKey):
continue
raise XBuddyException('Could not find remote build_id for %s %s' % (
@@ -428,10 +482,8 @@
'board': board,
'suffix': suffix,
'base_version': base_version})
- cmd = 'gsutil cat %s' % latest_addr
- msg = 'Failed to find build at %s' % latest_addr
# Full release + version is in the LATEST file.
- return gsutil_util.GSUtilRun(cmd, msg)
+ return self._ctx.Cat(latest_addr)
def _ResolveVersionToBuildId(self, board, suffix, version, image_dir=None):
"""Handle version aliases for remote payloads in GS.