blob: 096e4beee036e8a406faed950b37e19935a7de72 [file] [log] [blame]
joychen3cb228e2013-06-12 12:13:13 -07001# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
Chris Sosa0eecf962014-02-03 14:14:39 -08005"""Main module for parsing and interpreting XBuddy paths for the devserver."""
6
Gilad Arnold5f46d8e2015-02-19 12:17:55 -08007from __future__ import print_function
8
Yiming Chenaab488e2014-11-17 14:49:31 -08009import cherrypy
joychen562699a2013-08-13 15:22:14 -070010import ConfigParser
joychen3cb228e2013-06-12 12:13:13 -070011import datetime
xixuan44b55452016-09-06 15:35:56 -070012import distutils.version
joychen3cb228e2013-06-12 12:13:13 -070013import operator
14import os
joychenf8f07e22013-07-12 17:45:51 -070015import re
joychen3cb228e2013-06-12 12:13:13 -070016import shutil
joychenf8f07e22013-07-12 17:45:51 -070017import time
joychen3cb228e2013-06-12 12:13:13 -070018import threading
19
20import artifact_info
Gabe Black3b567202015-09-23 14:07:59 -070021import build_artifact
22import build_util
joychen3cb228e2013-06-12 12:13:13 -070023import common_util
24import devserver_constants
25import downloader
26import log_util
27
xixuan44b55452016-09-06 15:35:56 -070028# Make sure that chromite is available to import.
29import setup_chromite # pylint: disable=unused-import
30
31try:
32 from chromite.lib import gs
Gwendal Grignouad4cb982017-03-31 11:36:19 -070033except ImportError:
xixuan44b55452016-09-06 15:35:56 -070034 gs = None
35
joychen3cb228e2013-06-12 12:13:13 -070036# Module-local log function.
37def _Log(message, *args):
38 return log_util.LogWithTag('XBUDDY', message, *args)
39
joychen562699a2013-08-13 15:22:14 -070040# xBuddy config constants
41CONFIG_FILE = 'xbuddy_config.ini'
42SHADOW_CONFIG_FILE = 'shadow_xbuddy_config.ini'
43PATH_REWRITES = 'PATH_REWRITES'
44GENERAL = 'GENERAL'
Gilad Arnold896c6d82015-03-13 16:20:29 -070045LOCATION_SUFFIXES = 'LOCATION_SUFFIXES'
joychen921e1fb2013-06-28 11:12:20 -070046
Chris Sosac2abc722013-08-26 17:11:22 -070047# Path for shadow config in chroot.
48CHROOT_SHADOW_DIR = '/mnt/host/source/src/platform/dev'
49
joychen25d25972013-07-30 14:54:16 -070050# XBuddy aliases
51TEST = 'test'
52BASE = 'base'
53DEV = 'dev'
54FULL = 'full_payload'
55RECOVERY = 'recovery'
56STATEFUL = 'stateful'
57AUTOTEST = 'autotest'
Mike Frysingera0e6a282016-09-01 17:29:08 -040058FACTORY_SHIM = 'factory_shim'
joychen25d25972013-07-30 14:54:16 -070059
joychen921e1fb2013-06-28 11:12:20 -070060# Local build constants
joychenc3944cb2013-08-19 10:42:07 -070061ANY = "ANY"
joychen7df67f72013-07-18 14:21:12 -070062LATEST = "latest"
63LOCAL = "local"
64REMOTE = "remote"
Chris Sosa75490802013-09-30 17:21:45 -070065
66# TODO(sosa): Fix a lot of assumptions about these aliases. There is too much
67# implicit logic here that's unnecessary. What should be done:
68# 1) Collapse Alias logic to one set of aliases for xbuddy (not local/remote).
69# 2) Do not use zip when creating these dicts. Better to not rely on ordering.
70# 3) Move alias/artifact mapping to a central module rather than having it here.
71# 4) Be explicit when things are missing i.e. no dev images in image.zip.
72
joychen921e1fb2013-06-28 11:12:20 -070073LOCAL_ALIASES = [
Gilad Arnold5f46d8e2015-02-19 12:17:55 -080074 TEST,
75 DEV,
76 BASE,
77 RECOVERY,
Mike Frysingera0e6a282016-09-01 17:29:08 -040078 FACTORY_SHIM,
Gilad Arnold5f46d8e2015-02-19 12:17:55 -080079 FULL,
80 STATEFUL,
81 ANY,
joychen921e1fb2013-06-28 11:12:20 -070082]
83
84LOCAL_FILE_NAMES = [
Gilad Arnold5f46d8e2015-02-19 12:17:55 -080085 devserver_constants.TEST_IMAGE_FILE,
86 devserver_constants.IMAGE_FILE,
87 devserver_constants.BASE_IMAGE_FILE,
88 devserver_constants.RECOVERY_IMAGE_FILE,
Mike Frysingera0e6a282016-09-01 17:29:08 -040089 devserver_constants.FACTORY_SHIM_IMAGE_FILE,
Gilad Arnold5f46d8e2015-02-19 12:17:55 -080090 devserver_constants.UPDATE_FILE,
91 devserver_constants.STATEFUL_FILE,
92 None, # For ANY.
joychen921e1fb2013-06-28 11:12:20 -070093]
94
95LOCAL_ALIAS_TO_FILENAME = dict(zip(LOCAL_ALIASES, LOCAL_FILE_NAMES))
96
97# Google Storage constants
98GS_ALIASES = [
Gilad Arnold5f46d8e2015-02-19 12:17:55 -080099 TEST,
100 BASE,
101 RECOVERY,
Mike Frysingera0e6a282016-09-01 17:29:08 -0400102 FACTORY_SHIM,
Gilad Arnold5f46d8e2015-02-19 12:17:55 -0800103 FULL,
104 STATEFUL,
105 AUTOTEST,
joychen3cb228e2013-06-12 12:13:13 -0700106]
107
joychen921e1fb2013-06-28 11:12:20 -0700108GS_FILE_NAMES = [
Gilad Arnold5f46d8e2015-02-19 12:17:55 -0800109 devserver_constants.TEST_IMAGE_FILE,
110 devserver_constants.BASE_IMAGE_FILE,
111 devserver_constants.RECOVERY_IMAGE_FILE,
Mike Frysingera0e6a282016-09-01 17:29:08 -0400112 devserver_constants.FACTORY_SHIM_IMAGE_FILE,
Gilad Arnold5f46d8e2015-02-19 12:17:55 -0800113 devserver_constants.UPDATE_FILE,
114 devserver_constants.STATEFUL_FILE,
115 devserver_constants.AUTOTEST_DIR,
joychen3cb228e2013-06-12 12:13:13 -0700116]
117
118ARTIFACTS = [
Gilad Arnold5f46d8e2015-02-19 12:17:55 -0800119 artifact_info.TEST_IMAGE,
120 artifact_info.BASE_IMAGE,
121 artifact_info.RECOVERY_IMAGE,
Mike Frysingera0e6a282016-09-01 17:29:08 -0400122 artifact_info.FACTORY_SHIM_IMAGE,
Gilad Arnold5f46d8e2015-02-19 12:17:55 -0800123 artifact_info.FULL_PAYLOAD,
124 artifact_info.STATEFUL_PAYLOAD,
125 artifact_info.AUTOTEST,
joychen3cb228e2013-06-12 12:13:13 -0700126]
127
joychen921e1fb2013-06-28 11:12:20 -0700128GS_ALIAS_TO_FILENAME = dict(zip(GS_ALIASES, GS_FILE_NAMES))
129GS_ALIAS_TO_ARTIFACT = dict(zip(GS_ALIASES, ARTIFACTS))
joychen3cb228e2013-06-12 12:13:13 -0700130
joychen921e1fb2013-06-28 11:12:20 -0700131LATEST_OFFICIAL = "latest-official"
joychen3cb228e2013-06-12 12:13:13 -0700132
Chris Sosaea734d92013-10-11 11:28:58 -0700133RELEASE = "-release"
joychen3cb228e2013-06-12 12:13:13 -0700134
joychen3cb228e2013-06-12 12:13:13 -0700135
136class XBuddyException(Exception):
137 """Exception classes used by this module."""
138 pass
139
140
141# no __init__ method
142#pylint: disable=W0232
Gilad Arnold5f46d8e2015-02-19 12:17:55 -0800143class Timestamp(object):
joychen3cb228e2013-06-12 12:13:13 -0700144 """Class to translate build path strings and timestamp filenames."""
145
146 _TIMESTAMP_DELIMITER = 'SLASH'
147 XBUDDY_TIMESTAMP_DIR = 'xbuddy_UpdateTimestamps'
148
149 @staticmethod
150 def TimestampToBuild(timestamp_filename):
151 return timestamp_filename.replace(Timestamp._TIMESTAMP_DELIMITER, '/')
152
153 @staticmethod
154 def BuildToTimestamp(build_path):
155 return build_path.replace('/', Timestamp._TIMESTAMP_DELIMITER)
joychen921e1fb2013-06-28 11:12:20 -0700156
157 @staticmethod
158 def UpdateTimestamp(timestamp_dir, build_id):
159 """Update timestamp file of build with build_id."""
160 common_util.MkDirP(timestamp_dir)
joychen562699a2013-08-13 15:22:14 -0700161 _Log("Updating timestamp for %s", build_id)
joychen921e1fb2013-06-28 11:12:20 -0700162 time_file = os.path.join(timestamp_dir,
163 Timestamp.BuildToTimestamp(build_id))
164 with file(time_file, 'a'):
165 os.utime(time_file, None)
joychen3cb228e2013-06-12 12:13:13 -0700166#pylint: enable=W0232
167
168
joychen921e1fb2013-06-28 11:12:20 -0700169class XBuddy(build_util.BuildObject):
joychen3cb228e2013-06-12 12:13:13 -0700170 """Class that manages image retrieval and caching by the devserver.
171
172 Image retrieval by xBuddy path:
173 XBuddy accesses images and artifacts that it stores using an xBuddy
174 path of the form: board/version/alias
175 The primary xbuddy.Get call retrieves the correct artifact or url to where
176 the artifacts can be found.
177
178 Image caching:
179 Images and other artifacts are stored identically to how they would have
180 been if devserver's stage rpc was called and the xBuddy cache replaces
181 build versions on a LRU basis. Timestamps are maintained by last accessed
182 times of representative files in the a directory in the static serve
183 directory (XBUDDY_TIMESTAMP_DIR).
184
185 Private class members:
joychen121fc9b2013-08-02 14:30:30 -0700186 _true_values: used for interpreting boolean values
187 _staging_thread_count: track download requests
188 _timestamp_folder: directory with empty files standing in as timestamps
joychen921e1fb2013-06-28 11:12:20 -0700189 for each image currently cached by xBuddy
joychen3cb228e2013-06-12 12:13:13 -0700190 """
191 _true_values = ['true', 't', 'yes', 'y']
192
193 # Number of threads that are staging images.
194 _staging_thread_count = 0
195 # Lock used to lock increasing/decreasing count.
196 _staging_thread_count_lock = threading.Lock()
197
Gilad Arnoldd04fcab2015-02-19 12:00:45 -0800198 def __init__(self, manage_builds=False, board=None, version=None,
199 images_dir=None, log_screen=True, **kwargs):
joychen921e1fb2013-06-28 11:12:20 -0700200 super(XBuddy, self).__init__(**kwargs)
joychenb0dfe552013-07-30 10:02:06 -0700201
Yiming Chenaab488e2014-11-17 14:49:31 -0800202 if not log_screen:
203 cherrypy.config.update({'log.screen': False})
204
joychen562699a2013-08-13 15:22:14 -0700205 self.config = self._ReadConfig()
206 self._manage_builds = manage_builds or self._ManageBuilds()
Chris Sosa75490802013-09-30 17:21:45 -0700207 self._board = board
Gilad Arnoldd04fcab2015-02-19 12:00:45 -0800208 self._version = version
joychen921e1fb2013-06-28 11:12:20 -0700209 self._timestamp_folder = os.path.join(self.static_dir,
joychen3cb228e2013-06-12 12:13:13 -0700210 Timestamp.XBUDDY_TIMESTAMP_DIR)
Chris Sosa7cd23202013-10-15 17:22:57 -0700211 if images_dir:
212 self.images_dir = images_dir
213 else:
214 self.images_dir = os.path.join(self.GetSourceRoot(), 'src/build/images')
215
xixuan178263c2017-03-22 09:10:25 -0700216 if common_util.IsRunningOnMoblab():
217 self._ctx = gs.GSContext(cache_user='chronos') if gs else None
218 else:
219 self._ctx = gs.GSContext() if gs else None
xixuan44b55452016-09-06 15:35:56 -0700220
joychen7df67f72013-07-18 14:21:12 -0700221 common_util.MkDirP(self._timestamp_folder)
joychen3cb228e2013-06-12 12:13:13 -0700222
223 @classmethod
224 def ParseBoolean(cls, boolean_string):
225 """Evaluate a string to a boolean value"""
226 if boolean_string:
227 return boolean_string.lower() in cls._true_values
228 else:
229 return False
230
joychen562699a2013-08-13 15:22:14 -0700231 def _ReadConfig(self):
232 """Read xbuddy config from ini files.
233
234 Reads the base config from xbuddy_config.ini, and then merges in the
235 shadow config from shadow_xbuddy_config.ini
236
237 Returns:
238 The merged configuration.
Yu-Ju Hongc54658c2014-01-22 09:18:07 -0800239
joychen562699a2013-08-13 15:22:14 -0700240 Raises:
241 XBuddyException if the config file is missing.
242 """
243 xbuddy_config = ConfigParser.ConfigParser()
244 config_file = os.path.join(self.devserver_dir, CONFIG_FILE)
245 if os.path.exists(config_file):
246 xbuddy_config.read(config_file)
247 else:
Yiming Chend9202142014-11-07 14:56:52 -0800248 # Get the directory of xbuddy.py file.
249 file_dir = os.path.dirname(os.path.realpath(__file__))
250 # Read the default xbuddy_config.ini from the directory.
251 xbuddy_config.read(os.path.join(file_dir, CONFIG_FILE))
joychen562699a2013-08-13 15:22:14 -0700252
253 # Read the shadow file if there is one.
Chris Sosac2abc722013-08-26 17:11:22 -0700254 if os.path.isdir(CHROOT_SHADOW_DIR):
255 shadow_config_file = os.path.join(CHROOT_SHADOW_DIR, SHADOW_CONFIG_FILE)
256 else:
257 shadow_config_file = os.path.join(self.devserver_dir, SHADOW_CONFIG_FILE)
258
259 _Log('Using shadow config file stored at %s', shadow_config_file)
joychen562699a2013-08-13 15:22:14 -0700260 if os.path.exists(shadow_config_file):
261 shadow_xbuddy_config = ConfigParser.ConfigParser()
262 shadow_xbuddy_config.read(shadow_config_file)
263
264 # Merge shadow config in.
265 sections = shadow_xbuddy_config.sections()
266 for s in sections:
267 if not xbuddy_config.has_section(s):
268 xbuddy_config.add_section(s)
269 options = shadow_xbuddy_config.options(s)
270 for o in options:
271 val = shadow_xbuddy_config.get(s, o)
272 xbuddy_config.set(s, o, val)
273
274 return xbuddy_config
275
276 def _ManageBuilds(self):
277 """Checks if xBuddy is managing local builds using the current config."""
278 try:
279 return self.ParseBoolean(self.config.get(GENERAL, 'manage_builds'))
280 except ConfigParser.Error:
281 return False
282
283 def _Capacity(self):
284 """Gets the xbuddy capacity from the current config."""
285 try:
286 return int(self.config.get(GENERAL, 'capacity'))
287 except ConfigParser.Error:
288 return 5
289
Gilad Arnold38e828c2015-04-24 13:52:07 -0700290 def LookupAlias(self, alias, board=None, version=None):
joychen562699a2013-08-13 15:22:14 -0700291 """Given the full xbuddy config, look up an alias for path rewrite.
292
293 Args:
294 alias: The xbuddy path that could be one of the aliases in the
295 rewrite table.
296 board: The board to fill in with when paths are rewritten. Can be from
Gilad Arnold38e828c2015-04-24 13:52:07 -0700297 the update request xml or the default board from devserver. If None,
298 defers to the value given during XBuddy initialization.
Gilad Arnoldd04fcab2015-02-19 12:00:45 -0800299 version: The version to fill in when rewriting paths. Could be a specific
Gilad Arnold38e828c2015-04-24 13:52:07 -0700300 version number or a version alias like LATEST. If None, defers to the
301 value given during XBuddy initialization, or LATEST.
Yu-Ju Hongc54658c2014-01-22 09:18:07 -0800302
joychen562699a2013-08-13 15:22:14 -0700303 Returns:
Gilad Arnold896c6d82015-03-13 16:20:29 -0700304 A pair (val, suffix) where val is the rewritten path, or the original
305 string if no rewrite was found; and suffix is the assigned location
306 suffix, or the default suffix if none was found.
joychen562699a2013-08-13 15:22:14 -0700307 """
joychen562699a2013-08-13 15:22:14 -0700308 try:
Gilad Arnold896c6d82015-03-13 16:20:29 -0700309 suffix = self.config.get(LOCATION_SUFFIXES, alias)
310 except ConfigParser.Error:
311 suffix = RELEASE
312
313 try:
joychen562699a2013-08-13 15:22:14 -0700314 val = self.config.get(PATH_REWRITES, alias)
315 except ConfigParser.Error:
316 # No alias lookup found. Return original path.
Gilad Arnold896c6d82015-03-13 16:20:29 -0700317 val = None
joychen562699a2013-08-13 15:22:14 -0700318
Gilad Arnold896c6d82015-03-13 16:20:29 -0700319 if not (val and val.strip()):
320 val = alias
joychen562699a2013-08-13 15:22:14 -0700321 else:
Gilad Arnold896c6d82015-03-13 16:20:29 -0700322 # The found value is not an empty string.
Gilad Arnoldd04fcab2015-02-19 12:00:45 -0800323 # Fill in the board and version.
Gilad Arnold896c6d82015-03-13 16:20:29 -0700324 val = val.replace("BOARD", "%(board)s")
325 val = val.replace("VERSION", "%(version)s")
Gilad Arnold38e828c2015-04-24 13:52:07 -0700326 val = val % {'board': board or self._board,
327 'version': version or self._version or LATEST}
Gilad Arnold896c6d82015-03-13 16:20:29 -0700328
329 _Log("Path is %s, location suffix is %s", val, suffix)
330 return val, suffix
joychen562699a2013-08-13 15:22:14 -0700331
Simran Basi99e63c02014-05-20 10:39:52 -0700332 @staticmethod
333 def _ResolveImageDir(image_dir):
334 """Clean up and return the image dir to use.
335
336 Args:
337 image_dir: directory in Google Storage to use.
338
339 Returns:
340 |image_dir| if |image_dir| is not None. Otherwise, returns
341 devserver_constants.GS_IMAGE_DIR
342 """
343 image_dir = image_dir or devserver_constants.GS_IMAGE_DIR
344 # Remove trailing slashes.
345 return image_dir.rstrip('/')
346
Gilad Arnold896c6d82015-03-13 16:20:29 -0700347 def _LookupOfficial(self, board, suffix, image_dir=None):
joychenf8f07e22013-07-12 17:45:51 -0700348 """Check LATEST-master for the version number of interest."""
349 _Log("Checking gs for latest %s-%s image", board, suffix)
Simran Basi99e63c02014-05-20 10:39:52 -0700350 image_dir = XBuddy._ResolveImageDir(image_dir)
351 latest_addr = (devserver_constants.GS_LATEST_MASTER %
352 {'image_dir': image_dir,
353 'board': board,
354 'suffix': suffix})
joychen121fc9b2013-08-02 14:30:30 -0700355 # Full release + version is in the LATEST file.
xixuan44b55452016-09-06 15:35:56 -0700356 version = self._ctx.Cat(latest_addr)
joychen3cb228e2013-06-12 12:13:13 -0700357
joychenf8f07e22013-07-12 17:45:51 -0700358 return devserver_constants.IMAGE_DIR % {'board':board,
359 'suffix':suffix,
360 'version':version}
Gilad Arnold869e8ab2015-02-19 23:34:49 -0800361
xixuan44b55452016-09-06 15:35:56 -0700362 def _LS(self, path, list_subdirectory=False):
363 """Does a directory listing of the given gs path.
364
365 Args:
366 path: directory location on google storage to check.
367 list_subdirectory: whether to only list subdirectory for |path|.
368
369 Returns:
370 A list of paths that matched |path|.
371 """
372 if list_subdirectory:
373 return self._ctx.DoCommand(
Gwendal Grignou76d08792017-03-31 11:32:16 -0700374 ['ls', '-d', '--', path], redirect_stdout=True).output.splitlines()
xixuan44b55452016-09-06 15:35:56 -0700375 else:
376 return self._ctx.LS(path)
377
378 def _GetLatestVersionFromGsDir(self, path, list_subdirectory=False,
379 with_release=True):
380 """Returns most recent version number found in a google storage directory.
381
382 This lists out the contents of the given GS bucket or regex to GS buckets,
383 and tries to grab the newest version found in the directory names.
384
385 Args:
386 path: directory location on google storage to check.
387 list_subdirectory: whether to only list subdirectory for |path|.
388 with_release: whether versions include a release milestone (e.g. R12).
389
390 Returns:
391 The most recent version number found.
392 """
393 list_result = self._LS(path, list_subdirectory=list_subdirectory)
394 dir_names = [os.path.basename(p.rstrip('/')) for p in list_result]
395 try:
396 filter_re = re.compile(devserver_constants.VERSION_RE if with_release
397 else devserver_constants.VERSION)
398 versions = filter(filter_re.match, dir_names)
399 latest_version = max(versions, key=distutils.version.LooseVersion)
400 except ValueError:
401 raise gs.GSContextException(
402 'Failed to find most recent builds at %s' % path)
403
404 return latest_version
405
Gilad Arnold896c6d82015-03-13 16:20:29 -0700406 def _LookupChannel(self, board, suffix, channel='stable',
407 image_dir=None):
joychenf8f07e22013-07-12 17:45:51 -0700408 """Check the channel folder for the version number of interest."""
joychen121fc9b2013-08-02 14:30:30 -0700409 # Get all names in channel dir. Get 10 highest directories by version.
joychen7df67f72013-07-18 14:21:12 -0700410 _Log("Checking channel '%s' for latest '%s' image", channel, board)
Yu-Ju Hongc54658c2014-01-22 09:18:07 -0800411 # Due to historical reasons, gs://chromeos-releases uses
412 # daisy-spring as opposed to the board name daisy_spring. Convert
xixuan44b55452016-09-06 15:35:56 -0700413 # he board name for the lookup.
Yu-Ju Hongc54658c2014-01-22 09:18:07 -0800414 channel_dir = devserver_constants.GS_CHANNEL_DIR % {
415 'channel':channel,
416 'board':re.sub('_', '-', board)}
xixuan44b55452016-09-06 15:35:56 -0700417 latest_version = self._GetLatestVersionFromGsDir(channel_dir,
418 with_release=False)
joychenf8f07e22013-07-12 17:45:51 -0700419
joychen121fc9b2013-08-02 14:30:30 -0700420 # Figure out release number from the version number.
joychenc3944cb2013-08-19 10:42:07 -0700421 image_url = devserver_constants.IMAGE_DIR % {
Gilad Arnold896c6d82015-03-13 16:20:29 -0700422 'board': board,
423 'suffix': suffix,
424 'version': 'R*' + latest_version}
Simran Basi99e63c02014-05-20 10:39:52 -0700425 image_dir = XBuddy._ResolveImageDir(image_dir)
426 gs_url = os.path.join(image_dir, image_url)
joychenf8f07e22013-07-12 17:45:51 -0700427
428 # There should only be one match on cros-image-archive.
xixuan44b55452016-09-06 15:35:56 -0700429 full_version = self._GetLatestVersionFromGsDir(gs_url,
430 list_subdirectory=True)
joychenf8f07e22013-07-12 17:45:51 -0700431
Gilad Arnold896c6d82015-03-13 16:20:29 -0700432 return devserver_constants.IMAGE_DIR % {'board': board,
433 'suffix': suffix,
434 'version': full_version}
joychenf8f07e22013-07-12 17:45:51 -0700435
Gilad Arnold896c6d82015-03-13 16:20:29 -0700436 def _LookupVersion(self, board, suffix, version):
joychenf8f07e22013-07-12 17:45:51 -0700437 """Search GS image releases for the highest match to a version prefix."""
joychen121fc9b2013-08-02 14:30:30 -0700438 # Build the pattern for GS to match.
joychen7df67f72013-07-18 14:21:12 -0700439 _Log("Checking gs for latest '%s' image with prefix '%s'", board, version)
Gilad Arnold896c6d82015-03-13 16:20:29 -0700440 image_url = devserver_constants.IMAGE_DIR % {'board': board,
441 'suffix': suffix,
442 'version': version + '*'}
joychenf8f07e22013-07-12 17:45:51 -0700443 image_dir = os.path.join(devserver_constants.GS_IMAGE_DIR, image_url)
444
joychen121fc9b2013-08-02 14:30:30 -0700445 # Grab the newest version of the ones matched.
xixuan44b55452016-09-06 15:35:56 -0700446 full_version = self._GetLatestVersionFromGsDir(image_dir,
447 list_subdirectory=True)
Gilad Arnold896c6d82015-03-13 16:20:29 -0700448 return devserver_constants.IMAGE_DIR % {'board': board,
449 'suffix': suffix,
450 'version': full_version}
joychenf8f07e22013-07-12 17:45:51 -0700451
Gilad Arnold896c6d82015-03-13 16:20:29 -0700452 def _RemoteBuildId(self, board, suffix, version):
Chris Sosaea734d92013-10-11 11:28:58 -0700453 """Returns the remote build_id for the given board and version.
454
455 Raises:
456 XBuddyException: If we failed to resolve the version to a valid build_id.
457 """
Gilad Arnold896c6d82015-03-13 16:20:29 -0700458 build_id_as_is = devserver_constants.IMAGE_DIR % {'board': board,
459 'suffix': '',
460 'version': version}
461 build_id_suffix = devserver_constants.IMAGE_DIR % {'board': board,
462 'suffix': suffix,
463 'version': version}
Chris Sosaea734d92013-10-11 11:28:58 -0700464 # Return the first path that exists. We assume that what the user typed
465 # is better than with a default suffix added i.e. x86-generic/blah is
466 # more valuable than x86-generic-release/blah.
Gilad Arnold896c6d82015-03-13 16:20:29 -0700467 for build_id in build_id_as_is, build_id_suffix:
Chris Sosaea734d92013-10-11 11:28:58 -0700468 try:
xixuan44b55452016-09-06 15:35:56 -0700469 version = self._ctx.LS(
470 '%s/%s' % (devserver_constants.GS_IMAGE_DIR, build_id))
Chris Sosaea734d92013-10-11 11:28:58 -0700471 return build_id
xixuan44b55452016-09-06 15:35:56 -0700472 except (gs.GSCommandError, gs.GSContextException, gs.GSNoSuchKey):
Chris Sosaea734d92013-10-11 11:28:58 -0700473 continue
Gilad Arnold5f46d8e2015-02-19 12:17:55 -0800474
475 raise XBuddyException('Could not find remote build_id for %s %s' % (
476 board, version))
Chris Sosaea734d92013-10-11 11:28:58 -0700477
Gilad Arnold896c6d82015-03-13 16:20:29 -0700478 def _ResolveBuildVersion(self, board, suffix, base_version):
Gilad Arnold869e8ab2015-02-19 23:34:49 -0800479 """Check LATEST-<base_version> and returns a full build version."""
480 _Log('Checking gs for full version for %s of %s', base_version, board)
481 # TODO(garnold) We might want to accommodate version prefixes and pick the
482 # most recent found, as done in _LookupVersion().
483 latest_addr = (devserver_constants.GS_LATEST_BASE_VERSION %
484 {'image_dir': devserver_constants.GS_IMAGE_DIR,
485 'board': board,
Gilad Arnold896c6d82015-03-13 16:20:29 -0700486 'suffix': suffix,
Gilad Arnold869e8ab2015-02-19 23:34:49 -0800487 'base_version': base_version})
Gilad Arnold869e8ab2015-02-19 23:34:49 -0800488 # Full release + version is in the LATEST file.
xixuan44b55452016-09-06 15:35:56 -0700489 return self._ctx.Cat(latest_addr)
Gilad Arnold869e8ab2015-02-19 23:34:49 -0800490
Gilad Arnold896c6d82015-03-13 16:20:29 -0700491 def _ResolveVersionToBuildId(self, board, suffix, version, image_dir=None):
joychen121fc9b2013-08-02 14:30:30 -0700492 """Handle version aliases for remote payloads in GS.
joychen3cb228e2013-06-12 12:13:13 -0700493
494 Args:
495 board: as specified in the original call. (i.e. x86-generic, parrot)
Gilad Arnold896c6d82015-03-13 16:20:29 -0700496 suffix: The location suffix, to be added to board name.
joychen3cb228e2013-06-12 12:13:13 -0700497 version: as entered in the original call. can be
498 {TBD, 0. some custom alias as defined in a config file}
Ningning Xiab2a1af52016-04-22 11:14:42 -0700499 1. fully qualified build version.
Gilad Arnold869e8ab2015-02-19 23:34:49 -0800500 2. latest
501 3. latest-{channel}
502 4. latest-official-{board suffix}
503 5. version prefix (i.e. RX-Y.X, RX-Y, RX)
Simran Basi99e63c02014-05-20 10:39:52 -0700504 image_dir: image directory to check in Google Storage. If none,
505 the default bucket is used.
joychen3cb228e2013-06-12 12:13:13 -0700506
507 Returns:
Chris Sosaea734d92013-10-11 11:28:58 -0700508 Location where the image dir is actually found on GS (build_id)
joychen3cb228e2013-06-12 12:13:13 -0700509
Chris Sosaea734d92013-10-11 11:28:58 -0700510 Raises:
511 XBuddyException: If we failed to resolve the version to a valid url.
joychen3cb228e2013-06-12 12:13:13 -0700512 """
joychenf8f07e22013-07-12 17:45:51 -0700513 # Only the last segment of the alias is variable relative to the rest.
514 version_tuple = version.rsplit('-', 1)
joychen3cb228e2013-06-12 12:13:13 -0700515
joychenf8f07e22013-07-12 17:45:51 -0700516 if re.match(devserver_constants.VERSION_RE, version):
Gilad Arnold896c6d82015-03-13 16:20:29 -0700517 return self._RemoteBuildId(board, suffix, version)
Gilad Arnold869e8ab2015-02-19 23:34:49 -0800518 elif re.match(devserver_constants.VERSION, version):
Ningning Xiab2a1af52016-04-22 11:14:42 -0700519 raise XBuddyException('\'%s\' is not valid. Should provide the fully '
520 'qualified version with a version prefix \'RX-\' '
521 'due to crbug.com/585914' % version)
joychenf8f07e22013-07-12 17:45:51 -0700522 elif version == LATEST_OFFICIAL:
523 # latest-official --> LATEST build in board-release
Gilad Arnold896c6d82015-03-13 16:20:29 -0700524 return self._LookupOfficial(board, suffix, image_dir=image_dir)
joychenf8f07e22013-07-12 17:45:51 -0700525 elif version_tuple[0] == LATEST_OFFICIAL:
526 # latest-official-{suffix} --> LATEST build in board-{suffix}
Gilad Arnold896c6d82015-03-13 16:20:29 -0700527 return self._LookupOfficial(board, version_tuple[1],
528 image_dir=image_dir)
joychenf8f07e22013-07-12 17:45:51 -0700529 elif version == LATEST:
530 # latest --> latest build on stable channel
Gilad Arnold896c6d82015-03-13 16:20:29 -0700531 return self._LookupChannel(board, suffix, image_dir=image_dir)
joychenf8f07e22013-07-12 17:45:51 -0700532 elif version_tuple[0] == LATEST:
533 if re.match(devserver_constants.VERSION_RE, version_tuple[1]):
534 # latest-R* --> most recent qualifying build
Gilad Arnold896c6d82015-03-13 16:20:29 -0700535 return self._LookupVersion(board, suffix, version_tuple[1])
joychenf8f07e22013-07-12 17:45:51 -0700536 else:
537 # latest-{channel} --> latest build within that channel
Gilad Arnold896c6d82015-03-13 16:20:29 -0700538 return self._LookupChannel(board, suffix, channel=version_tuple[1],
Simran Basi99e63c02014-05-20 10:39:52 -0700539 image_dir=image_dir)
joychen3cb228e2013-06-12 12:13:13 -0700540 else:
541 # The given version doesn't match any known patterns.
joychen921e1fb2013-06-28 11:12:20 -0700542 raise XBuddyException("Version %s unknown. Can't find on GS." % version)
joychen3cb228e2013-06-12 12:13:13 -0700543
joychen5260b9a2013-07-16 14:48:01 -0700544 @staticmethod
545 def _Symlink(link, target):
546 """Symlinks link to target, and removes whatever link was there before."""
547 _Log("Linking to %s from %s", link, target)
548 if os.path.lexists(link):
549 os.unlink(link)
550 os.symlink(target, link)
551
joychen121fc9b2013-08-02 14:30:30 -0700552 def _GetLatestLocalVersion(self, board):
joychen921e1fb2013-06-28 11:12:20 -0700553 """Get the version of the latest image built for board by build_image
554
555 Updates the symlink reference within the xBuddy static dir to point to
556 the real image dir in the local /build/images directory.
557
558 Args:
joychenc3944cb2013-08-19 10:42:07 -0700559 board: board that image was built for.
joychen921e1fb2013-06-28 11:12:20 -0700560
561 Returns:
joychen121fc9b2013-08-02 14:30:30 -0700562 The discovered version of the image.
joychenc3944cb2013-08-19 10:42:07 -0700563
564 Raises:
565 XBuddyException if neither test nor dev image was found in latest built
566 directory.
joychen3cb228e2013-06-12 12:13:13 -0700567 """
joychen921e1fb2013-06-28 11:12:20 -0700568 latest_local_dir = self.GetLatestImageDir(board)
joychenb0dfe552013-07-30 10:02:06 -0700569 if not latest_local_dir or not os.path.exists(latest_local_dir):
joychen921e1fb2013-06-28 11:12:20 -0700570 raise XBuddyException('No builds found for %s. Did you run build_image?' %
571 board)
572
joychen121fc9b2013-08-02 14:30:30 -0700573 # Assume that the version number is the name of the directory.
joychenc3944cb2013-08-19 10:42:07 -0700574 return os.path.basename(latest_local_dir.rstrip('/'))
joychen921e1fb2013-06-28 11:12:20 -0700575
joychenc3944cb2013-08-19 10:42:07 -0700576 @staticmethod
577 def _FindAny(local_dir):
578 """Returns the image_type for ANY given the local_dir."""
joychenc3944cb2013-08-19 10:42:07 -0700579 test_image = os.path.join(local_dir, devserver_constants.TEST_IMAGE_FILE)
Yu-Ju Hongc23c79b2014-03-17 12:40:33 -0700580 dev_image = os.path.join(local_dir, devserver_constants.IMAGE_FILE)
581 # Prioritize test images over dev images.
joychenc3944cb2013-08-19 10:42:07 -0700582 if os.path.exists(test_image):
583 return 'test'
584
Yu-Ju Hongc23c79b2014-03-17 12:40:33 -0700585 if os.path.exists(dev_image):
586 return 'dev'
587
joychenc3944cb2013-08-19 10:42:07 -0700588 raise XBuddyException('No images found in %s' % local_dir)
589
590 @staticmethod
Gilad Arnoldd04fcab2015-02-19 12:00:45 -0800591 def _InterpretPath(path, default_board=None, default_version=None):
joychen121fc9b2013-08-02 14:30:30 -0700592 """Split and return the pieces of an xBuddy path name
joychen921e1fb2013-06-28 11:12:20 -0700593
joychen121fc9b2013-08-02 14:30:30 -0700594 Args:
595 path: the path xBuddy Get was called with.
Chris Sosa0eecf962014-02-03 14:14:39 -0800596 default_board: board to use in case board isn't in path.
Gilad Arnoldd04fcab2015-02-19 12:00:45 -0800597 default_version: Version to use in case version isn't in path.
joychen3cb228e2013-06-12 12:13:13 -0700598
Yu-Ju Hongc54658c2014-01-22 09:18:07 -0800599 Returns:
Chris Sosa75490802013-09-30 17:21:45 -0700600 tuple of (image_type, board, version, whether the path is local)
joychen3cb228e2013-06-12 12:13:13 -0700601
602 Raises:
603 XBuddyException: if the path can't be resolved into valid components
604 """
joychen121fc9b2013-08-02 14:30:30 -0700605 path_list = filter(None, path.split('/'))
joychen7df67f72013-07-18 14:21:12 -0700606
Chris Sosa0eecf962014-02-03 14:14:39 -0800607 # Do the stuff that is well known first. We know that if paths have a
608 # image_type, it must be one of the GS/LOCAL aliases and it must be at the
609 # end. Similarly, local/remote are well-known and must start the path list.
610 is_local = True
611 if path_list and path_list[0] in (REMOTE, LOCAL):
612 is_local = (path_list.pop(0) == LOCAL)
joychen7df67f72013-07-18 14:21:12 -0700613
Chris Sosa0eecf962014-02-03 14:14:39 -0800614 # Default image type is determined by remote vs. local.
615 if is_local:
616 image_type = ANY
617 else:
618 image_type = TEST
joychen7df67f72013-07-18 14:21:12 -0700619
Chris Sosa0eecf962014-02-03 14:14:39 -0800620 if path_list and path_list[-1] in GS_ALIASES + LOCAL_ALIASES:
621 image_type = path_list.pop(-1)
joychen3cb228e2013-06-12 12:13:13 -0700622
Chris Sosa0eecf962014-02-03 14:14:39 -0800623 # Now for the tricky part. We don't actually know at this point if the rest
624 # of the path is just a board | version (like R33-2341.0.0) or just a board
625 # or just a version. So we do our best to do the right thing.
626 board = default_board
Gilad Arnoldd04fcab2015-02-19 12:00:45 -0800627 version = default_version or LATEST
Chris Sosa0eecf962014-02-03 14:14:39 -0800628 if len(path_list) == 1:
629 path = path_list.pop(0)
Gilad Arnoldd04fcab2015-02-19 12:00:45 -0800630 # Treat this as a version if it's one we know (contains default or
631 # latest), or we were given an actual default board.
632 if default_version in path or LATEST in path or default_board is not None:
Chris Sosa0eecf962014-02-03 14:14:39 -0800633 version = path
joychen7df67f72013-07-18 14:21:12 -0700634 else:
Chris Sosa0eecf962014-02-03 14:14:39 -0800635 board = path
joychen7df67f72013-07-18 14:21:12 -0700636
Chris Sosa0eecf962014-02-03 14:14:39 -0800637 elif len(path_list) == 2:
638 # Assumes board/version.
639 board = path_list.pop(0)
640 version = path_list.pop(0)
641
642 if path_list:
643 raise XBuddyException("Path isn't valid. Could not figure out how to "
644 "parse remaining components: %s." % path_list)
645
646 _Log("Get artifact '%s' with board %s and version %s'. Locally? %s",
joychen7df67f72013-07-18 14:21:12 -0700647 image_type, board, version, is_local)
648
649 return image_type, board, version, is_local
joychen3cb228e2013-06-12 12:13:13 -0700650
joychen921e1fb2013-06-28 11:12:20 -0700651 def _SyncRegistryWithBuildImages(self):
Gilad Arnold5f46d8e2015-02-19 12:17:55 -0800652 """Crawl images_dir for build_ids of images generated from build_image.
joychen5260b9a2013-07-16 14:48:01 -0700653
654 This will find images and symlink them in xBuddy's static dir so that
655 xBuddy's cache can serve them.
656 If xBuddy's _manage_builds option is on, then a timestamp will also be
657 generated, and xBuddy will clear them from the directory they are in, as
658 necessary.
659 """
Yu-Ju Hong235d1b52014-04-16 11:01:47 -0700660 if not os.path.isdir(self.images_dir):
661 # Skip syncing if images_dir does not exist.
662 _Log('Cannot find %s; skip syncing image registry.', self.images_dir)
663 return
664
joychen921e1fb2013-06-28 11:12:20 -0700665 build_ids = []
666 for b in os.listdir(self.images_dir):
Mike Frysingera2c24252017-11-28 19:14:45 -0500667 # Ignore random files in the build dir.
668 board_dir = os.path.join(self.images_dir, b)
669 if not os.path.isdir(board_dir):
670 continue
671
joychen5260b9a2013-07-16 14:48:01 -0700672 # Ensure we have directories to track all boards in build/images
673 common_util.MkDirP(os.path.join(self.static_dir, b))
joychen921e1fb2013-06-28 11:12:20 -0700674 build_ids.extend(['/'.join([b, v]) for v
joychenc3944cb2013-08-19 10:42:07 -0700675 in os.listdir(board_dir) if not v == LATEST])
joychen921e1fb2013-06-28 11:12:20 -0700676
joychen121fc9b2013-08-02 14:30:30 -0700677 # Symlink undiscovered images, and update timestamps if manage_builds is on.
joychen5260b9a2013-07-16 14:48:01 -0700678 for build_id in build_ids:
679 link = os.path.join(self.static_dir, build_id)
680 target = os.path.join(self.images_dir, build_id)
681 XBuddy._Symlink(link, target)
682 if self._manage_builds:
683 Timestamp.UpdateTimestamp(self._timestamp_folder, build_id)
joychen921e1fb2013-06-28 11:12:20 -0700684
685 def _ListBuildTimes(self):
Gilad Arnold5f46d8e2015-02-19 12:17:55 -0800686 """Returns the currently cached builds and their last access timestamp.
joychen3cb228e2013-06-12 12:13:13 -0700687
688 Returns:
689 list of tuples that matches xBuddy build/version to timestamps in long
690 """
joychen121fc9b2013-08-02 14:30:30 -0700691 # Update currently cached builds.
joychen3cb228e2013-06-12 12:13:13 -0700692 build_dict = {}
693
joychen7df67f72013-07-18 14:21:12 -0700694 for f in os.listdir(self._timestamp_folder):
joychen3cb228e2013-06-12 12:13:13 -0700695 last_accessed = os.path.getmtime(os.path.join(self._timestamp_folder, f))
696 build_id = Timestamp.TimestampToBuild(f)
joychenc3944cb2013-08-19 10:42:07 -0700697 stale_time = datetime.timedelta(seconds=(time.time() - last_accessed))
joychen921e1fb2013-06-28 11:12:20 -0700698 build_dict[build_id] = stale_time
joychen3cb228e2013-06-12 12:13:13 -0700699 return_tup = sorted(build_dict.iteritems(), key=operator.itemgetter(1))
700 return return_tup
701
Chris Sosa75490802013-09-30 17:21:45 -0700702 def _Download(self, gs_url, artifacts):
703 """Download the artifacts from the given gs_url.
704
705 Raises:
706 build_artifact.ArtifactDownloadError: If we failed to download the
707 artifact.
708 """
joychen3cb228e2013-06-12 12:13:13 -0700709 with XBuddy._staging_thread_count_lock:
710 XBuddy._staging_thread_count += 1
711 try:
Chris Sosa75490802013-09-30 17:21:45 -0700712 _Log("Downloading %s from %s", artifacts, gs_url)
Gabe Black3b567202015-09-23 14:07:59 -0700713 dl = downloader.GoogleStorageDownloader(self.static_dir, gs_url)
714 factory = build_artifact.ChromeOSArtifactFactory(
715 dl.GetBuildDir(), artifacts, [], dl.GetBuild())
716 dl.Download(factory)
joychen3cb228e2013-06-12 12:13:13 -0700717 finally:
718 with XBuddy._staging_thread_count_lock:
719 XBuddy._staging_thread_count -= 1
720
Chris Sosa75490802013-09-30 17:21:45 -0700721 def CleanCache(self):
joychen562699a2013-08-13 15:22:14 -0700722 """Delete all builds besides the newest N builds"""
joychen121fc9b2013-08-02 14:30:30 -0700723 if not self._manage_builds:
724 return
joychen921e1fb2013-06-28 11:12:20 -0700725 cached_builds = [e[0] for e in self._ListBuildTimes()]
joychen3cb228e2013-06-12 12:13:13 -0700726 _Log('In cache now: %s', cached_builds)
727
joychen562699a2013-08-13 15:22:14 -0700728 for b in range(self._Capacity(), len(cached_builds)):
joychen3cb228e2013-06-12 12:13:13 -0700729 b_path = cached_builds[b]
joychen7df67f72013-07-18 14:21:12 -0700730 _Log("Clearing '%s' from cache", b_path)
joychen3cb228e2013-06-12 12:13:13 -0700731
732 time_file = os.path.join(self._timestamp_folder,
733 Timestamp.BuildToTimestamp(b_path))
joychen921e1fb2013-06-28 11:12:20 -0700734 os.unlink(time_file)
735 clear_dir = os.path.join(self.static_dir, b_path)
joychen3cb228e2013-06-12 12:13:13 -0700736 try:
joychen121fc9b2013-08-02 14:30:30 -0700737 # Handle symlinks, in the case of links to local builds if enabled.
738 if os.path.islink(clear_dir):
joychen5260b9a2013-07-16 14:48:01 -0700739 target = os.readlink(clear_dir)
740 _Log('Deleting locally built image at %s', target)
joychen921e1fb2013-06-28 11:12:20 -0700741
742 os.unlink(clear_dir)
joychen5260b9a2013-07-16 14:48:01 -0700743 if os.path.exists(target):
joychen921e1fb2013-06-28 11:12:20 -0700744 shutil.rmtree(target)
745 elif os.path.exists(clear_dir):
joychen5260b9a2013-07-16 14:48:01 -0700746 _Log('Deleting downloaded image at %s', clear_dir)
joychen3cb228e2013-06-12 12:13:13 -0700747 shutil.rmtree(clear_dir)
joychen921e1fb2013-06-28 11:12:20 -0700748
joychen121fc9b2013-08-02 14:30:30 -0700749 except Exception as err:
750 raise XBuddyException('Failed to clear %s: %s' % (clear_dir, err))
joychen3cb228e2013-06-12 12:13:13 -0700751
Simran Basi99e63c02014-05-20 10:39:52 -0700752 def _GetFromGS(self, build_id, image_type, image_dir=None):
Chris Sosa75490802013-09-30 17:21:45 -0700753 """Check if the artifact is available locally. Download from GS if not.
754
Simran Basi99e63c02014-05-20 10:39:52 -0700755 Args:
756 build_id: Path to the image or update directory on the devserver or
757 in Google Storage. e.g. 'x86-generic/R26-4000.0.0'
758 image_type: Image type to download. Look at aliases at top of file for
759 options.
760 image_dir: Google Storage image archive to search in if requesting a
761 remote artifact. If none uses the default bucket.
762
Chris Sosa75490802013-09-30 17:21:45 -0700763 Raises:
764 build_artifact.ArtifactDownloadError: If we failed to download the
765 artifact.
766 """
Simran Basi99e63c02014-05-20 10:39:52 -0700767 image_dir = XBuddy._ResolveImageDir(image_dir)
768 gs_url = os.path.join(image_dir, build_id)
joychen921e1fb2013-06-28 11:12:20 -0700769
joychen121fc9b2013-08-02 14:30:30 -0700770 # Stage image if not found in cache.
joychen921e1fb2013-06-28 11:12:20 -0700771 file_name = GS_ALIAS_TO_FILENAME[image_type]
joychen346531c2013-07-24 16:55:56 -0700772 file_loc = os.path.join(self.static_dir, build_id, file_name)
773 cached = os.path.exists(file_loc)
774
joychen921e1fb2013-06-28 11:12:20 -0700775 if not cached:
Chris Sosa75490802013-09-30 17:21:45 -0700776 artifact = GS_ALIAS_TO_ARTIFACT[image_type]
777 self._Download(gs_url, [artifact])
joychen921e1fb2013-06-28 11:12:20 -0700778 else:
779 _Log('Image already cached.')
780
Gilad Arnoldd04fcab2015-02-19 12:00:45 -0800781 def _GetArtifact(self, path_list, board=None, version=None,
782 lookup_only=False, image_dir=None):
joychen346531c2013-07-24 16:55:56 -0700783 """Interpret an xBuddy path and return directory/file_name to resource.
784
Chris Sosa75490802013-09-30 17:21:45 -0700785 Note board can be passed that in but by default if self._board is set,
786 that is used rather than board.
787
Simran Basi99e63c02014-05-20 10:39:52 -0700788 Args:
789 path_list: [board, version, alias] as split from the xbuddy call url.
Gilad Arnoldd04fcab2015-02-19 12:00:45 -0800790 board: Board whos artifacts we are looking for. Only used if no board was
791 given during XBuddy initialization.
792 version: Version whose artifacts we are looking for. Used if no version
793 was given during XBuddy initialization. If None, defers to LATEST.
Simran Basi99e63c02014-05-20 10:39:52 -0700794 lookup_only: If true just look up the artifact, if False stage it on
795 the devserver as well.
796 image_dir: Google Storage image archive to search in if requesting a
797 remote artifact. If none uses the default bucket.
798
joychen346531c2013-07-24 16:55:56 -0700799 Returns:
Simran Basi99e63c02014-05-20 10:39:52 -0700800 build_id: Path to the image or update directory on the devserver or
801 in Google Storage. e.g. 'x86-generic/R26-4000.0.0'
802 file_name: of the artifact in the build_id directory.
joychen346531c2013-07-24 16:55:56 -0700803
804 Raises:
joychen121fc9b2013-08-02 14:30:30 -0700805 XBuddyException: if the path could not be translated
Chris Sosa75490802013-09-30 17:21:45 -0700806 build_artifact.ArtifactDownloadError: if we failed to download the
807 artifact.
joychen346531c2013-07-24 16:55:56 -0700808 """
joychen121fc9b2013-08-02 14:30:30 -0700809 path = '/'.join(path_list)
Chris Sosa0eecf962014-02-03 14:14:39 -0800810 default_board = self._board if self._board else board
Gilad Arnoldd04fcab2015-02-19 12:00:45 -0800811 default_version = self._version or version or LATEST
joychenb0dfe552013-07-30 10:02:06 -0700812 # Rewrite the path if there is an appropriate default.
Gilad Arnold38e828c2015-04-24 13:52:07 -0700813 path, suffix = self.LookupAlias(path, board=default_board,
814 version=default_version)
joychen121fc9b2013-08-02 14:30:30 -0700815 # Parse the path.
Chris Sosa0eecf962014-02-03 14:14:39 -0800816 image_type, board, version, is_local = self._InterpretPath(
Gilad Arnoldd04fcab2015-02-19 12:00:45 -0800817 path, default_board, default_version)
joychen7df67f72013-07-18 14:21:12 -0700818 if is_local:
joychen121fc9b2013-08-02 14:30:30 -0700819 # Get a local image.
joychen7df67f72013-07-18 14:21:12 -0700820 if version == LATEST:
joychen121fc9b2013-08-02 14:30:30 -0700821 # Get the latest local image for the given board.
822 version = self._GetLatestLocalVersion(board)
joychen7df67f72013-07-18 14:21:12 -0700823
joychenc3944cb2013-08-19 10:42:07 -0700824 build_id = os.path.join(board, version)
825 artifact_dir = os.path.join(self.static_dir, build_id)
826 if image_type == ANY:
827 image_type = self._FindAny(artifact_dir)
joychen121fc9b2013-08-02 14:30:30 -0700828
joychenc3944cb2013-08-19 10:42:07 -0700829 file_name = LOCAL_ALIAS_TO_FILENAME[image_type]
830 artifact_path = os.path.join(artifact_dir, file_name)
831 if not os.path.exists(artifact_path):
832 raise XBuddyException('Local %s artifact not in static_dir at %s' %
833 (image_type, artifact_path))
joychen121fc9b2013-08-02 14:30:30 -0700834
joychen921e1fb2013-06-28 11:12:20 -0700835 else:
joychen121fc9b2013-08-02 14:30:30 -0700836 # Get a remote image.
joychen921e1fb2013-06-28 11:12:20 -0700837 if image_type not in GS_ALIASES:
joychen7df67f72013-07-18 14:21:12 -0700838 raise XBuddyException('Bad remote image type: %s. Use one of: %s' %
joychen921e1fb2013-06-28 11:12:20 -0700839 (image_type, GS_ALIASES))
Gilad Arnold896c6d82015-03-13 16:20:29 -0700840 build_id = self._ResolveVersionToBuildId(board, suffix, version,
Simran Basi99e63c02014-05-20 10:39:52 -0700841 image_dir=image_dir)
Chris Sosa75490802013-09-30 17:21:45 -0700842 _Log('Resolved version %s to %s.', version, build_id)
843 file_name = GS_ALIAS_TO_FILENAME[image_type]
844 if not lookup_only:
Simran Basi99e63c02014-05-20 10:39:52 -0700845 self._GetFromGS(build_id, image_type, image_dir=image_dir)
joychenf8f07e22013-07-12 17:45:51 -0700846
joychenc3944cb2013-08-19 10:42:07 -0700847 return build_id, file_name
joychen3cb228e2013-06-12 12:13:13 -0700848
849 ############################ BEGIN PUBLIC METHODS
850
851 def List(self):
852 """Lists the currently available images & time since last access."""
joychen921e1fb2013-06-28 11:12:20 -0700853 self._SyncRegistryWithBuildImages()
854 builds = self._ListBuildTimes()
855 return_string = ''
856 for build, timestamp in builds:
857 return_string += '<b>' + build + '</b> '
858 return_string += '(time since last access: ' + str(timestamp) + ')<br>'
859 return return_string
joychen3cb228e2013-06-12 12:13:13 -0700860
861 def Capacity(self):
862 """Returns the number of images cached by xBuddy."""
joychen562699a2013-08-13 15:22:14 -0700863 return str(self._Capacity())
joychen3cb228e2013-06-12 12:13:13 -0700864
Gilad Arnoldd04fcab2015-02-19 12:00:45 -0800865 def Translate(self, path_list, board=None, version=None, image_dir=None):
joychen346531c2013-07-24 16:55:56 -0700866 """Translates an xBuddy path to a real path to artifact if it exists.
867
joychen121fc9b2013-08-02 14:30:30 -0700868 Equivalent to the Get call, minus downloading and updating timestamps,
joychen346531c2013-07-24 16:55:56 -0700869
Simran Basi99e63c02014-05-20 10:39:52 -0700870 Args:
871 path_list: [board, version, alias] as split from the xbuddy call url.
872 board: Board whos artifacts we are looking for. If None, use the board
873 XBuddy was initialized to use.
Gilad Arnoldd04fcab2015-02-19 12:00:45 -0800874 version: Version whose artifacts we are looking for. If None, use the
875 version XBuddy was initialized with, or LATEST.
Simran Basi99e63c02014-05-20 10:39:52 -0700876 image_dir: image directory to check in Google Storage. If none,
877 the default bucket is used.
878
joychen7c2054a2013-07-25 11:14:07 -0700879 Returns:
joychen121fc9b2013-08-02 14:30:30 -0700880 build_id: Path to the image or update directory on the devserver.
881 e.g. 'x86-generic/R26-4000.0.0'
882 The returned path is always the path to the directory within
883 static_dir, so it is always the build_id of the image.
884 file_name: The file name of the artifact. Can take any of the file
885 values in devserver_constants.
886 e.g. 'chromiumos_test_image.bin' or 'update.gz' if the path list
887 specified 'test' or 'full_payload' artifacts, respectively.
joychen7c2054a2013-07-25 11:14:07 -0700888
joychen121fc9b2013-08-02 14:30:30 -0700889 Raises:
890 XBuddyException: if the path couldn't be translated
joychen346531c2013-07-24 16:55:56 -0700891 """
892 self._SyncRegistryWithBuildImages()
Chris Sosa75490802013-09-30 17:21:45 -0700893 build_id, file_name = self._GetArtifact(path_list, board=board,
Gilad Arnoldd04fcab2015-02-19 12:00:45 -0800894 version=version,
Simran Basi99e63c02014-05-20 10:39:52 -0700895 lookup_only=True,
896 image_dir=image_dir)
joychen346531c2013-07-24 16:55:56 -0700897
joychen121fc9b2013-08-02 14:30:30 -0700898 _Log('Returning path to payload: %s/%s', build_id, file_name)
899 return build_id, file_name
joychen346531c2013-07-24 16:55:56 -0700900
Yu-Ju Hong1bdb7a92014-04-10 16:02:11 -0700901 def StageTestArtifactsForUpdate(self, path_list):
Chris Sosa75490802013-09-30 17:21:45 -0700902 """Stages test artifacts for update and returns build_id.
903
904 Raises:
905 XBuddyException: if the path could not be translated
906 build_artifact.ArtifactDownloadError: if we failed to download the test
907 artifacts.
908 """
909 build_id, file_name = self.Translate(path_list)
910 if file_name == devserver_constants.TEST_IMAGE_FILE:
911 gs_url = os.path.join(devserver_constants.GS_IMAGE_DIR,
912 build_id)
913 artifacts = [FULL, STATEFUL]
914 self._Download(gs_url, artifacts)
915 return build_id
916
Simran Basi99e63c02014-05-20 10:39:52 -0700917 def Get(self, path_list, image_dir=None):
joychen921e1fb2013-06-28 11:12:20 -0700918 """The full xBuddy call, returns resource specified by path_list.
joychen3cb228e2013-06-12 12:13:13 -0700919
920 Please see devserver.py:xbuddy for full documentation.
joychen121fc9b2013-08-02 14:30:30 -0700921
joychen3cb228e2013-06-12 12:13:13 -0700922 Args:
Simran Basi99e63c02014-05-20 10:39:52 -0700923 path_list: [board, version, alias] as split from the xbuddy call url.
924 image_dir: image directory to check in Google Storage. If none,
925 the default bucket is used.
joychen3cb228e2013-06-12 12:13:13 -0700926
927 Returns:
joychen121fc9b2013-08-02 14:30:30 -0700928 build_id: Path to the image or update directory on the devserver.
Simran Basi99e63c02014-05-20 10:39:52 -0700929 e.g. 'x86-generic/R26-4000.0.0'
930 The returned path is always the path to the directory within
931 static_dir, so it is always the build_id of the image.
joychen121fc9b2013-08-02 14:30:30 -0700932 file_name: The file name of the artifact. Can take any of the file
Simran Basi99e63c02014-05-20 10:39:52 -0700933 values in devserver_constants.
934 e.g. 'chromiumos_test_image.bin' or 'update.gz' if the path list
935 specified 'test' or 'full_payload' artifacts, respectively.
joychen3cb228e2013-06-12 12:13:13 -0700936
937 Raises:
Chris Sosa75490802013-09-30 17:21:45 -0700938 XBuddyException: if the path could not be translated
939 build_artifact.ArtifactDownloadError: if we failed to download the
940 artifact.
joychen3cb228e2013-06-12 12:13:13 -0700941 """
joychen7df67f72013-07-18 14:21:12 -0700942 self._SyncRegistryWithBuildImages()
Simran Basi99e63c02014-05-20 10:39:52 -0700943 build_id, file_name = self._GetArtifact(path_list, image_dir=image_dir)
joychen921e1fb2013-06-28 11:12:20 -0700944 Timestamp.UpdateTimestamp(self._timestamp_folder, build_id)
joychen3cb228e2013-06-12 12:13:13 -0700945 #TODO (joyc): run in sep thread
Chris Sosa75490802013-09-30 17:21:45 -0700946 self.CleanCache()
joychen3cb228e2013-06-12 12:13:13 -0700947
joychen121fc9b2013-08-02 14:30:30 -0700948 _Log('Returning path to payload: %s/%s', build_id, file_name)
949 return build_id, file_name