blob: 2659002ca42a3b69f84ac52cb16026d1928ceae1 [file] [log] [blame]
joychen3cb228e2013-06-12 12:13:13 -07001# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
joychen562699a2013-08-13 15:22:14 -07005import ConfigParser
joychen3cb228e2013-06-12 12:13:13 -07006import datetime
7import operator
8import os
joychenf8f07e22013-07-12 17:45:51 -07009import re
joychen3cb228e2013-06-12 12:13:13 -070010import shutil
joychenf8f07e22013-07-12 17:45:51 -070011import time
joychen3cb228e2013-06-12 12:13:13 -070012import threading
13
joychen921e1fb2013-06-28 11:12:20 -070014import build_util
joychen3cb228e2013-06-12 12:13:13 -070015import artifact_info
joychen3cb228e2013-06-12 12:13:13 -070016import common_util
17import devserver_constants
18import downloader
joychenf8f07e22013-07-12 17:45:51 -070019import gsutil_util
joychen3cb228e2013-06-12 12:13:13 -070020import log_util
21
22# Module-local log function.
23def _Log(message, *args):
24 return log_util.LogWithTag('XBUDDY', message, *args)
25
joychen562699a2013-08-13 15:22:14 -070026# xBuddy config constants
27CONFIG_FILE = 'xbuddy_config.ini'
28SHADOW_CONFIG_FILE = 'shadow_xbuddy_config.ini'
29PATH_REWRITES = 'PATH_REWRITES'
30GENERAL = 'GENERAL'
joychen921e1fb2013-06-28 11:12:20 -070031
Chris Sosac2abc722013-08-26 17:11:22 -070032# Path for shadow config in chroot.
33CHROOT_SHADOW_DIR = '/mnt/host/source/src/platform/dev'
34
joychen25d25972013-07-30 14:54:16 -070035# XBuddy aliases
36TEST = 'test'
37BASE = 'base'
38DEV = 'dev'
39FULL = 'full_payload'
40RECOVERY = 'recovery'
41STATEFUL = 'stateful'
42AUTOTEST = 'autotest'
43
joychen921e1fb2013-06-28 11:12:20 -070044# Local build constants
joychenc3944cb2013-08-19 10:42:07 -070045ANY = "ANY"
joychen7df67f72013-07-18 14:21:12 -070046LATEST = "latest"
47LOCAL = "local"
48REMOTE = "remote"
joychen921e1fb2013-06-28 11:12:20 -070049LOCAL_ALIASES = [
joychen25d25972013-07-30 14:54:16 -070050 TEST,
51 BASE,
52 DEV,
joychenc3944cb2013-08-19 10:42:07 -070053 FULL,
54 ANY,
joychen921e1fb2013-06-28 11:12:20 -070055]
56
57LOCAL_FILE_NAMES = [
58 devserver_constants.TEST_IMAGE_FILE,
59 devserver_constants.BASE_IMAGE_FILE,
60 devserver_constants.IMAGE_FILE,
joychen7c2054a2013-07-25 11:14:07 -070061 devserver_constants.UPDATE_FILE,
joychen921e1fb2013-06-28 11:12:20 -070062]
63
64LOCAL_ALIAS_TO_FILENAME = dict(zip(LOCAL_ALIASES, LOCAL_FILE_NAMES))
65
66# Google Storage constants
67GS_ALIASES = [
joychen25d25972013-07-30 14:54:16 -070068 TEST,
69 BASE,
70 RECOVERY,
71 FULL,
72 STATEFUL,
73 AUTOTEST,
joychen3cb228e2013-06-12 12:13:13 -070074]
75
joychen921e1fb2013-06-28 11:12:20 -070076GS_FILE_NAMES = [
77 devserver_constants.TEST_IMAGE_FILE,
78 devserver_constants.BASE_IMAGE_FILE,
79 devserver_constants.RECOVERY_IMAGE_FILE,
joychen7c2054a2013-07-25 11:14:07 -070080 devserver_constants.UPDATE_FILE,
joychen121fc9b2013-08-02 14:30:30 -070081 devserver_constants.STATEFUL_FILE,
joychen3cb228e2013-06-12 12:13:13 -070082 devserver_constants.AUTOTEST_DIR,
83]
84
85ARTIFACTS = [
86 artifact_info.TEST_IMAGE,
87 artifact_info.BASE_IMAGE,
88 artifact_info.RECOVERY_IMAGE,
89 artifact_info.FULL_PAYLOAD,
90 artifact_info.STATEFUL_PAYLOAD,
91 artifact_info.AUTOTEST,
92]
93
joychen921e1fb2013-06-28 11:12:20 -070094GS_ALIAS_TO_FILENAME = dict(zip(GS_ALIASES, GS_FILE_NAMES))
95GS_ALIAS_TO_ARTIFACT = dict(zip(GS_ALIASES, ARTIFACTS))
joychen3cb228e2013-06-12 12:13:13 -070096
joychen921e1fb2013-06-28 11:12:20 -070097LATEST_OFFICIAL = "latest-official"
joychen3cb228e2013-06-12 12:13:13 -070098
joychenf8f07e22013-07-12 17:45:51 -070099RELEASE = "release"
joychen3cb228e2013-06-12 12:13:13 -0700100
joychen3cb228e2013-06-12 12:13:13 -0700101
102class XBuddyException(Exception):
103 """Exception classes used by this module."""
104 pass
105
106
107# no __init__ method
108#pylint: disable=W0232
109class Timestamp():
110 """Class to translate build path strings and timestamp filenames."""
111
112 _TIMESTAMP_DELIMITER = 'SLASH'
113 XBUDDY_TIMESTAMP_DIR = 'xbuddy_UpdateTimestamps'
114
115 @staticmethod
116 def TimestampToBuild(timestamp_filename):
117 return timestamp_filename.replace(Timestamp._TIMESTAMP_DELIMITER, '/')
118
119 @staticmethod
120 def BuildToTimestamp(build_path):
121 return build_path.replace('/', Timestamp._TIMESTAMP_DELIMITER)
joychen921e1fb2013-06-28 11:12:20 -0700122
123 @staticmethod
124 def UpdateTimestamp(timestamp_dir, build_id):
125 """Update timestamp file of build with build_id."""
126 common_util.MkDirP(timestamp_dir)
joychen562699a2013-08-13 15:22:14 -0700127 _Log("Updating timestamp for %s", build_id)
joychen921e1fb2013-06-28 11:12:20 -0700128 time_file = os.path.join(timestamp_dir,
129 Timestamp.BuildToTimestamp(build_id))
130 with file(time_file, 'a'):
131 os.utime(time_file, None)
joychen3cb228e2013-06-12 12:13:13 -0700132#pylint: enable=W0232
133
134
joychen921e1fb2013-06-28 11:12:20 -0700135class XBuddy(build_util.BuildObject):
joychen3cb228e2013-06-12 12:13:13 -0700136 """Class that manages image retrieval and caching by the devserver.
137
138 Image retrieval by xBuddy path:
139 XBuddy accesses images and artifacts that it stores using an xBuddy
140 path of the form: board/version/alias
141 The primary xbuddy.Get call retrieves the correct artifact or url to where
142 the artifacts can be found.
143
144 Image caching:
145 Images and other artifacts are stored identically to how they would have
146 been if devserver's stage rpc was called and the xBuddy cache replaces
147 build versions on a LRU basis. Timestamps are maintained by last accessed
148 times of representative files in the a directory in the static serve
149 directory (XBUDDY_TIMESTAMP_DIR).
150
151 Private class members:
joychen121fc9b2013-08-02 14:30:30 -0700152 _true_values: used for interpreting boolean values
153 _staging_thread_count: track download requests
154 _timestamp_folder: directory with empty files standing in as timestamps
joychen921e1fb2013-06-28 11:12:20 -0700155 for each image currently cached by xBuddy
joychen3cb228e2013-06-12 12:13:13 -0700156 """
157 _true_values = ['true', 't', 'yes', 'y']
158
159 # Number of threads that are staging images.
160 _staging_thread_count = 0
161 # Lock used to lock increasing/decreasing count.
162 _staging_thread_count_lock = threading.Lock()
163
joychenb0dfe552013-07-30 10:02:06 -0700164 def __init__(self, manage_builds=False, board=None, **kwargs):
joychen921e1fb2013-06-28 11:12:20 -0700165 super(XBuddy, self).__init__(**kwargs)
joychenb0dfe552013-07-30 10:02:06 -0700166
joychen562699a2013-08-13 15:22:14 -0700167 self.config = self._ReadConfig()
168 self._manage_builds = manage_builds or self._ManageBuilds()
169 self._board = board or self.GetDefaultBoardID()
joychenb0dfe552013-07-30 10:02:06 -0700170 _Log("Default board used by xBuddy: %s", self._board)
joychenb0dfe552013-07-30 10:02:06 -0700171
joychen921e1fb2013-06-28 11:12:20 -0700172 self._timestamp_folder = os.path.join(self.static_dir,
joychen3cb228e2013-06-12 12:13:13 -0700173 Timestamp.XBUDDY_TIMESTAMP_DIR)
joychen7df67f72013-07-18 14:21:12 -0700174 common_util.MkDirP(self._timestamp_folder)
joychen3cb228e2013-06-12 12:13:13 -0700175
176 @classmethod
177 def ParseBoolean(cls, boolean_string):
178 """Evaluate a string to a boolean value"""
179 if boolean_string:
180 return boolean_string.lower() in cls._true_values
181 else:
182 return False
183
joychen562699a2013-08-13 15:22:14 -0700184 def _ReadConfig(self):
185 """Read xbuddy config from ini files.
186
187 Reads the base config from xbuddy_config.ini, and then merges in the
188 shadow config from shadow_xbuddy_config.ini
189
190 Returns:
191 The merged configuration.
192 Raises:
193 XBuddyException if the config file is missing.
194 """
195 xbuddy_config = ConfigParser.ConfigParser()
196 config_file = os.path.join(self.devserver_dir, CONFIG_FILE)
197 if os.path.exists(config_file):
198 xbuddy_config.read(config_file)
199 else:
200 raise XBuddyException('%s not found' % (CONFIG_FILE))
201
202 # Read the shadow file if there is one.
Chris Sosac2abc722013-08-26 17:11:22 -0700203 if os.path.isdir(CHROOT_SHADOW_DIR):
204 shadow_config_file = os.path.join(CHROOT_SHADOW_DIR, SHADOW_CONFIG_FILE)
205 else:
206 shadow_config_file = os.path.join(self.devserver_dir, SHADOW_CONFIG_FILE)
207
208 _Log('Using shadow config file stored at %s', shadow_config_file)
joychen562699a2013-08-13 15:22:14 -0700209 if os.path.exists(shadow_config_file):
210 shadow_xbuddy_config = ConfigParser.ConfigParser()
211 shadow_xbuddy_config.read(shadow_config_file)
212
213 # Merge shadow config in.
214 sections = shadow_xbuddy_config.sections()
215 for s in sections:
216 if not xbuddy_config.has_section(s):
217 xbuddy_config.add_section(s)
218 options = shadow_xbuddy_config.options(s)
219 for o in options:
220 val = shadow_xbuddy_config.get(s, o)
221 xbuddy_config.set(s, o, val)
222
223 return xbuddy_config
224
225 def _ManageBuilds(self):
226 """Checks if xBuddy is managing local builds using the current config."""
227 try:
228 return self.ParseBoolean(self.config.get(GENERAL, 'manage_builds'))
229 except ConfigParser.Error:
230 return False
231
232 def _Capacity(self):
233 """Gets the xbuddy capacity from the current config."""
234 try:
235 return int(self.config.get(GENERAL, 'capacity'))
236 except ConfigParser.Error:
237 return 5
238
239 def _LookupAlias(self, alias, board):
240 """Given the full xbuddy config, look up an alias for path rewrite.
241
242 Args:
243 alias: The xbuddy path that could be one of the aliases in the
244 rewrite table.
245 board: The board to fill in with when paths are rewritten. Can be from
246 the update request xml or the default board from devserver.
247 Returns:
248 If a rewrite is found, a string with the current board substituted in.
249 If no rewrite is found, just return the original string.
250 """
251 if alias == '':
252 alias = 'update_default'
253
254 try:
255 val = self.config.get(PATH_REWRITES, alias)
256 except ConfigParser.Error:
257 # No alias lookup found. Return original path.
258 return alias
259
260 if not val.strip():
261 # The found value was an empty string.
262 return alias
263 else:
264 # Fill in the board.
joychenc3944cb2013-08-19 10:42:07 -0700265 rewrite = val.replace("BOARD", "%(board)s") % {
joychen562699a2013-08-13 15:22:14 -0700266 'board': board}
267 _Log("Path was rewritten to %s", rewrite)
268 return rewrite
269
joychenf8f07e22013-07-12 17:45:51 -0700270 def _LookupOfficial(self, board, suffix=RELEASE):
271 """Check LATEST-master for the version number of interest."""
272 _Log("Checking gs for latest %s-%s image", board, suffix)
273 latest_addr = devserver_constants.GS_LATEST_MASTER % {'board':board,
274 'suffix':suffix}
275 cmd = 'gsutil cat %s' % latest_addr
276 msg = 'Failed to find build at %s' % latest_addr
joychen121fc9b2013-08-02 14:30:30 -0700277 # Full release + version is in the LATEST file.
joychenf8f07e22013-07-12 17:45:51 -0700278 version = gsutil_util.GSUtilRun(cmd, msg)
joychen3cb228e2013-06-12 12:13:13 -0700279
joychenf8f07e22013-07-12 17:45:51 -0700280 return devserver_constants.IMAGE_DIR % {'board':board,
281 'suffix':suffix,
282 'version':version}
283
284 def _LookupChannel(self, board, channel='stable'):
285 """Check the channel folder for the version number of interest."""
joychen121fc9b2013-08-02 14:30:30 -0700286 # Get all names in channel dir. Get 10 highest directories by version.
joychen7df67f72013-07-18 14:21:12 -0700287 _Log("Checking channel '%s' for latest '%s' image", channel, board)
joychenf8f07e22013-07-12 17:45:51 -0700288 channel_dir = devserver_constants.GS_CHANNEL_DIR % {'channel':channel,
289 'board':board}
joychen562699a2013-08-13 15:22:14 -0700290 latest_version = gsutil_util.GetLatestVersionFromGSDir(
291 channel_dir, with_release=False)
joychenf8f07e22013-07-12 17:45:51 -0700292
joychen121fc9b2013-08-02 14:30:30 -0700293 # Figure out release number from the version number.
joychenc3944cb2013-08-19 10:42:07 -0700294 image_url = devserver_constants.IMAGE_DIR % {
295 'board':board,
296 'suffix':RELEASE,
297 'version':'R*' + latest_version}
joychenf8f07e22013-07-12 17:45:51 -0700298 image_dir = os.path.join(devserver_constants.GS_IMAGE_DIR, image_url)
299
300 # There should only be one match on cros-image-archive.
301 full_version = gsutil_util.GetLatestVersionFromGSDir(image_dir)
302
303 return devserver_constants.IMAGE_DIR % {'board':board,
304 'suffix':RELEASE,
305 'version':full_version}
306
307 def _LookupVersion(self, board, version):
308 """Search GS image releases for the highest match to a version prefix."""
joychen121fc9b2013-08-02 14:30:30 -0700309 # Build the pattern for GS to match.
joychen7df67f72013-07-18 14:21:12 -0700310 _Log("Checking gs for latest '%s' image with prefix '%s'", board, version)
joychenf8f07e22013-07-12 17:45:51 -0700311 image_url = devserver_constants.IMAGE_DIR % {'board':board,
312 'suffix':RELEASE,
313 'version':version + '*'}
314 image_dir = os.path.join(devserver_constants.GS_IMAGE_DIR, image_url)
315
joychen121fc9b2013-08-02 14:30:30 -0700316 # Grab the newest version of the ones matched.
joychenf8f07e22013-07-12 17:45:51 -0700317 full_version = gsutil_util.GetLatestVersionFromGSDir(image_dir)
318 return devserver_constants.IMAGE_DIR % {'board':board,
319 'suffix':RELEASE,
320 'version':full_version}
321
322 def _ResolveVersionToUrl(self, board, version):
joychen121fc9b2013-08-02 14:30:30 -0700323 """Handle version aliases for remote payloads in GS.
joychen3cb228e2013-06-12 12:13:13 -0700324
325 Args:
326 board: as specified in the original call. (i.e. x86-generic, parrot)
327 version: as entered in the original call. can be
328 {TBD, 0. some custom alias as defined in a config file}
329 1. latest
330 2. latest-{channel}
331 3. latest-official-{board suffix}
332 4. version prefix (i.e. RX-Y.X, RX-Y, RX)
joychen3cb228e2013-06-12 12:13:13 -0700333
334 Returns:
joychen121fc9b2013-08-02 14:30:30 -0700335 Location where the image dir is actually found on GS
joychen3cb228e2013-06-12 12:13:13 -0700336
337 """
joychen121fc9b2013-08-02 14:30:30 -0700338 # TODO(joychen): Convert separate calls to a dict + error out bad paths.
joychen3cb228e2013-06-12 12:13:13 -0700339
joychenf8f07e22013-07-12 17:45:51 -0700340 # Only the last segment of the alias is variable relative to the rest.
341 version_tuple = version.rsplit('-', 1)
joychen3cb228e2013-06-12 12:13:13 -0700342
joychenf8f07e22013-07-12 17:45:51 -0700343 if re.match(devserver_constants.VERSION_RE, version):
344 # This is supposed to be a complete version number on GS. Return it.
345 return devserver_constants.IMAGE_DIR % {'board':board,
346 'suffix':RELEASE,
347 'version':version}
348 elif version == LATEST_OFFICIAL:
349 # latest-official --> LATEST build in board-release
350 return self._LookupOfficial(board)
351 elif version_tuple[0] == LATEST_OFFICIAL:
352 # latest-official-{suffix} --> LATEST build in board-{suffix}
353 return self._LookupOfficial(board, version_tuple[1])
354 elif version == LATEST:
355 # latest --> latest build on stable channel
356 return self._LookupChannel(board)
357 elif version_tuple[0] == LATEST:
358 if re.match(devserver_constants.VERSION_RE, version_tuple[1]):
359 # latest-R* --> most recent qualifying build
360 return self._LookupVersion(board, version_tuple[1])
361 else:
362 # latest-{channel} --> latest build within that channel
363 return self._LookupChannel(board, version_tuple[1])
joychen3cb228e2013-06-12 12:13:13 -0700364 else:
365 # The given version doesn't match any known patterns.
joychen921e1fb2013-06-28 11:12:20 -0700366 raise XBuddyException("Version %s unknown. Can't find on GS." % version)
joychen3cb228e2013-06-12 12:13:13 -0700367
joychen5260b9a2013-07-16 14:48:01 -0700368 @staticmethod
369 def _Symlink(link, target):
370 """Symlinks link to target, and removes whatever link was there before."""
371 _Log("Linking to %s from %s", link, target)
372 if os.path.lexists(link):
373 os.unlink(link)
374 os.symlink(target, link)
375
joychen121fc9b2013-08-02 14:30:30 -0700376 def _GetLatestLocalVersion(self, board):
joychen921e1fb2013-06-28 11:12:20 -0700377 """Get the version of the latest image built for board by build_image
378
379 Updates the symlink reference within the xBuddy static dir to point to
380 the real image dir in the local /build/images directory.
381
382 Args:
joychenc3944cb2013-08-19 10:42:07 -0700383 board: board that image was built for.
joychen921e1fb2013-06-28 11:12:20 -0700384
385 Returns:
joychen121fc9b2013-08-02 14:30:30 -0700386 The discovered version of the image.
joychenc3944cb2013-08-19 10:42:07 -0700387
388 Raises:
389 XBuddyException if neither test nor dev image was found in latest built
390 directory.
joychen3cb228e2013-06-12 12:13:13 -0700391 """
joychen921e1fb2013-06-28 11:12:20 -0700392 latest_local_dir = self.GetLatestImageDir(board)
joychenb0dfe552013-07-30 10:02:06 -0700393 if not latest_local_dir or not os.path.exists(latest_local_dir):
joychen921e1fb2013-06-28 11:12:20 -0700394 raise XBuddyException('No builds found for %s. Did you run build_image?' %
395 board)
396
joychen121fc9b2013-08-02 14:30:30 -0700397 # Assume that the version number is the name of the directory.
joychenc3944cb2013-08-19 10:42:07 -0700398 return os.path.basename(latest_local_dir.rstrip('/'))
joychen921e1fb2013-06-28 11:12:20 -0700399
joychenc3944cb2013-08-19 10:42:07 -0700400 @staticmethod
401 def _FindAny(local_dir):
402 """Returns the image_type for ANY given the local_dir."""
403 dev_image = os.path.join(local_dir, devserver_constants.IMAGE_FILE)
404 test_image = os.path.join(local_dir, devserver_constants.TEST_IMAGE_FILE)
405 if os.path.exists(dev_image):
406 return 'dev'
407
408 if os.path.exists(test_image):
409 return 'test'
410
411 raise XBuddyException('No images found in %s' % local_dir)
412
413 @staticmethod
414 def _InterpretPath(path):
joychen121fc9b2013-08-02 14:30:30 -0700415 """Split and return the pieces of an xBuddy path name
joychen921e1fb2013-06-28 11:12:20 -0700416
joychen121fc9b2013-08-02 14:30:30 -0700417 Args:
418 path: the path xBuddy Get was called with.
joychen3cb228e2013-06-12 12:13:13 -0700419
420 Return:
joychenf8f07e22013-07-12 17:45:51 -0700421 tuple of (image_type, board, version)
joychen3cb228e2013-06-12 12:13:13 -0700422
423 Raises:
424 XBuddyException: if the path can't be resolved into valid components
425 """
joychen121fc9b2013-08-02 14:30:30 -0700426 path_list = filter(None, path.split('/'))
joychen7df67f72013-07-18 14:21:12 -0700427
428 # Required parts of path parsing.
429 try:
430 # Determine if image is explicitly local or remote.
joychen121fc9b2013-08-02 14:30:30 -0700431 is_local = True
432 if path_list[0] in (REMOTE, LOCAL):
joychen18737f32013-08-16 17:18:12 -0700433 is_local = (path_list.pop(0) == LOCAL)
joychen7df67f72013-07-18 14:21:12 -0700434
joychen121fc9b2013-08-02 14:30:30 -0700435 # Set board.
joychen7df67f72013-07-18 14:21:12 -0700436 board = path_list.pop(0)
joychen7df67f72013-07-18 14:21:12 -0700437
joychen121fc9b2013-08-02 14:30:30 -0700438 # Set defaults.
joychen3cb228e2013-06-12 12:13:13 -0700439 version = LATEST
joychen921e1fb2013-06-28 11:12:20 -0700440 image_type = GS_ALIASES[0]
joychen7df67f72013-07-18 14:21:12 -0700441 except IndexError:
442 msg = "Specify at least the board in your xBuddy call. Your path: %s"
443 raise XBuddyException(msg % os.path.join(path_list))
joychen3cb228e2013-06-12 12:13:13 -0700444
joychen121fc9b2013-08-02 14:30:30 -0700445 # Read as much of the xBuddy path as possible.
joychen7df67f72013-07-18 14:21:12 -0700446 try:
joychen121fc9b2013-08-02 14:30:30 -0700447 # Override default if terminal is a valid artifact alias or a version.
joychen7df67f72013-07-18 14:21:12 -0700448 terminal = path_list[-1]
449 if terminal in GS_ALIASES + LOCAL_ALIASES:
450 image_type = terminal
451 version = path_list[-2]
452 else:
453 version = terminal
454 except IndexError:
455 # This path doesn't have an alias or a version. That's fine.
456 _Log("Some parts of the path not specified. Using defaults.")
457
joychen346531c2013-07-24 16:55:56 -0700458 _Log("Get artifact '%s' in '%s/%s'. Locally? %s",
joychen7df67f72013-07-18 14:21:12 -0700459 image_type, board, version, is_local)
460
461 return image_type, board, version, is_local
joychen3cb228e2013-06-12 12:13:13 -0700462
joychen921e1fb2013-06-28 11:12:20 -0700463 def _SyncRegistryWithBuildImages(self):
joychen5260b9a2013-07-16 14:48:01 -0700464 """ Crawl images_dir for build_ids of images generated from build_image.
465
466 This will find images and symlink them in xBuddy's static dir so that
467 xBuddy's cache can serve them.
468 If xBuddy's _manage_builds option is on, then a timestamp will also be
469 generated, and xBuddy will clear them from the directory they are in, as
470 necessary.
471 """
joychen921e1fb2013-06-28 11:12:20 -0700472 build_ids = []
473 for b in os.listdir(self.images_dir):
joychen5260b9a2013-07-16 14:48:01 -0700474 # Ensure we have directories to track all boards in build/images
475 common_util.MkDirP(os.path.join(self.static_dir, b))
joychen921e1fb2013-06-28 11:12:20 -0700476 board_dir = os.path.join(self.images_dir, b)
477 build_ids.extend(['/'.join([b, v]) for v
joychenc3944cb2013-08-19 10:42:07 -0700478 in os.listdir(board_dir) if not v == LATEST])
joychen921e1fb2013-06-28 11:12:20 -0700479
joychen121fc9b2013-08-02 14:30:30 -0700480 # Check currently registered images.
joychen921e1fb2013-06-28 11:12:20 -0700481 for f in os.listdir(self._timestamp_folder):
482 build_id = Timestamp.TimestampToBuild(f)
483 if build_id in build_ids:
484 build_ids.remove(build_id)
485
joychen121fc9b2013-08-02 14:30:30 -0700486 # Symlink undiscovered images, and update timestamps if manage_builds is on.
joychen5260b9a2013-07-16 14:48:01 -0700487 for build_id in build_ids:
488 link = os.path.join(self.static_dir, build_id)
489 target = os.path.join(self.images_dir, build_id)
490 XBuddy._Symlink(link, target)
491 if self._manage_builds:
492 Timestamp.UpdateTimestamp(self._timestamp_folder, build_id)
joychen921e1fb2013-06-28 11:12:20 -0700493
494 def _ListBuildTimes(self):
joychen3cb228e2013-06-12 12:13:13 -0700495 """ Returns the currently cached builds and their last access timestamp.
496
497 Returns:
498 list of tuples that matches xBuddy build/version to timestamps in long
499 """
joychen121fc9b2013-08-02 14:30:30 -0700500 # Update currently cached builds.
joychen3cb228e2013-06-12 12:13:13 -0700501 build_dict = {}
502
joychen7df67f72013-07-18 14:21:12 -0700503 for f in os.listdir(self._timestamp_folder):
joychen3cb228e2013-06-12 12:13:13 -0700504 last_accessed = os.path.getmtime(os.path.join(self._timestamp_folder, f))
505 build_id = Timestamp.TimestampToBuild(f)
joychenc3944cb2013-08-19 10:42:07 -0700506 stale_time = datetime.timedelta(seconds=(time.time() - last_accessed))
joychen921e1fb2013-06-28 11:12:20 -0700507 build_dict[build_id] = stale_time
joychen3cb228e2013-06-12 12:13:13 -0700508 return_tup = sorted(build_dict.iteritems(), key=operator.itemgetter(1))
509 return return_tup
510
joychen3cb228e2013-06-12 12:13:13 -0700511 def _Download(self, gs_url, artifact):
512 """Download the single artifact from the given gs_url."""
513 with XBuddy._staging_thread_count_lock:
514 XBuddy._staging_thread_count += 1
515 try:
joychen7df67f72013-07-18 14:21:12 -0700516 _Log("Downloading '%s' from '%s'", artifact, gs_url)
joychen921e1fb2013-06-28 11:12:20 -0700517 downloader.Downloader(self.static_dir, gs_url).Download(
joychen18737f32013-08-16 17:18:12 -0700518 [artifact], [])
joychen3cb228e2013-06-12 12:13:13 -0700519 finally:
520 with XBuddy._staging_thread_count_lock:
521 XBuddy._staging_thread_count -= 1
522
523 def _CleanCache(self):
joychen562699a2013-08-13 15:22:14 -0700524 """Delete all builds besides the newest N builds"""
joychen121fc9b2013-08-02 14:30:30 -0700525 if not self._manage_builds:
526 return
joychen921e1fb2013-06-28 11:12:20 -0700527 cached_builds = [e[0] for e in self._ListBuildTimes()]
joychen3cb228e2013-06-12 12:13:13 -0700528 _Log('In cache now: %s', cached_builds)
529
joychen562699a2013-08-13 15:22:14 -0700530 for b in range(self._Capacity(), len(cached_builds)):
joychen3cb228e2013-06-12 12:13:13 -0700531 b_path = cached_builds[b]
joychen7df67f72013-07-18 14:21:12 -0700532 _Log("Clearing '%s' from cache", b_path)
joychen3cb228e2013-06-12 12:13:13 -0700533
534 time_file = os.path.join(self._timestamp_folder,
535 Timestamp.BuildToTimestamp(b_path))
joychen921e1fb2013-06-28 11:12:20 -0700536 os.unlink(time_file)
537 clear_dir = os.path.join(self.static_dir, b_path)
joychen3cb228e2013-06-12 12:13:13 -0700538 try:
joychen121fc9b2013-08-02 14:30:30 -0700539 # Handle symlinks, in the case of links to local builds if enabled.
540 if os.path.islink(clear_dir):
joychen5260b9a2013-07-16 14:48:01 -0700541 target = os.readlink(clear_dir)
542 _Log('Deleting locally built image at %s', target)
joychen921e1fb2013-06-28 11:12:20 -0700543
544 os.unlink(clear_dir)
joychen5260b9a2013-07-16 14:48:01 -0700545 if os.path.exists(target):
joychen921e1fb2013-06-28 11:12:20 -0700546 shutil.rmtree(target)
547 elif os.path.exists(clear_dir):
joychen5260b9a2013-07-16 14:48:01 -0700548 _Log('Deleting downloaded image at %s', clear_dir)
joychen3cb228e2013-06-12 12:13:13 -0700549 shutil.rmtree(clear_dir)
joychen921e1fb2013-06-28 11:12:20 -0700550
joychen121fc9b2013-08-02 14:30:30 -0700551 except Exception as err:
552 raise XBuddyException('Failed to clear %s: %s' % (clear_dir, err))
joychen3cb228e2013-06-12 12:13:13 -0700553
joychen346531c2013-07-24 16:55:56 -0700554 def _GetFromGS(self, build_id, image_type, lookup_only):
joychen121fc9b2013-08-02 14:30:30 -0700555 """Check if the artifact is available locally. Download from GS if not."""
joychenf8f07e22013-07-12 17:45:51 -0700556 gs_url = os.path.join(devserver_constants.GS_IMAGE_DIR,
joychen921e1fb2013-06-28 11:12:20 -0700557 build_id)
558
joychen121fc9b2013-08-02 14:30:30 -0700559 # Stage image if not found in cache.
joychen921e1fb2013-06-28 11:12:20 -0700560 file_name = GS_ALIAS_TO_FILENAME[image_type]
joychen346531c2013-07-24 16:55:56 -0700561 file_loc = os.path.join(self.static_dir, build_id, file_name)
562 cached = os.path.exists(file_loc)
563
joychen921e1fb2013-06-28 11:12:20 -0700564 if not cached:
joychen121fc9b2013-08-02 14:30:30 -0700565 if not lookup_only:
joychen346531c2013-07-24 16:55:56 -0700566 artifact = GS_ALIAS_TO_ARTIFACT[image_type]
567 self._Download(gs_url, artifact)
joychen921e1fb2013-06-28 11:12:20 -0700568 else:
569 _Log('Image already cached.')
570
joychen562699a2013-08-13 15:22:14 -0700571 def _GetArtifact(self, path_list, board, lookup_only=False):
joychen346531c2013-07-24 16:55:56 -0700572 """Interpret an xBuddy path and return directory/file_name to resource.
573
574 Returns:
joychenc3944cb2013-08-19 10:42:07 -0700575 build_id to the directory
joychen346531c2013-07-24 16:55:56 -0700576 file_name of the artifact
joychen346531c2013-07-24 16:55:56 -0700577
578 Raises:
joychen121fc9b2013-08-02 14:30:30 -0700579 XBuddyException: if the path could not be translated
joychen346531c2013-07-24 16:55:56 -0700580 """
joychen121fc9b2013-08-02 14:30:30 -0700581 path = '/'.join(path_list)
joychenb0dfe552013-07-30 10:02:06 -0700582 # Rewrite the path if there is an appropriate default.
joychen562699a2013-08-13 15:22:14 -0700583 path = self._LookupAlias(path, board)
joychenb0dfe552013-07-30 10:02:06 -0700584
joychen121fc9b2013-08-02 14:30:30 -0700585 # Parse the path.
joychen7df67f72013-07-18 14:21:12 -0700586 image_type, board, version, is_local = self._InterpretPath(path)
joychen921e1fb2013-06-28 11:12:20 -0700587
joychen7df67f72013-07-18 14:21:12 -0700588 if is_local:
joychen121fc9b2013-08-02 14:30:30 -0700589 # Get a local image.
joychen7df67f72013-07-18 14:21:12 -0700590 if version == LATEST:
joychen121fc9b2013-08-02 14:30:30 -0700591 # Get the latest local image for the given board.
592 version = self._GetLatestLocalVersion(board)
joychen7df67f72013-07-18 14:21:12 -0700593
joychenc3944cb2013-08-19 10:42:07 -0700594 build_id = os.path.join(board, version)
595 artifact_dir = os.path.join(self.static_dir, build_id)
596 if image_type == ANY:
597 image_type = self._FindAny(artifact_dir)
joychen121fc9b2013-08-02 14:30:30 -0700598
joychenc3944cb2013-08-19 10:42:07 -0700599 file_name = LOCAL_ALIAS_TO_FILENAME[image_type]
600 artifact_path = os.path.join(artifact_dir, file_name)
601 if not os.path.exists(artifact_path):
602 raise XBuddyException('Local %s artifact not in static_dir at %s' %
603 (image_type, artifact_path))
joychen121fc9b2013-08-02 14:30:30 -0700604
joychen921e1fb2013-06-28 11:12:20 -0700605 else:
joychen121fc9b2013-08-02 14:30:30 -0700606 # Get a remote image.
joychen921e1fb2013-06-28 11:12:20 -0700607 if image_type not in GS_ALIASES:
joychen7df67f72013-07-18 14:21:12 -0700608 raise XBuddyException('Bad remote image type: %s. Use one of: %s' %
joychen921e1fb2013-06-28 11:12:20 -0700609 (image_type, GS_ALIASES))
joychen921e1fb2013-06-28 11:12:20 -0700610 file_name = GS_ALIAS_TO_FILENAME[image_type]
joychen921e1fb2013-06-28 11:12:20 -0700611
joychen121fc9b2013-08-02 14:30:30 -0700612 # Interpret the version (alias), and get gs address.
joychenc3944cb2013-08-19 10:42:07 -0700613 build_id = self._ResolveVersionToUrl(board, version)
614 _Log('Found on GS: %s', build_id)
615 self._GetFromGS(build_id, image_type, lookup_only)
joychenf8f07e22013-07-12 17:45:51 -0700616
joychenc3944cb2013-08-19 10:42:07 -0700617 return build_id, file_name
joychen3cb228e2013-06-12 12:13:13 -0700618
619 ############################ BEGIN PUBLIC METHODS
620
621 def List(self):
622 """Lists the currently available images & time since last access."""
joychen921e1fb2013-06-28 11:12:20 -0700623 self._SyncRegistryWithBuildImages()
624 builds = self._ListBuildTimes()
625 return_string = ''
626 for build, timestamp in builds:
627 return_string += '<b>' + build + '</b> '
628 return_string += '(time since last access: ' + str(timestamp) + ')<br>'
629 return return_string
joychen3cb228e2013-06-12 12:13:13 -0700630
631 def Capacity(self):
632 """Returns the number of images cached by xBuddy."""
joychen562699a2013-08-13 15:22:14 -0700633 return str(self._Capacity())
joychen3cb228e2013-06-12 12:13:13 -0700634
joychen562699a2013-08-13 15:22:14 -0700635 def Translate(self, path_list, board):
joychen346531c2013-07-24 16:55:56 -0700636 """Translates an xBuddy path to a real path to artifact if it exists.
637
joychen121fc9b2013-08-02 14:30:30 -0700638 Equivalent to the Get call, minus downloading and updating timestamps,
joychen346531c2013-07-24 16:55:56 -0700639
joychen7c2054a2013-07-25 11:14:07 -0700640 Returns:
joychen121fc9b2013-08-02 14:30:30 -0700641 build_id: Path to the image or update directory on the devserver.
642 e.g. 'x86-generic/R26-4000.0.0'
643 The returned path is always the path to the directory within
644 static_dir, so it is always the build_id of the image.
645 file_name: The file name of the artifact. Can take any of the file
646 values in devserver_constants.
647 e.g. 'chromiumos_test_image.bin' or 'update.gz' if the path list
648 specified 'test' or 'full_payload' artifacts, respectively.
joychen7c2054a2013-07-25 11:14:07 -0700649
joychen121fc9b2013-08-02 14:30:30 -0700650 Raises:
651 XBuddyException: if the path couldn't be translated
joychen346531c2013-07-24 16:55:56 -0700652 """
653 self._SyncRegistryWithBuildImages()
joychen121fc9b2013-08-02 14:30:30 -0700654 build_id, file_name = self._GetArtifact(path_list, board, lookup_only=True)
joychen346531c2013-07-24 16:55:56 -0700655
joychen121fc9b2013-08-02 14:30:30 -0700656 _Log('Returning path to payload: %s/%s', build_id, file_name)
657 return build_id, file_name
joychen346531c2013-07-24 16:55:56 -0700658
joychen562699a2013-08-13 15:22:14 -0700659 def Get(self, path_list):
joychen921e1fb2013-06-28 11:12:20 -0700660 """The full xBuddy call, returns resource specified by path_list.
joychen3cb228e2013-06-12 12:13:13 -0700661
662 Please see devserver.py:xbuddy for full documentation.
joychen121fc9b2013-08-02 14:30:30 -0700663
joychen3cb228e2013-06-12 12:13:13 -0700664 Args:
joychen921e1fb2013-06-28 11:12:20 -0700665 path_list: [board, version, alias] as split from the xbuddy call url
joychen3cb228e2013-06-12 12:13:13 -0700666
667 Returns:
joychen121fc9b2013-08-02 14:30:30 -0700668 build_id: Path to the image or update directory on the devserver.
669 e.g. 'x86-generic/R26-4000.0.0'
670 The returned path is always the path to the directory within
671 static_dir, so it is always the build_id of the image.
672 file_name: The file name of the artifact. Can take any of the file
673 values in devserver_constants.
674 e.g. 'chromiumos_test_image.bin' or 'update.gz' if the path list
675 specified 'test' or 'full_payload' artifacts, respectively.
joychen3cb228e2013-06-12 12:13:13 -0700676
677 Raises:
joychen121fc9b2013-08-02 14:30:30 -0700678 XBuddyException: if path is invalid
joychen3cb228e2013-06-12 12:13:13 -0700679 """
joychen7df67f72013-07-18 14:21:12 -0700680 self._SyncRegistryWithBuildImages()
joychen562699a2013-08-13 15:22:14 -0700681 build_id, file_name = self._GetArtifact(path_list, self._board)
joychen3cb228e2013-06-12 12:13:13 -0700682
joychen921e1fb2013-06-28 11:12:20 -0700683 Timestamp.UpdateTimestamp(self._timestamp_folder, build_id)
joychen3cb228e2013-06-12 12:13:13 -0700684 #TODO (joyc): run in sep thread
685 self._CleanCache()
686
joychen121fc9b2013-08-02 14:30:30 -0700687 _Log('Returning path to payload: %s/%s', build_id, file_name)
688 return build_id, file_name