blob: 389abaa00b6c1b0273262a8f67ffa3befc8eee9b [file] [log] [blame]
Chris Sosa7c931362010-10-11 19:49:01 -07001#!/usr/bin/python
2
Chris Sosa781ba6d2012-04-11 12:44:43 -07003# Copyright (c) 2009-2012 The Chromium OS Authors. All rights reserved.
rtc@google.comded22402009-10-26 22:36:21 +00004# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
Chris Sosa7c931362010-10-11 19:49:01 -07007"""A CherryPy-based webserver to host images and build packages."""
8
Chris Sosadbc20082012-12-10 13:39:11 -08009import cherrypy
Gilad Arnold55a2a372012-10-02 09:46:32 -070010import json
Chris Sosa781ba6d2012-04-11 12:44:43 -070011import logging
Sean O'Connor14b6a0a2010-03-20 23:23:48 -070012import optparse
rtc@google.comded22402009-10-26 22:36:21 +000013import os
Scott Zawalski4647ce62012-01-03 17:17:28 -050014import re
Mandeep Singh Baines38dcdda2012-12-07 17:55:33 -080015import socket
chocobo@google.com4dc25812009-10-27 23:46:26 +000016import sys
Chris Masone816e38c2012-05-02 12:22:36 -070017import subprocess
18import tempfile
Gilad Arnold0b8c3f32012-09-19 14:35:44 -070019import threading
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -070020import types
rtc@google.comded22402009-10-26 22:36:21 +000021
Chris Sosa0356d3b2010-09-16 15:46:22 -070022import autoupdate
Gilad Arnoldc65330c2012-09-20 15:17:48 -070023import common_util
Chris Sosa47a7d4e2012-03-28 11:26:55 -070024import downloader
Gilad Arnoldc65330c2012-09-20 15:17:48 -070025import log_util
26
27
28# Module-local log function.
Chris Sosa8dd80092012-12-10 13:39:11 -080029def _Log(message, *args):
30 return log_util.LogWithTag('DEVSERVER', message, *args)
Chris Sosa0356d3b2010-09-16 15:46:22 -070031
Frank Farzan40160872011-12-12 18:39:18 -080032
Chris Sosa417e55d2011-01-25 16:40:48 -080033CACHED_ENTRIES = 12
Don Garrettf90edf02010-11-16 17:36:14 -080034
Chris Sosa0356d3b2010-09-16 15:46:22 -070035# Sets up global to share between classes.
rtc@google.com21a5ca32009-11-04 18:23:23 +000036global updater
37updater = None
rtc@google.comded22402009-10-26 22:36:21 +000038
Frank Farzan40160872011-12-12 18:39:18 -080039
Chris Sosa9164ca32012-03-28 11:04:50 -070040class DevServerError(Exception):
Chris Sosa47a7d4e2012-03-28 11:26:55 -070041 """Exception class used by this module."""
42 pass
43
44
Gilad Arnold0b8c3f32012-09-19 14:35:44 -070045class LockDict(object):
46 """A dictionary of locks.
47
48 This class provides a thread-safe store of threading.Lock objects, which can
49 be used to regulate access to any set of hashable resources. Usage:
50
51 foo_lock_dict = LockDict()
52 ...
53 with foo_lock_dict.lock('bar'):
54 # Critical section for 'bar'
55 """
56 def __init__(self):
57 self._lock = self._new_lock()
58 self._dict = {}
59
60 def _new_lock(self):
61 return threading.Lock()
62
63 def lock(self, key):
64 with self._lock:
65 lock = self._dict.get(key)
66 if not lock:
67 lock = self._new_lock()
68 self._dict[key] = lock
69 return lock
70
71
Scott Zawalski4647ce62012-01-03 17:17:28 -050072def _LeadingWhiteSpaceCount(string):
73 """Count the amount of leading whitespace in a string.
74
75 Args:
76 string: The string to count leading whitespace in.
77 Returns:
78 number of white space chars before characters start.
79 """
80 matched = re.match('^\s+', string)
81 if matched:
82 return len(matched.group())
83
84 return 0
85
86
87def _PrintDocStringAsHTML(func):
88 """Make a functions docstring somewhat HTML style.
89
90 Args:
91 func: The function to return the docstring from.
92 Returns:
93 A string that is somewhat formated for a web browser.
94 """
95 # TODO(scottz): Make this parse Args/Returns in a prettier way.
96 # Arguments could be bolded and indented etc.
97 html_doc = []
98 for line in func.__doc__.splitlines():
99 leading_space = _LeadingWhiteSpaceCount(line)
100 if leading_space > 0:
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700101 line = ' ' * leading_space + line
Scott Zawalski4647ce62012-01-03 17:17:28 -0500102
103 html_doc.append('<BR>%s' % line)
104
105 return '\n'.join(html_doc)
106
107
Chris Sosa7c931362010-10-11 19:49:01 -0700108def _GetConfig(options):
109 """Returns the configuration for the devserver."""
Mandeep Singh Baines38dcdda2012-12-07 17:55:33 -0800110
111 # On a system with IPv6 not compiled into the kernel,
112 # AF_INET6 sockets will return a socket.error exception.
113 # On such systems, fall-back to IPv4.
114 socket_host = '::'
115 try:
116 socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
117 except socket.error:
118 socket_host = '0.0.0.0'
119
Chris Sosa7c931362010-10-11 19:49:01 -0700120 base_config = { 'global':
121 { 'server.log_request_headers': True,
122 'server.protocol_version': 'HTTP/1.1',
Mandeep Singh Baines38dcdda2012-12-07 17:55:33 -0800123 'server.socket_host': socket_host,
Chris Sosa7c931362010-10-11 19:49:01 -0700124 'server.socket_port': int(options.port),
Chris Sosa374c62d2010-10-14 09:13:54 -0700125 'response.timeout': 6000,
Chris Sosa6fe23942012-07-02 15:44:46 -0700126 'request.show_tracebacks': True,
Chris Sosa72333d12012-06-13 11:28:05 -0700127 'server.socket_timeout': 60,
Zdenek Behan1347a312011-02-10 03:59:17 +0100128 'tools.staticdir.root':
129 os.path.dirname(os.path.abspath(sys.argv[0])),
Chris Sosa7c931362010-10-11 19:49:01 -0700130 },
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700131 '/api':
132 {
133 # Gets rid of cherrypy parsing post file for args.
134 'request.process_request_body': False,
135 },
Chris Sosaa1ef0102010-10-21 16:22:35 -0700136 '/build':
137 {
138 'response.timeout': 100000,
139 },
Chris Sosa7c931362010-10-11 19:49:01 -0700140 '/update':
141 {
142 # Gets rid of cherrypy parsing post file for args.
143 'request.process_request_body': False,
Chris Sosaf65f4b92010-10-21 15:57:51 -0700144 'response.timeout': 10000,
Chris Sosa7c931362010-10-11 19:49:01 -0700145 },
146 # Sets up the static dir for file hosting.
147 '/static':
148 { 'tools.staticdir.dir': 'static',
149 'tools.staticdir.on': True,
Chris Sosaf65f4b92010-10-21 15:57:51 -0700150 'response.timeout': 10000,
Chris Sosa7c931362010-10-11 19:49:01 -0700151 },
152 }
Chris Sosa5f118ef2012-07-12 11:37:50 -0700153 if options.production:
Chris Sosad1ea86b2012-07-12 13:35:37 -0700154 base_config['global'].update({'server.thread_pool': 75})
Scott Zawalski1c5e7cd2012-02-27 13:12:52 -0500155
Chris Sosa7c931362010-10-11 19:49:01 -0700156 return base_config
rtc@google.com64244662009-11-12 00:52:08 +0000157
Darin Petkove17164a2010-08-11 13:24:41 -0700158
Zdenek Behan608f46c2011-02-19 00:47:16 +0100159def _PrepareToServeUpdatesOnly(image_dir, static_dir):
Chris Sosa0356d3b2010-09-16 15:46:22 -0700160 """Sets up symlink to image_dir for serving purposes."""
161 assert os.path.exists(image_dir), '%s must exist.' % image_dir
162 # If we're serving out of an archived build dir (e.g. a
163 # buildbot), prepare this webserver's magic 'static/' dir with a
164 # link to the build archive.
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700165 _Log('Preparing autoupdate for "serve updates only" mode.')
Zdenek Behan608f46c2011-02-19 00:47:16 +0100166 if os.path.lexists('%s/archive' % static_dir):
167 if image_dir != os.readlink('%s/archive' % static_dir):
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700168 _Log('removing stale symlink to %s' % image_dir)
Zdenek Behan608f46c2011-02-19 00:47:16 +0100169 os.unlink('%s/archive' % static_dir)
170 os.symlink(image_dir, '%s/archive' % static_dir)
Chris Sosacde6bf42012-05-31 18:36:39 -0700171
Chris Sosa0356d3b2010-09-16 15:46:22 -0700172 else:
Zdenek Behan608f46c2011-02-19 00:47:16 +0100173 os.symlink(image_dir, '%s/archive' % static_dir)
Chris Sosacde6bf42012-05-31 18:36:39 -0700174
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700175 _Log('archive dir: %s ready to be used to serve images.' % image_dir)
Chris Sosa7c931362010-10-11 19:49:01 -0700176
177
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700178def _GetRecursiveMemberObject(root, member_list):
179 """Returns an object corresponding to a nested member list.
180
181 Args:
182 root: the root object to search
183 member_list: list of nested members to search
184 Returns:
185 An object corresponding to the member name list; None otherwise.
186 """
187 for member in member_list:
188 next_root = root.__class__.__dict__.get(member)
189 if not next_root:
190 return None
191 root = next_root
192 return root
193
194
195def _IsExposed(name):
196 """Returns True iff |name| has an `exposed' attribute and it is set."""
197 return hasattr(name, 'exposed') and name.exposed
198
199
Gilad Arnold748c8322012-10-12 09:51:35 -0700200def _GetExposedMethod(root, nested_member, ignored=None):
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700201 """Returns a CherryPy-exposed method, if such exists.
202
203 Args:
204 root: the root object for searching
205 nested_member: a slash-joined path to the nested member
206 ignored: method paths to be ignored
207 Returns:
208 A function object corresponding to the path defined by |member_list| from
209 the |root| object, if the function is exposed and not ignored; None
210 otherwise.
211 """
Gilad Arnold748c8322012-10-12 09:51:35 -0700212 method = (not (ignored and nested_member in ignored) and
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700213 _GetRecursiveMemberObject(root, nested_member.split('/')))
214 if (method and type(method) == types.FunctionType and _IsExposed(method)):
215 return method
216
217
Gilad Arnold748c8322012-10-12 09:51:35 -0700218def _FindExposedMethods(root, prefix, unlisted=None):
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700219 """Finds exposed CherryPy methods.
220
221 Args:
222 root: the root object for searching
223 prefix: slash-joined chain of members leading to current object
224 unlisted: URLs to be excluded regardless of their exposed status
225 Returns:
226 List of exposed URLs that are not unlisted.
227 """
228 method_list = []
229 for member in sorted(root.__class__.__dict__.keys()):
230 prefixed_member = prefix + '/' + member if prefix else member
Gilad Arnold748c8322012-10-12 09:51:35 -0700231 if unlisted and prefixed_member in unlisted:
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700232 continue
233 member_obj = root.__class__.__dict__[member]
234 if _IsExposed(member_obj):
235 if type(member_obj) == types.FunctionType:
236 method_list.append(prefixed_member)
237 else:
238 method_list += _FindExposedMethods(
239 member_obj, prefixed_member, unlisted)
240 return method_list
241
242
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700243class ApiRoot(object):
244 """RESTful API for Dev Server information."""
245 exposed = True
246
247 @cherrypy.expose
248 def hostinfo(self, ip):
249 """Returns a JSON dictionary containing information about the given ip.
250
Gilad Arnold1b908392012-10-05 11:36:27 -0700251 Args:
252 ip: address of host whose info is requested
253 Returns:
254 A JSON dictionary containing all or some of the following fields:
255 last_event_type (int): last update event type received
256 last_event_status (int): last update event status received
257 last_known_version (string): last known version reported in update ping
258 forced_update_label (string): update label to force next update ping to
259 use, set by setnextupdate
260 See the OmahaEvent class in update_engine/omaha_request_action.h for
261 event type and status code definitions. If the ip does not exist an empty
262 string is returned.
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700263
Gilad Arnold1b908392012-10-05 11:36:27 -0700264 Example URL:
265 http://myhost/api/hostinfo?ip=192.168.1.5
266 """
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700267 return updater.HandleHostInfoPing(ip)
268
269 @cherrypy.expose
Gilad Arnold286a0062012-01-12 13:47:02 -0800270 def hostlog(self, ip):
Gilad Arnold1b908392012-10-05 11:36:27 -0700271 """Returns a JSON object containing a log of host event.
272
273 Args:
274 ip: address of host whose event log is requested, or `all'
275 Returns:
276 A JSON encoded list (log) of dictionaries (events), each of which
277 containing a `timestamp' and other event fields, as described under
278 /api/hostinfo.
279
280 Example URL:
281 http://myhost/api/hostlog?ip=192.168.1.5
282 """
Gilad Arnold286a0062012-01-12 13:47:02 -0800283 return updater.HandleHostLogPing(ip)
284
285 @cherrypy.expose
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700286 def setnextupdate(self, ip):
287 """Allows the response to the next update ping from a host to be set.
288
289 Takes the IP of the host and an update label as normally provided to the
Gilad Arnold1b908392012-10-05 11:36:27 -0700290 /update command.
291 """
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700292 body_length = int(cherrypy.request.headers['Content-Length'])
293 label = cherrypy.request.rfile.read(body_length)
294
295 if label:
296 label = label.strip()
297 if label:
298 return updater.HandleSetUpdatePing(ip, label)
299 raise cherrypy.HTTPError(400, 'No label provided.')
300
301
Gilad Arnold55a2a372012-10-02 09:46:32 -0700302 @cherrypy.expose
303 def fileinfo(self, *path_args):
304 """Returns information about a given staged file.
305
306 Args:
307 path_args: path to the file inside the server's static staging directory
308 Returns:
309 A JSON encoded dictionary with information about the said file, which may
310 contain the following keys/values:
Gilad Arnold1b908392012-10-05 11:36:27 -0700311 size (int): the file size in bytes
312 sha1 (string): a base64 encoded SHA1 hash
313 sha256 (string): a base64 encoded SHA256 hash
314
315 Example URL:
316 http://myhost/api/fileinfo/some/path/to/file
Gilad Arnold55a2a372012-10-02 09:46:32 -0700317 """
318 file_path = os.path.join(updater.static_dir, *path_args)
319 if not os.path.exists(file_path):
320 raise DevServerError('file not found: %s' % file_path)
321 try:
322 file_size = os.path.getsize(file_path)
323 file_sha1 = common_util.GetFileSha1(file_path)
324 file_sha256 = common_util.GetFileSha256(file_path)
325 except os.error, e:
326 raise DevServerError('failed to get info for file %s: %s' %
327 (file_path, str(e)))
328 return json.dumps(
329 {'size': file_size, 'sha1': file_sha1, 'sha256': file_sha256})
330
David Rochberg7c79a812011-01-19 14:24:45 -0500331class DevServerRoot(object):
Chris Sosa7c931362010-10-11 19:49:01 -0700332 """The Root Class for the Dev Server.
333
334 CherryPy works as follows:
335 For each method in this class, cherrpy interprets root/path
336 as a call to an instance of DevServerRoot->method_name. For example,
337 a call to http://myhost/build will call build. CherryPy automatically
338 parses http args and places them as keyword arguments in each method.
339 For paths http://myhost/update/dir1/dir2, you can use *args so that
340 cherrypy uses the update method and puts the extra paths in args.
341 """
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700342 # Method names that should not be listed on the index page.
343 _UNLISTED_METHODS = ['index', 'doc']
344
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700345 api = ApiRoot()
Chris Sosa7c931362010-10-11 19:49:01 -0700346
David Rochberg7c79a812011-01-19 14:24:45 -0500347 def __init__(self):
Nick Sanders7dcaa2e2011-08-04 15:20:41 -0700348 self._builder = None
Gilad Arnold0b8c3f32012-09-19 14:35:44 -0700349 self._download_lock_dict = LockDict()
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700350 self._downloader_dict = {}
David Rochberg7c79a812011-01-19 14:24:45 -0500351
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700352 @cherrypy.expose
David Rochberg7c79a812011-01-19 14:24:45 -0500353 def build(self, board, pkg, **kwargs):
Chris Sosa7c931362010-10-11 19:49:01 -0700354 """Builds the package specified."""
Nick Sanders7dcaa2e2011-08-04 15:20:41 -0700355 import builder
356 if self._builder is None:
357 self._builder = builder.Builder()
David Rochberg7c79a812011-01-19 14:24:45 -0500358 return self._builder.Build(board, pkg, kwargs)
Chris Sosa7c931362010-10-11 19:49:01 -0700359
Chris Sosacde6bf42012-05-31 18:36:39 -0700360 @staticmethod
361 def _canonicalize_archive_url(archive_url):
362 """Canonicalizes archive_url strings.
363
364 Raises:
365 DevserverError: if archive_url is not set.
366 """
367 if archive_url:
368 return archive_url.rstrip('/')
369 else:
370 raise DevServerError("Must specify an archive_url in the request")
371
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700372 @cherrypy.expose
Frank Farzanbcb571e2012-01-03 11:48:17 -0800373 def download(self, **kwargs):
374 """Downloads and archives full/delta payloads from Google Storage.
375
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700376 This methods downloads artifacts. It may download artifacts in the
377 background in which case a caller should call wait_for_status to get
378 the status of the background artifact downloads. They should use the same
379 args passed to download.
380
Frank Farzanbcb571e2012-01-03 11:48:17 -0800381 Args:
382 archive_url: Google Storage URL for the build.
383
384 Example URL:
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700385 http://myhost/download?archive_url=gs://chromeos-image-archive/
386 x86-generic/R17-1208.0.0-a1-b338
Frank Farzanbcb571e2012-01-03 11:48:17 -0800387 """
Chris Sosacde6bf42012-05-31 18:36:39 -0700388 archive_url = self._canonicalize_archive_url(kwargs.get('archive_url'))
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700389
Chris Sosacde6bf42012-05-31 18:36:39 -0700390 # Guarantees that no two downloads for the same url can run this code
391 # at the same time.
Gilad Arnold0b8c3f32012-09-19 14:35:44 -0700392 with self._download_lock_dict.lock(archive_url):
Chris Sosacde6bf42012-05-31 18:36:39 -0700393 try:
394 # If we are currently downloading, return. Note, due to the above lock
395 # we know that the foreground artifacts must have finished downloading
396 # and returned Success if this downloader instance exists.
397 if (self._downloader_dict.get(archive_url) or
398 downloader.Downloader.BuildStaged(archive_url, updater.static_dir)):
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700399 _Log('Build %s has already been processed.' % archive_url)
Chris Sosacde6bf42012-05-31 18:36:39 -0700400 return 'Success'
401
402 downloader_instance = downloader.Downloader(updater.static_dir)
403 self._downloader_dict[archive_url] = downloader_instance
404 return downloader_instance.Download(archive_url, background=True)
405
406 except:
407 # On any exception, reset the state of the downloader_dict.
408 self._downloader_dict[archive_url] = None
Chris Sosa4d9c4d42012-06-29 15:23:23 -0700409 raise
Chris Sosacde6bf42012-05-31 18:36:39 -0700410
411 @cherrypy.expose
412 def wait_for_status(self, **kwargs):
413 """Waits for background artifacts to be downloaded from Google Storage.
414
415 Args:
416 archive_url: Google Storage URL for the build.
417
418 Example URL:
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700419 http://myhost/wait_for_status?archive_url=gs://chromeos-image-archive/
420 x86-generic/R17-1208.0.0-a1-b338
Chris Sosacde6bf42012-05-31 18:36:39 -0700421 """
422 archive_url = self._canonicalize_archive_url(kwargs.get('archive_url'))
423 downloader_instance = self._downloader_dict.get(archive_url)
424 if downloader_instance:
425 status = downloader_instance.GetStatusOfBackgroundDownloads()
Chris Sosa781ba6d2012-04-11 12:44:43 -0700426 self._downloader_dict[archive_url] = None
Chris Sosacde6bf42012-05-31 18:36:39 -0700427 return status
428 else:
429 # We may have previously downloaded but removed the downloader instance
430 # from the cache.
431 if downloader.Downloader.BuildStaged(archive_url, updater.static_dir):
432 logging.info('%s not found in downloader cache but previously staged.',
433 archive_url)
434 return 'Success'
435 else:
436 raise DevServerError('No download for the given archive_url found.')
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700437
438 @cherrypy.expose
Chris Masone816e38c2012-05-02 12:22:36 -0700439 def stage_debug(self, **kwargs):
440 """Downloads and stages debug symbol payloads from Google Storage.
441
442 This methods downloads the debug symbol build artifact synchronously,
443 and then stages it for use by symbolicate_dump/.
444
445 Args:
446 archive_url: Google Storage URL for the build.
447
448 Example URL:
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700449 http://myhost/stage_debug?archive_url=gs://chromeos-image-archive/
450 x86-generic/R17-1208.0.0-a1-b338
Chris Masone816e38c2012-05-02 12:22:36 -0700451 """
Chris Sosacde6bf42012-05-31 18:36:39 -0700452 archive_url = self._canonicalize_archive_url(kwargs.get('archive_url'))
Chris Masone816e38c2012-05-02 12:22:36 -0700453 return downloader.SymbolDownloader(updater.static_dir).Download(archive_url)
454
455 @cherrypy.expose
456 def symbolicate_dump(self, minidump):
457 """Symbolicates a minidump using pre-downloaded symbols, returns it.
458
459 Callers will need to POST to this URL with a body of MIME-type
460 "multipart/form-data".
461 The body should include a single argument, 'minidump', containing the
462 binary-formatted minidump to symbolicate.
463
464 It is up to the caller to ensure that the symbols they want are currently
465 staged.
466
467 Args:
468 minidump: The binary minidump file to symbolicate.
469 """
470 to_return = ''
471 with tempfile.NamedTemporaryFile() as local:
472 while True:
473 data = minidump.file.read(8192)
474 if not data:
475 break
476 local.write(data)
477 local.flush()
478 stackwalk = subprocess.Popen(['minidump_stackwalk',
479 local.name,
480 updater.static_dir + '/debug/breakpad'],
481 stdout=subprocess.PIPE,
482 stderr=subprocess.PIPE)
483 to_return, error_text = stackwalk.communicate()
484 if stackwalk.returncode != 0:
485 raise DevServerError("Can't generate stack trace: %s (rc=%d)" % (
486 error_text, stackwalk.returncode))
487
488 return to_return
489
490 @cherrypy.expose
Scott Zawalski16954532012-03-20 15:31:36 -0400491 def latestbuild(self, **params):
492 """Return a string representing the latest build for a given target.
493
494 Args:
495 target: The build target, typically a combination of the board and the
496 type of build e.g. x86-mario-release.
497 milestone: The milestone to filter builds on. E.g. R16. Optional, if not
498 provided the latest RXX build will be returned.
499 Returns:
500 A string representation of the latest build if one exists, i.e.
501 R19-1993.0.0-a1-b1480.
502 An empty string if no latest could be found.
503 """
504 if not params:
505 return _PrintDocStringAsHTML(self.latestbuild)
506
507 if 'target' not in params:
508 raise cherrypy.HTTPError('500 Internal Server Error',
509 'Error: target= is required!')
510 try:
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700511 return common_util.GetLatestBuildVersion(
Scott Zawalski16954532012-03-20 15:31:36 -0400512 updater.static_dir, params['target'],
513 milestone=params.get('milestone'))
Gilad Arnold17fe03d2012-10-02 10:05:01 -0700514 except common_util.CommonUtilError as errmsg:
Scott Zawalski16954532012-03-20 15:31:36 -0400515 raise cherrypy.HTTPError('500 Internal Server Error', str(errmsg))
516
517 @cherrypy.expose
Scott Zawalski84a39c92012-01-13 15:12:42 -0500518 def controlfiles(self, **params):
Scott Zawalski4647ce62012-01-03 17:17:28 -0500519 """Return a control file or a list of all known control files.
520
521 Example URL:
522 To List all control files:
Scott Zawalski84a39c92012-01-13 15:12:42 -0500523 http://dev-server/controlfiles?board=x86-alex-release&build=R18-1514.0.0
Scott Zawalski4647ce62012-01-03 17:17:28 -0500524 To return the contents of a path:
Scott Zawalski84a39c92012-01-13 15:12:42 -0500525 http://dev-server/controlfiles?board=x86-alex-release&build=R18-1514.0.0&control_path=client/sleeptest/control
Scott Zawalski4647ce62012-01-03 17:17:28 -0500526
527 Args:
Scott Zawalski84a39c92012-01-13 15:12:42 -0500528 build: The build i.e. x86-alex-release/R18-1514.0.0-a1-b1450.
Scott Zawalski4647ce62012-01-03 17:17:28 -0500529 control_path: If you want the contents of a control file set this
530 to the path. E.g. client/site_tests/sleeptest/control
531 Optional, if not provided return a list of control files is returned.
532 Returns:
533 Contents of a control file if control_path is provided.
534 A list of control files if no control_path is provided.
535 """
Scott Zawalski4647ce62012-01-03 17:17:28 -0500536 if not params:
537 return _PrintDocStringAsHTML(self.controlfiles)
538
Scott Zawalski84a39c92012-01-13 15:12:42 -0500539 if 'build' not in params:
Scott Zawalski4647ce62012-01-03 17:17:28 -0500540 raise cherrypy.HTTPError('500 Internal Server Error',
Scott Zawalski84a39c92012-01-13 15:12:42 -0500541 'Error: build= is required!')
Scott Zawalski4647ce62012-01-03 17:17:28 -0500542
543 if 'control_path' not in params:
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700544 return common_util.GetControlFileList(
545 updater.static_dir, params['build'])
Scott Zawalski4647ce62012-01-03 17:17:28 -0500546 else:
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700547 return common_util.GetControlFile(
548 updater.static_dir, params['build'], params['control_path'])
Frank Farzan40160872011-12-12 18:39:18 -0800549
550 @cherrypy.expose
Gilad Arnold6f99b982012-09-12 10:49:40 -0700551 def stage_images(self, **kwargs):
552 """Downloads and stages a Chrome OS image from Google Storage.
553
554 This method downloads a zipped archive from a specified GS location, then
555 extracts and stages the specified list of images and stages them under
556 static/images/BOARD/BUILD/. Download is synchronous.
557
558 Args:
559 archive_url: Google Storage URL for the build.
560 image_types: comma-separated list of images to download, may include
561 'test', 'recovery', and 'base'
562
563 Example URL:
564 http://myhost/stage_images?archive_url=gs://chromeos-image-archive/
565 x86-generic/R17-1208.0.0-a1-b338&image_types=test,base
566 """
567 # TODO(garnold) This needs to turn into an async operation, to avoid
568 # unnecessary failure of concurrent secondary requests (chromium-os:34661).
569 archive_url = self._canonicalize_archive_url(kwargs.get('archive_url'))
570 image_types = kwargs.get('image_types').split(',')
571 return (downloader.ImagesDownloader(
572 updater.static_dir).Download(archive_url, image_types))
573
574 @cherrypy.expose
Chris Sosa7c931362010-10-11 19:49:01 -0700575 def index(self):
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700576 """Presents a welcome message and documentation links."""
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700577 return ('Welcome to the Dev Server!<br>\n'
578 '<br>\n'
579 'Here are the available methods, click for documentation:<br>\n'
580 '<br>\n'
581 '%s' %
582 '<br>\n'.join(
583 [('<a href=doc/%s>%s</a>' % (name, name))
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700584 for name in _FindExposedMethods(
585 self, '', unlisted=self._UNLISTED_METHODS)]))
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700586
587 @cherrypy.expose
588 def doc(self, *args):
589 """Shows the documentation for available methods / URLs.
590
591 Example:
592 http://myhost/doc/update
593 """
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700594 name = '/'.join(args)
595 method = _GetExposedMethod(self, name)
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700596 if not method:
597 raise DevServerError("No exposed method named `%s'" % name)
598 if not method.__doc__:
599 raise DevServerError("No documentation for exposed method `%s'" % name)
600 return '<pre>\n%s</pre>' % method.__doc__
Chris Sosa7c931362010-10-11 19:49:01 -0700601
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700602 @cherrypy.expose
Chris Sosa7c931362010-10-11 19:49:01 -0700603 def update(self, *args):
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700604 """Handles an update check from a Chrome OS client.
605
606 The HTTP request should contain the standard Omaha-style XML blob. The URL
607 line may contain an additional intermediate path to the update payload.
608
609 Example:
610 http://myhost/update/optional/path/to/payload
611 """
Chris Sosa7c931362010-10-11 19:49:01 -0700612 label = '/'.join(args)
Gilad Arnold286a0062012-01-12 13:47:02 -0800613 body_length = int(cherrypy.request.headers.get('Content-Length', 0))
Chris Sosa7c931362010-10-11 19:49:01 -0700614 data = cherrypy.request.rfile.read(body_length)
615 return updater.HandleUpdatePing(data, label)
616
Chris Sosa0356d3b2010-09-16 15:46:22 -0700617
Chris Sosadbc20082012-12-10 13:39:11 -0800618def _CleanCache(cache_dir, wipe):
619 """Wipes any excess cached items in the cache_dir.
620
621 Args:
622 cache_dir: the directory we are wiping from.
623 wipe: If True, wipe all the contents -- not just the excess.
624 """
625 if wipe:
626 # Clear the cache and exit on error.
627 cmd = 'rm -rf %s/*' % cache_dir
628 if os.system(cmd) != 0:
629 _Log('Failed to clear the cache with %s' % cmd)
630 sys.exit(1)
631 else:
632 # Clear all but the last N cached updates
633 cmd = ('cd %s; ls -tr | head --lines=-%d | xargs rm -rf' %
634 (cache_dir, CACHED_ENTRIES))
635 if os.system(cmd) != 0:
636 _Log('Failed to clean up old delta cache files with %s' % cmd)
637 sys.exit(1)
638
639
Chris Sosacde6bf42012-05-31 18:36:39 -0700640def main():
Sean O'Connor14b6a0a2010-03-20 23:23:48 -0700641 usage = 'usage: %prog [options]'
Gilad Arnold286a0062012-01-12 13:47:02 -0800642 parser = optparse.OptionParser(usage=usage)
Gilad Arnold9714d9b2012-10-04 10:09:42 -0700643 parser.add_option('--archive_dir',
644 metavar='PATH',
Chris Sosadbc20082012-12-10 13:39:11 -0800645 help='Enables serve-only mode. Serves archived builds only')
Gilad Arnold9714d9b2012-10-04 10:09:42 -0700646 parser.add_option('--board',
647 help='when pre-generating update, board for latest image')
648 parser.add_option('--clear_cache',
Satoru Takabayashid733cbe2011-11-15 09:36:32 -0800649 action='store_true', default=False,
Gilad Arnold9714d9b2012-10-04 10:09:42 -0700650 help='clear out all cached updates and exit')
651 parser.add_option('--critical_update',
652 action='store_true', default=False,
653 help='present update payload as critical')
654 parser.add_option('--data_dir',
655 metavar='PATH',
656 default=os.path.dirname(os.path.abspath(sys.argv[0])),
657 help='writable directory where static lives')
658 parser.add_option('--exit',
659 action='store_true',
660 help='do not start server (yet pregenerate/clear cache)')
Gilad Arnold9714d9b2012-10-04 10:09:42 -0700661 parser.add_option('--for_vm',
662 dest='vm', action='store_true',
663 help='update is for a vm image')
Gilad Arnold8318eac2012-10-04 12:52:23 -0700664 parser.add_option('--host_log',
665 action='store_true', default=False,
666 help='record history of host update events (/api/hostlog)')
Gilad Arnold9714d9b2012-10-04 10:09:42 -0700667 parser.add_option('--image',
668 metavar='FILE',
Chris Sosadbc20082012-12-10 13:39:11 -0800669 help='Force update using this image. Can only be used when '
670 'not in serve-only mode as it is used to generate a '
671 'payload.')
Gilad Arnold9714d9b2012-10-04 10:09:42 -0700672 parser.add_option('--logfile',
673 metavar='PATH',
674 help='log output to this file instead of stdout')
Gilad Arnolda564b4b2012-10-04 10:32:44 -0700675 parser.add_option('--max_updates',
676 metavar='NUM', default=-1, type='int',
677 help='maximum number of update checks handled positively '
678 '(default: unlimited)')
Gilad Arnold9714d9b2012-10-04 10:09:42 -0700679 parser.add_option('-p', '--pregenerate_update',
680 action='store_true', default=False,
Chris Sosadbc20082012-12-10 13:39:11 -0800681 help='pre-generate update payload. Can only be used when '
682 'not in serve-only mode as it is used to generate a '
683 'payload.')
Gilad Arnold9714d9b2012-10-04 10:09:42 -0700684 parser.add_option('--payload',
685 metavar='PATH',
686 help='use update payload from specified directory')
687 parser.add_option('--port',
688 default=8080, type='int',
689 help='port for the dev server to use (default: 8080)')
690 parser.add_option('--private_key',
691 metavar='PATH', default=None,
692 help='path to the private key in pem format')
693 parser.add_option('--production',
694 action='store_true', default=False,
695 help='have the devserver use production values')
696 parser.add_option('--proxy_port',
697 metavar='PORT', default=None, type='int',
698 help='port to have the client connect to (testing support)')
699 parser.add_option('--remote_payload',
700 action='store_true', default=False,
701 help='Payload is being served from a remote machine')
702 parser.add_option('--src_image',
703 metavar='PATH', default='',
704 help='source image for generating delta updates from')
705 parser.add_option('-t', '--test_image',
706 action='store_true',
707 help='whether or not to use test images')
708 parser.add_option('-u', '--urlbase',
709 metavar='URL',
710 help='base URL for update images, other than the devserver')
Chris Sosa7c931362010-10-11 19:49:01 -0700711 (options, _) = parser.parse_args()
rtc@google.com21a5ca32009-11-04 18:23:23 +0000712
Chris Sosa7c931362010-10-11 19:49:01 -0700713 devserver_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
714 root_dir = os.path.realpath('%s/../..' % devserver_dir)
Chris Sosa0356d3b2010-09-16 15:46:22 -0700715 serve_only = False
716
Zdenek Behan608f46c2011-02-19 00:47:16 +0100717 static_dir = os.path.realpath('%s/static' % options.data_dir)
718 os.system('mkdir -p %s' % static_dir)
719
Sean O'Connor14b6a0a2010-03-20 23:23:48 -0700720 if options.archive_dir:
Zdenek Behan608f46c2011-02-19 00:47:16 +0100721 # TODO(zbehan) Remove legacy support:
722 # archive_dir is the directory where static/archive will point.
723 # If this is an absolute path, all is fine. If someone calls this
724 # using a relative path, that is relative to src/platform/dev/.
725 # That use case is unmaintainable, but since applications use it
726 # with =./static, instead of a boolean flag, we'll make this relative
727 # to devserver_dir to keep these unbroken. For now.
728 archive_dir = options.archive_dir
729 if not os.path.isabs(archive_dir):
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700730 archive_dir = os.path.realpath(os.path.join(devserver_dir, archive_dir))
Zdenek Behan608f46c2011-02-19 00:47:16 +0100731 _PrepareToServeUpdatesOnly(archive_dir, static_dir)
Zdenek Behan6d93e552011-03-02 22:35:49 +0100732 static_dir = os.path.realpath(archive_dir)
Chris Sosa0356d3b2010-09-16 15:46:22 -0700733 serve_only = True
Chris Sosa0356d3b2010-09-16 15:46:22 -0700734
Don Garrettf90edf02010-11-16 17:36:14 -0800735 cache_dir = os.path.join(static_dir, 'cache')
Chris Sosadbc20082012-12-10 13:39:11 -0800736 # If our devserver is only supposed to serve payloads, we shouldn't be mucking
737 # with the cache at all. If the devserver hadn't previously generated a cache
738 # and is expected, the caller is using it wrong.
739 if serve_only:
740 # Extra check to make sure we're not being called incorrectly.
741 if (options.clear_cache or options.exit or options.pregenerate_update or
742 options.board or options.image):
743 parser.error('Incompatible flags detected for serve_only mode.')
Don Garrettf90edf02010-11-16 17:36:14 -0800744
Chris Sosadbc20082012-12-10 13:39:11 -0800745 elif os.path.exists(cache_dir):
746 _CleanCache(cache_dir, options.clear_cache)
Chris Sosa6b8c3742011-01-31 12:12:17 -0800747 else:
748 os.makedirs(cache_dir)
Don Garrettf90edf02010-11-16 17:36:14 -0800749
Chris Sosadbc20082012-12-10 13:39:11 -0800750 _Log('Using cache directory %s' % cache_dir)
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700751 _Log('Data dir is %s' % options.data_dir)
752 _Log('Source root is %s' % root_dir)
753 _Log('Serving from %s' % static_dir)
rtc@google.com21a5ca32009-11-04 18:23:23 +0000754
Chris Sosacde6bf42012-05-31 18:36:39 -0700755 global updater
Andrew de los Reyes52620802010-04-12 13:40:07 -0700756 updater = autoupdate.Autoupdate(
757 root_dir=root_dir,
758 static_dir=static_dir,
Chris Sosa0356d3b2010-09-16 15:46:22 -0700759 serve_only=serve_only,
Andrew de los Reyes52620802010-04-12 13:40:07 -0700760 urlbase=options.urlbase,
761 test_image=options.test_image,
Chris Sosa5d342a22010-09-28 16:54:41 -0700762 forced_image=options.image,
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700763 payload_path=options.payload,
Don Garrett0ad09372010-12-06 16:20:30 -0800764 proxy_port=options.proxy_port,
Chris Sosa4136e692010-10-28 23:42:37 -0700765 src_image=options.src_image,
Chris Sosae67b78f2010-11-04 17:33:16 -0700766 vm=options.vm,
Chris Sosa08d55a22011-01-19 16:08:02 -0800767 board=options.board,
Chris Sosa0f1ec842011-02-14 16:33:22 -0800768 copy_to_static_root=not options.exit,
769 private_key=options.private_key,
Satoru Takabayashid733cbe2011-11-15 09:36:32 -0800770 critical_update=options.critical_update,
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700771 remote_payload=options.remote_payload,
Gilad Arnolda564b4b2012-10-04 10:32:44 -0700772 max_updates=options.max_updates,
Gilad Arnold8318eac2012-10-04 12:52:23 -0700773 host_log=options.host_log,
Chris Sosa0f1ec842011-02-14 16:33:22 -0800774 )
Chris Sosa7c931362010-10-11 19:49:01 -0700775
Chris Sosa8dd80092012-12-10 13:39:11 -0800776 if options.pregenerate_update:
777 updater.PreGenerateUpdate()
Chris Sosa0356d3b2010-09-16 15:46:22 -0700778
Don Garrett0c880e22010-11-17 18:13:37 -0800779 # If the command line requested after setup, it's time to do it.
780 if not options.exit:
Chris Sosa66e2d9c2012-07-11 14:14:14 -0700781 # Handle options that must be set globally in cherrypy.
Chris Sosa2f1c41e2012-07-10 14:32:33 -0700782 if options.production:
Chris Sosa66e2d9c2012-07-11 14:14:14 -0700783 cherrypy.config.update({'environment': 'production'})
784 if not options.logfile:
785 cherrypy.config.update({'log.screen': True})
786 else:
787 cherrypy.config.update({'log.error_file': options.logfile,
788 'log.access_file': options.logfile})
Chris Sosa2f1c41e2012-07-10 14:32:33 -0700789
Don Garrett0c880e22010-11-17 18:13:37 -0800790 cherrypy.quickstart(DevServerRoot(), config=_GetConfig(options))
Chris Sosacde6bf42012-05-31 18:36:39 -0700791
792
793if __name__ == '__main__':
794 main()