blob: eb5afc07b457a06294791f5b9eba981a2d271141 [file] [log] [blame]
Chris Sosa7c931362010-10-11 19:49:01 -07001#!/usr/bin/python
2
Chris Sosa781ba6d2012-04-11 12:44:43 -07003# Copyright (c) 2009-2012 The Chromium OS Authors. All rights reserved.
rtc@google.comded22402009-10-26 22:36:21 +00004# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
Chris Sosa7c931362010-10-11 19:49:01 -07007"""A CherryPy-based webserver to host images and build packages."""
8
Chris Sosadbc20082012-12-10 13:39:11 -08009import cherrypy
Gilad Arnold55a2a372012-10-02 09:46:32 -070010import json
Chris Sosa781ba6d2012-04-11 12:44:43 -070011import logging
Sean O'Connor14b6a0a2010-03-20 23:23:48 -070012import optparse
rtc@google.comded22402009-10-26 22:36:21 +000013import os
Scott Zawalski4647ce62012-01-03 17:17:28 -050014import re
Mandeep Singh Baines38dcdda2012-12-07 17:55:33 -080015import socket
chocobo@google.com4dc25812009-10-27 23:46:26 +000016import sys
Chris Masone816e38c2012-05-02 12:22:36 -070017import subprocess
18import tempfile
Gilad Arnold0b8c3f32012-09-19 14:35:44 -070019import threading
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -070020import types
rtc@google.comded22402009-10-26 22:36:21 +000021
Chris Sosa0356d3b2010-09-16 15:46:22 -070022import autoupdate
Gilad Arnoldc65330c2012-09-20 15:17:48 -070023import common_util
Chris Sosa47a7d4e2012-03-28 11:26:55 -070024import downloader
Gilad Arnoldc65330c2012-09-20 15:17:48 -070025import log_util
26
27
28# Module-local log function.
Chris Sosa6a3697f2013-01-29 16:44:43 -080029def _Log(message, *args):
30 return log_util.LogWithTag('DEVSERVER', message, *args)
Chris Sosa0356d3b2010-09-16 15:46:22 -070031
Frank Farzan40160872011-12-12 18:39:18 -080032
Chris Sosa417e55d2011-01-25 16:40:48 -080033CACHED_ENTRIES = 12
Don Garrettf90edf02010-11-16 17:36:14 -080034
Chris Sosa0356d3b2010-09-16 15:46:22 -070035# Sets up global to share between classes.
rtc@google.com21a5ca32009-11-04 18:23:23 +000036updater = None
rtc@google.comded22402009-10-26 22:36:21 +000037
Frank Farzan40160872011-12-12 18:39:18 -080038
Chris Sosa9164ca32012-03-28 11:04:50 -070039class DevServerError(Exception):
Chris Sosa47a7d4e2012-03-28 11:26:55 -070040 """Exception class used by this module."""
41 pass
42
43
Gilad Arnold0b8c3f32012-09-19 14:35:44 -070044class LockDict(object):
45 """A dictionary of locks.
46
47 This class provides a thread-safe store of threading.Lock objects, which can
48 be used to regulate access to any set of hashable resources. Usage:
49
50 foo_lock_dict = LockDict()
51 ...
52 with foo_lock_dict.lock('bar'):
53 # Critical section for 'bar'
54 """
55 def __init__(self):
56 self._lock = self._new_lock()
57 self._dict = {}
58
59 def _new_lock(self):
60 return threading.Lock()
61
62 def lock(self, key):
63 with self._lock:
64 lock = self._dict.get(key)
65 if not lock:
66 lock = self._new_lock()
67 self._dict[key] = lock
68 return lock
69
70
Scott Zawalski4647ce62012-01-03 17:17:28 -050071def _LeadingWhiteSpaceCount(string):
72 """Count the amount of leading whitespace in a string.
73
74 Args:
75 string: The string to count leading whitespace in.
76 Returns:
77 number of white space chars before characters start.
78 """
79 matched = re.match('^\s+', string)
80 if matched:
81 return len(matched.group())
82
83 return 0
84
85
86def _PrintDocStringAsHTML(func):
87 """Make a functions docstring somewhat HTML style.
88
89 Args:
90 func: The function to return the docstring from.
91 Returns:
92 A string that is somewhat formated for a web browser.
93 """
94 # TODO(scottz): Make this parse Args/Returns in a prettier way.
95 # Arguments could be bolded and indented etc.
96 html_doc = []
97 for line in func.__doc__.splitlines():
98 leading_space = _LeadingWhiteSpaceCount(line)
99 if leading_space > 0:
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700100 line = ' ' * leading_space + line
Scott Zawalski4647ce62012-01-03 17:17:28 -0500101
102 html_doc.append('<BR>%s' % line)
103
104 return '\n'.join(html_doc)
105
106
Chris Sosa7c931362010-10-11 19:49:01 -0700107def _GetConfig(options):
108 """Returns the configuration for the devserver."""
Mandeep Singh Baines38dcdda2012-12-07 17:55:33 -0800109
110 # On a system with IPv6 not compiled into the kernel,
111 # AF_INET6 sockets will return a socket.error exception.
112 # On such systems, fall-back to IPv4.
113 socket_host = '::'
114 try:
115 socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
116 except socket.error:
117 socket_host = '0.0.0.0'
118
Chris Sosa7c931362010-10-11 19:49:01 -0700119 base_config = { 'global':
120 { 'server.log_request_headers': True,
121 'server.protocol_version': 'HTTP/1.1',
Mandeep Singh Baines38dcdda2012-12-07 17:55:33 -0800122 'server.socket_host': socket_host,
Chris Sosa7c931362010-10-11 19:49:01 -0700123 'server.socket_port': int(options.port),
Chris Sosa374c62d2010-10-14 09:13:54 -0700124 'response.timeout': 6000,
Chris Sosa6fe23942012-07-02 15:44:46 -0700125 'request.show_tracebacks': True,
Chris Sosa72333d12012-06-13 11:28:05 -0700126 'server.socket_timeout': 60,
Zdenek Behan1347a312011-02-10 03:59:17 +0100127 'tools.staticdir.root':
128 os.path.dirname(os.path.abspath(sys.argv[0])),
Chris Sosa7c931362010-10-11 19:49:01 -0700129 },
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700130 '/api':
131 {
132 # Gets rid of cherrypy parsing post file for args.
133 'request.process_request_body': False,
134 },
Chris Sosaa1ef0102010-10-21 16:22:35 -0700135 '/build':
136 {
137 'response.timeout': 100000,
138 },
Chris Sosa7c931362010-10-11 19:49:01 -0700139 '/update':
140 {
141 # Gets rid of cherrypy parsing post file for args.
142 'request.process_request_body': False,
Chris Sosaf65f4b92010-10-21 15:57:51 -0700143 'response.timeout': 10000,
Chris Sosa7c931362010-10-11 19:49:01 -0700144 },
145 # Sets up the static dir for file hosting.
146 '/static':
147 { 'tools.staticdir.dir': 'static',
148 'tools.staticdir.on': True,
Chris Sosaf65f4b92010-10-21 15:57:51 -0700149 'response.timeout': 10000,
Chris Sosa7c931362010-10-11 19:49:01 -0700150 },
151 }
Chris Sosa5f118ef2012-07-12 11:37:50 -0700152 if options.production:
Chris Sosad1ea86b2012-07-12 13:35:37 -0700153 base_config['global'].update({'server.thread_pool': 75})
Scott Zawalski1c5e7cd2012-02-27 13:12:52 -0500154
Chris Sosa7c931362010-10-11 19:49:01 -0700155 return base_config
rtc@google.com64244662009-11-12 00:52:08 +0000156
Darin Petkove17164a2010-08-11 13:24:41 -0700157
Zdenek Behan608f46c2011-02-19 00:47:16 +0100158def _PrepareToServeUpdatesOnly(image_dir, static_dir):
Chris Sosa0356d3b2010-09-16 15:46:22 -0700159 """Sets up symlink to image_dir for serving purposes."""
160 assert os.path.exists(image_dir), '%s must exist.' % image_dir
161 # If we're serving out of an archived build dir (e.g. a
162 # buildbot), prepare this webserver's magic 'static/' dir with a
163 # link to the build archive.
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700164 _Log('Preparing autoupdate for "serve updates only" mode.')
Zdenek Behan608f46c2011-02-19 00:47:16 +0100165 if os.path.lexists('%s/archive' % static_dir):
166 if image_dir != os.readlink('%s/archive' % static_dir):
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700167 _Log('removing stale symlink to %s' % image_dir)
Zdenek Behan608f46c2011-02-19 00:47:16 +0100168 os.unlink('%s/archive' % static_dir)
169 os.symlink(image_dir, '%s/archive' % static_dir)
Chris Sosacde6bf42012-05-31 18:36:39 -0700170
Chris Sosa0356d3b2010-09-16 15:46:22 -0700171 else:
Zdenek Behan608f46c2011-02-19 00:47:16 +0100172 os.symlink(image_dir, '%s/archive' % static_dir)
Chris Sosacde6bf42012-05-31 18:36:39 -0700173
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700174 _Log('archive dir: %s ready to be used to serve images.' % image_dir)
Chris Sosa7c931362010-10-11 19:49:01 -0700175
176
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700177def _GetRecursiveMemberObject(root, member_list):
178 """Returns an object corresponding to a nested member list.
179
180 Args:
181 root: the root object to search
182 member_list: list of nested members to search
183 Returns:
184 An object corresponding to the member name list; None otherwise.
185 """
186 for member in member_list:
187 next_root = root.__class__.__dict__.get(member)
188 if not next_root:
189 return None
190 root = next_root
191 return root
192
193
194def _IsExposed(name):
195 """Returns True iff |name| has an `exposed' attribute and it is set."""
196 return hasattr(name, 'exposed') and name.exposed
197
198
Gilad Arnold748c8322012-10-12 09:51:35 -0700199def _GetExposedMethod(root, nested_member, ignored=None):
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700200 """Returns a CherryPy-exposed method, if such exists.
201
202 Args:
203 root: the root object for searching
204 nested_member: a slash-joined path to the nested member
205 ignored: method paths to be ignored
206 Returns:
207 A function object corresponding to the path defined by |member_list| from
208 the |root| object, if the function is exposed and not ignored; None
209 otherwise.
210 """
Gilad Arnold748c8322012-10-12 09:51:35 -0700211 method = (not (ignored and nested_member in ignored) and
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700212 _GetRecursiveMemberObject(root, nested_member.split('/')))
213 if (method and type(method) == types.FunctionType and _IsExposed(method)):
214 return method
215
216
Gilad Arnold748c8322012-10-12 09:51:35 -0700217def _FindExposedMethods(root, prefix, unlisted=None):
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700218 """Finds exposed CherryPy methods.
219
220 Args:
221 root: the root object for searching
222 prefix: slash-joined chain of members leading to current object
223 unlisted: URLs to be excluded regardless of their exposed status
224 Returns:
225 List of exposed URLs that are not unlisted.
226 """
227 method_list = []
228 for member in sorted(root.__class__.__dict__.keys()):
229 prefixed_member = prefix + '/' + member if prefix else member
Gilad Arnold748c8322012-10-12 09:51:35 -0700230 if unlisted and prefixed_member in unlisted:
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700231 continue
232 member_obj = root.__class__.__dict__[member]
233 if _IsExposed(member_obj):
234 if type(member_obj) == types.FunctionType:
235 method_list.append(prefixed_member)
236 else:
237 method_list += _FindExposedMethods(
238 member_obj, prefixed_member, unlisted)
239 return method_list
240
241
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700242class ApiRoot(object):
243 """RESTful API for Dev Server information."""
244 exposed = True
245
246 @cherrypy.expose
247 def hostinfo(self, ip):
248 """Returns a JSON dictionary containing information about the given ip.
249
Gilad Arnold1b908392012-10-05 11:36:27 -0700250 Args:
251 ip: address of host whose info is requested
252 Returns:
253 A JSON dictionary containing all or some of the following fields:
254 last_event_type (int): last update event type received
255 last_event_status (int): last update event status received
256 last_known_version (string): last known version reported in update ping
257 forced_update_label (string): update label to force next update ping to
258 use, set by setnextupdate
259 See the OmahaEvent class in update_engine/omaha_request_action.h for
260 event type and status code definitions. If the ip does not exist an empty
261 string is returned.
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700262
Gilad Arnold1b908392012-10-05 11:36:27 -0700263 Example URL:
264 http://myhost/api/hostinfo?ip=192.168.1.5
265 """
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700266 return updater.HandleHostInfoPing(ip)
267
268 @cherrypy.expose
Gilad Arnold286a0062012-01-12 13:47:02 -0800269 def hostlog(self, ip):
Gilad Arnold1b908392012-10-05 11:36:27 -0700270 """Returns a JSON object containing a log of host event.
271
272 Args:
273 ip: address of host whose event log is requested, or `all'
274 Returns:
275 A JSON encoded list (log) of dictionaries (events), each of which
276 containing a `timestamp' and other event fields, as described under
277 /api/hostinfo.
278
279 Example URL:
280 http://myhost/api/hostlog?ip=192.168.1.5
281 """
Gilad Arnold286a0062012-01-12 13:47:02 -0800282 return updater.HandleHostLogPing(ip)
283
284 @cherrypy.expose
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700285 def setnextupdate(self, ip):
286 """Allows the response to the next update ping from a host to be set.
287
288 Takes the IP of the host and an update label as normally provided to the
Gilad Arnold1b908392012-10-05 11:36:27 -0700289 /update command.
290 """
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700291 body_length = int(cherrypy.request.headers['Content-Length'])
292 label = cherrypy.request.rfile.read(body_length)
293
294 if label:
295 label = label.strip()
296 if label:
297 return updater.HandleSetUpdatePing(ip, label)
298 raise cherrypy.HTTPError(400, 'No label provided.')
299
300
Gilad Arnold55a2a372012-10-02 09:46:32 -0700301 @cherrypy.expose
302 def fileinfo(self, *path_args):
303 """Returns information about a given staged file.
304
305 Args:
306 path_args: path to the file inside the server's static staging directory
307 Returns:
308 A JSON encoded dictionary with information about the said file, which may
309 contain the following keys/values:
Gilad Arnold1b908392012-10-05 11:36:27 -0700310 size (int): the file size in bytes
311 sha1 (string): a base64 encoded SHA1 hash
312 sha256 (string): a base64 encoded SHA256 hash
313
314 Example URL:
315 http://myhost/api/fileinfo/some/path/to/file
Gilad Arnold55a2a372012-10-02 09:46:32 -0700316 """
317 file_path = os.path.join(updater.static_dir, *path_args)
318 if not os.path.exists(file_path):
319 raise DevServerError('file not found: %s' % file_path)
320 try:
321 file_size = os.path.getsize(file_path)
322 file_sha1 = common_util.GetFileSha1(file_path)
323 file_sha256 = common_util.GetFileSha256(file_path)
324 except os.error, e:
325 raise DevServerError('failed to get info for file %s: %s' %
326 (file_path, str(e)))
327 return json.dumps(
328 {'size': file_size, 'sha1': file_sha1, 'sha256': file_sha256})
329
David Rochberg7c79a812011-01-19 14:24:45 -0500330class DevServerRoot(object):
Chris Sosa7c931362010-10-11 19:49:01 -0700331 """The Root Class for the Dev Server.
332
333 CherryPy works as follows:
334 For each method in this class, cherrpy interprets root/path
335 as a call to an instance of DevServerRoot->method_name. For example,
336 a call to http://myhost/build will call build. CherryPy automatically
337 parses http args and places them as keyword arguments in each method.
338 For paths http://myhost/update/dir1/dir2, you can use *args so that
339 cherrypy uses the update method and puts the extra paths in args.
340 """
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700341 # Method names that should not be listed on the index page.
342 _UNLISTED_METHODS = ['index', 'doc']
343
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700344 api = ApiRoot()
Chris Sosa7c931362010-10-11 19:49:01 -0700345
David Rochberg7c79a812011-01-19 14:24:45 -0500346 def __init__(self):
Nick Sanders7dcaa2e2011-08-04 15:20:41 -0700347 self._builder = None
Gilad Arnold0b8c3f32012-09-19 14:35:44 -0700348 self._download_lock_dict = LockDict()
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700349 self._downloader_dict = {}
David Rochberg7c79a812011-01-19 14:24:45 -0500350
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700351 @cherrypy.expose
David Rochberg7c79a812011-01-19 14:24:45 -0500352 def build(self, board, pkg, **kwargs):
Chris Sosa7c931362010-10-11 19:49:01 -0700353 """Builds the package specified."""
Nick Sanders7dcaa2e2011-08-04 15:20:41 -0700354 import builder
355 if self._builder is None:
356 self._builder = builder.Builder()
David Rochberg7c79a812011-01-19 14:24:45 -0500357 return self._builder.Build(board, pkg, kwargs)
Chris Sosa7c931362010-10-11 19:49:01 -0700358
Chris Sosacde6bf42012-05-31 18:36:39 -0700359 @staticmethod
360 def _canonicalize_archive_url(archive_url):
361 """Canonicalizes archive_url strings.
362
363 Raises:
364 DevserverError: if archive_url is not set.
365 """
366 if archive_url:
367 return archive_url.rstrip('/')
368 else:
369 raise DevServerError("Must specify an archive_url in the request")
370
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700371 @cherrypy.expose
Frank Farzanbcb571e2012-01-03 11:48:17 -0800372 def download(self, **kwargs):
373 """Downloads and archives full/delta payloads from Google Storage.
374
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700375 This methods downloads artifacts. It may download artifacts in the
376 background in which case a caller should call wait_for_status to get
377 the status of the background artifact downloads. They should use the same
378 args passed to download.
379
Frank Farzanbcb571e2012-01-03 11:48:17 -0800380 Args:
381 archive_url: Google Storage URL for the build.
382
383 Example URL:
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700384 http://myhost/download?archive_url=gs://chromeos-image-archive/
385 x86-generic/R17-1208.0.0-a1-b338
Frank Farzanbcb571e2012-01-03 11:48:17 -0800386 """
Chris Sosacde6bf42012-05-31 18:36:39 -0700387 archive_url = self._canonicalize_archive_url(kwargs.get('archive_url'))
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700388
Chris Sosacde6bf42012-05-31 18:36:39 -0700389 # Guarantees that no two downloads for the same url can run this code
390 # at the same time.
Gilad Arnold0b8c3f32012-09-19 14:35:44 -0700391 with self._download_lock_dict.lock(archive_url):
Chris Sosacde6bf42012-05-31 18:36:39 -0700392 try:
393 # If we are currently downloading, return. Note, due to the above lock
394 # we know that the foreground artifacts must have finished downloading
395 # and returned Success if this downloader instance exists.
396 if (self._downloader_dict.get(archive_url) or
397 downloader.Downloader.BuildStaged(archive_url, updater.static_dir)):
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700398 _Log('Build %s has already been processed.' % archive_url)
Chris Sosacde6bf42012-05-31 18:36:39 -0700399 return 'Success'
400
401 downloader_instance = downloader.Downloader(updater.static_dir)
402 self._downloader_dict[archive_url] = downloader_instance
403 return downloader_instance.Download(archive_url, background=True)
404
405 except:
406 # On any exception, reset the state of the downloader_dict.
407 self._downloader_dict[archive_url] = None
Chris Sosa4d9c4d42012-06-29 15:23:23 -0700408 raise
Chris Sosacde6bf42012-05-31 18:36:39 -0700409
410 @cherrypy.expose
411 def wait_for_status(self, **kwargs):
412 """Waits for background artifacts to be downloaded from Google Storage.
413
414 Args:
415 archive_url: Google Storage URL for the build.
416
417 Example URL:
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700418 http://myhost/wait_for_status?archive_url=gs://chromeos-image-archive/
419 x86-generic/R17-1208.0.0-a1-b338
Chris Sosacde6bf42012-05-31 18:36:39 -0700420 """
421 archive_url = self._canonicalize_archive_url(kwargs.get('archive_url'))
422 downloader_instance = self._downloader_dict.get(archive_url)
423 if downloader_instance:
424 status = downloader_instance.GetStatusOfBackgroundDownloads()
Chris Sosa781ba6d2012-04-11 12:44:43 -0700425 self._downloader_dict[archive_url] = None
Chris Sosacde6bf42012-05-31 18:36:39 -0700426 return status
427 else:
428 # We may have previously downloaded but removed the downloader instance
429 # from the cache.
430 if downloader.Downloader.BuildStaged(archive_url, updater.static_dir):
431 logging.info('%s not found in downloader cache but previously staged.',
432 archive_url)
433 return 'Success'
434 else:
435 raise DevServerError('No download for the given archive_url found.')
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700436
437 @cherrypy.expose
Chris Masone816e38c2012-05-02 12:22:36 -0700438 def stage_debug(self, **kwargs):
439 """Downloads and stages debug symbol payloads from Google Storage.
440
441 This methods downloads the debug symbol build artifact synchronously,
442 and then stages it for use by symbolicate_dump/.
443
444 Args:
445 archive_url: Google Storage URL for the build.
446
447 Example URL:
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700448 http://myhost/stage_debug?archive_url=gs://chromeos-image-archive/
449 x86-generic/R17-1208.0.0-a1-b338
Chris Masone816e38c2012-05-02 12:22:36 -0700450 """
Chris Sosacde6bf42012-05-31 18:36:39 -0700451 archive_url = self._canonicalize_archive_url(kwargs.get('archive_url'))
Chris Masone816e38c2012-05-02 12:22:36 -0700452 return downloader.SymbolDownloader(updater.static_dir).Download(archive_url)
453
454 @cherrypy.expose
455 def symbolicate_dump(self, minidump):
456 """Symbolicates a minidump using pre-downloaded symbols, returns it.
457
458 Callers will need to POST to this URL with a body of MIME-type
459 "multipart/form-data".
460 The body should include a single argument, 'minidump', containing the
461 binary-formatted minidump to symbolicate.
462
463 It is up to the caller to ensure that the symbols they want are currently
464 staged.
465
466 Args:
467 minidump: The binary minidump file to symbolicate.
468 """
469 to_return = ''
470 with tempfile.NamedTemporaryFile() as local:
471 while True:
472 data = minidump.file.read(8192)
473 if not data:
474 break
475 local.write(data)
476 local.flush()
477 stackwalk = subprocess.Popen(['minidump_stackwalk',
478 local.name,
479 updater.static_dir + '/debug/breakpad'],
480 stdout=subprocess.PIPE,
481 stderr=subprocess.PIPE)
482 to_return, error_text = stackwalk.communicate()
483 if stackwalk.returncode != 0:
484 raise DevServerError("Can't generate stack trace: %s (rc=%d)" % (
485 error_text, stackwalk.returncode))
486
487 return to_return
488
489 @cherrypy.expose
Scott Zawalski16954532012-03-20 15:31:36 -0400490 def latestbuild(self, **params):
491 """Return a string representing the latest build for a given target.
492
493 Args:
494 target: The build target, typically a combination of the board and the
495 type of build e.g. x86-mario-release.
496 milestone: The milestone to filter builds on. E.g. R16. Optional, if not
497 provided the latest RXX build will be returned.
498 Returns:
499 A string representation of the latest build if one exists, i.e.
500 R19-1993.0.0-a1-b1480.
501 An empty string if no latest could be found.
502 """
503 if not params:
504 return _PrintDocStringAsHTML(self.latestbuild)
505
506 if 'target' not in params:
507 raise cherrypy.HTTPError('500 Internal Server Error',
508 'Error: target= is required!')
509 try:
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700510 return common_util.GetLatestBuildVersion(
Scott Zawalski16954532012-03-20 15:31:36 -0400511 updater.static_dir, params['target'],
512 milestone=params.get('milestone'))
Gilad Arnold17fe03d2012-10-02 10:05:01 -0700513 except common_util.CommonUtilError as errmsg:
Scott Zawalski16954532012-03-20 15:31:36 -0400514 raise cherrypy.HTTPError('500 Internal Server Error', str(errmsg))
515
516 @cherrypy.expose
Scott Zawalski84a39c92012-01-13 15:12:42 -0500517 def controlfiles(self, **params):
Scott Zawalski4647ce62012-01-03 17:17:28 -0500518 """Return a control file or a list of all known control files.
519
520 Example URL:
521 To List all control files:
Scott Zawalski84a39c92012-01-13 15:12:42 -0500522 http://dev-server/controlfiles?board=x86-alex-release&build=R18-1514.0.0
Scott Zawalski4647ce62012-01-03 17:17:28 -0500523 To return the contents of a path:
Scott Zawalski84a39c92012-01-13 15:12:42 -0500524 http://dev-server/controlfiles?board=x86-alex-release&build=R18-1514.0.0&control_path=client/sleeptest/control
Scott Zawalski4647ce62012-01-03 17:17:28 -0500525
526 Args:
Scott Zawalski84a39c92012-01-13 15:12:42 -0500527 build: The build i.e. x86-alex-release/R18-1514.0.0-a1-b1450.
Scott Zawalski4647ce62012-01-03 17:17:28 -0500528 control_path: If you want the contents of a control file set this
529 to the path. E.g. client/site_tests/sleeptest/control
530 Optional, if not provided return a list of control files is returned.
531 Returns:
532 Contents of a control file if control_path is provided.
533 A list of control files if no control_path is provided.
534 """
Scott Zawalski4647ce62012-01-03 17:17:28 -0500535 if not params:
536 return _PrintDocStringAsHTML(self.controlfiles)
537
Scott Zawalski84a39c92012-01-13 15:12:42 -0500538 if 'build' not in params:
Scott Zawalski4647ce62012-01-03 17:17:28 -0500539 raise cherrypy.HTTPError('500 Internal Server Error',
Scott Zawalski84a39c92012-01-13 15:12:42 -0500540 'Error: build= is required!')
Scott Zawalski4647ce62012-01-03 17:17:28 -0500541
542 if 'control_path' not in params:
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700543 return common_util.GetControlFileList(
544 updater.static_dir, params['build'])
Scott Zawalski4647ce62012-01-03 17:17:28 -0500545 else:
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700546 return common_util.GetControlFile(
547 updater.static_dir, params['build'], params['control_path'])
Frank Farzan40160872011-12-12 18:39:18 -0800548
549 @cherrypy.expose
Gilad Arnold6f99b982012-09-12 10:49:40 -0700550 def stage_images(self, **kwargs):
551 """Downloads and stages a Chrome OS image from Google Storage.
552
553 This method downloads a zipped archive from a specified GS location, then
554 extracts and stages the specified list of images and stages them under
555 static/images/BOARD/BUILD/. Download is synchronous.
556
557 Args:
558 archive_url: Google Storage URL for the build.
559 image_types: comma-separated list of images to download, may include
560 'test', 'recovery', and 'base'
561
562 Example URL:
563 http://myhost/stage_images?archive_url=gs://chromeos-image-archive/
564 x86-generic/R17-1208.0.0-a1-b338&image_types=test,base
565 """
566 # TODO(garnold) This needs to turn into an async operation, to avoid
567 # unnecessary failure of concurrent secondary requests (chromium-os:34661).
568 archive_url = self._canonicalize_archive_url(kwargs.get('archive_url'))
569 image_types = kwargs.get('image_types').split(',')
570 return (downloader.ImagesDownloader(
571 updater.static_dir).Download(archive_url, image_types))
572
573 @cherrypy.expose
Chris Sosa7c931362010-10-11 19:49:01 -0700574 def index(self):
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700575 """Presents a welcome message and documentation links."""
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700576 return ('Welcome to the Dev Server!<br>\n'
577 '<br>\n'
578 'Here are the available methods, click for documentation:<br>\n'
579 '<br>\n'
580 '%s' %
581 '<br>\n'.join(
582 [('<a href=doc/%s>%s</a>' % (name, name))
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700583 for name in _FindExposedMethods(
584 self, '', unlisted=self._UNLISTED_METHODS)]))
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700585
586 @cherrypy.expose
587 def doc(self, *args):
588 """Shows the documentation for available methods / URLs.
589
590 Example:
591 http://myhost/doc/update
592 """
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700593 name = '/'.join(args)
594 method = _GetExposedMethod(self, name)
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700595 if not method:
596 raise DevServerError("No exposed method named `%s'" % name)
597 if not method.__doc__:
598 raise DevServerError("No documentation for exposed method `%s'" % name)
599 return '<pre>\n%s</pre>' % method.__doc__
Chris Sosa7c931362010-10-11 19:49:01 -0700600
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700601 @cherrypy.expose
Chris Sosa7c931362010-10-11 19:49:01 -0700602 def update(self, *args):
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700603 """Handles an update check from a Chrome OS client.
604
605 The HTTP request should contain the standard Omaha-style XML blob. The URL
606 line may contain an additional intermediate path to the update payload.
607
608 Example:
609 http://myhost/update/optional/path/to/payload
610 """
Chris Sosa7c931362010-10-11 19:49:01 -0700611 label = '/'.join(args)
Gilad Arnold286a0062012-01-12 13:47:02 -0800612 body_length = int(cherrypy.request.headers.get('Content-Length', 0))
Chris Sosa7c931362010-10-11 19:49:01 -0700613 data = cherrypy.request.rfile.read(body_length)
614 return updater.HandleUpdatePing(data, label)
615
Chris Sosa0356d3b2010-09-16 15:46:22 -0700616
Chris Sosadbc20082012-12-10 13:39:11 -0800617def _CleanCache(cache_dir, wipe):
618 """Wipes any excess cached items in the cache_dir.
619
620 Args:
621 cache_dir: the directory we are wiping from.
622 wipe: If True, wipe all the contents -- not just the excess.
623 """
624 if wipe:
625 # Clear the cache and exit on error.
626 cmd = 'rm -rf %s/*' % cache_dir
627 if os.system(cmd) != 0:
628 _Log('Failed to clear the cache with %s' % cmd)
629 sys.exit(1)
630 else:
631 # Clear all but the last N cached updates
632 cmd = ('cd %s; ls -tr | head --lines=-%d | xargs rm -rf' %
633 (cache_dir, CACHED_ENTRIES))
634 if os.system(cmd) != 0:
635 _Log('Failed to clean up old delta cache files with %s' % cmd)
636 sys.exit(1)
637
638
Chris Sosacde6bf42012-05-31 18:36:39 -0700639def main():
Sean O'Connor14b6a0a2010-03-20 23:23:48 -0700640 usage = 'usage: %prog [options]'
Gilad Arnold286a0062012-01-12 13:47:02 -0800641 parser = optparse.OptionParser(usage=usage)
Gilad Arnold9714d9b2012-10-04 10:09:42 -0700642 parser.add_option('--archive_dir',
643 metavar='PATH',
Chris Sosadbc20082012-12-10 13:39:11 -0800644 help='Enables serve-only mode. Serves archived builds only')
Gilad Arnold9714d9b2012-10-04 10:09:42 -0700645 parser.add_option('--board',
646 help='when pre-generating update, board for latest image')
647 parser.add_option('--clear_cache',
Satoru Takabayashid733cbe2011-11-15 09:36:32 -0800648 action='store_true', default=False,
Gilad Arnold9714d9b2012-10-04 10:09:42 -0700649 help='clear out all cached updates and exit')
650 parser.add_option('--critical_update',
651 action='store_true', default=False,
652 help='present update payload as critical')
653 parser.add_option('--data_dir',
654 metavar='PATH',
655 default=os.path.dirname(os.path.abspath(sys.argv[0])),
656 help='writable directory where static lives')
657 parser.add_option('--exit',
658 action='store_true',
659 help='do not start server (yet pregenerate/clear cache)')
Gilad Arnold9714d9b2012-10-04 10:09:42 -0700660 parser.add_option('--for_vm',
661 dest='vm', action='store_true',
662 help='update is for a vm image')
Gilad Arnold8318eac2012-10-04 12:52:23 -0700663 parser.add_option('--host_log',
664 action='store_true', default=False,
665 help='record history of host update events (/api/hostlog)')
Gilad Arnold9714d9b2012-10-04 10:09:42 -0700666 parser.add_option('--image',
667 metavar='FILE',
Chris Sosadbc20082012-12-10 13:39:11 -0800668 help='Force update using this image. Can only be used when '
669 'not in serve-only mode as it is used to generate a '
670 'payload.')
Gilad Arnold9714d9b2012-10-04 10:09:42 -0700671 parser.add_option('--logfile',
672 metavar='PATH',
673 help='log output to this file instead of stdout')
Gilad Arnolda564b4b2012-10-04 10:32:44 -0700674 parser.add_option('--max_updates',
675 metavar='NUM', default=-1, type='int',
676 help='maximum number of update checks handled positively '
677 '(default: unlimited)')
Gilad Arnold9714d9b2012-10-04 10:09:42 -0700678 parser.add_option('-p', '--pregenerate_update',
679 action='store_true', default=False,
Chris Sosadbc20082012-12-10 13:39:11 -0800680 help='pre-generate update payload. Can only be used when '
681 'not in serve-only mode as it is used to generate a '
682 'payload.')
Gilad Arnold9714d9b2012-10-04 10:09:42 -0700683 parser.add_option('--payload',
684 metavar='PATH',
685 help='use update payload from specified directory')
686 parser.add_option('--port',
687 default=8080, type='int',
688 help='port for the dev server to use (default: 8080)')
689 parser.add_option('--private_key',
690 metavar='PATH', default=None,
691 help='path to the private key in pem format')
692 parser.add_option('--production',
693 action='store_true', default=False,
694 help='have the devserver use production values')
695 parser.add_option('--proxy_port',
696 metavar='PORT', default=None, type='int',
697 help='port to have the client connect to (testing support)')
698 parser.add_option('--remote_payload',
699 action='store_true', default=False,
700 help='Payload is being served from a remote machine')
701 parser.add_option('--src_image',
702 metavar='PATH', default='',
703 help='source image for generating delta updates from')
704 parser.add_option('-t', '--test_image',
705 action='store_true',
706 help='whether or not to use test images')
707 parser.add_option('-u', '--urlbase',
708 metavar='URL',
709 help='base URL for update images, other than the devserver')
Chris Sosa7c931362010-10-11 19:49:01 -0700710 (options, _) = parser.parse_args()
rtc@google.com21a5ca32009-11-04 18:23:23 +0000711
Chris Sosa7c931362010-10-11 19:49:01 -0700712 devserver_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
713 root_dir = os.path.realpath('%s/../..' % devserver_dir)
Chris Sosa0356d3b2010-09-16 15:46:22 -0700714 serve_only = False
715
Zdenek Behan608f46c2011-02-19 00:47:16 +0100716 static_dir = os.path.realpath('%s/static' % options.data_dir)
717 os.system('mkdir -p %s' % static_dir)
718
Sean O'Connor14b6a0a2010-03-20 23:23:48 -0700719 if options.archive_dir:
Zdenek Behan608f46c2011-02-19 00:47:16 +0100720 # TODO(zbehan) Remove legacy support:
721 # archive_dir is the directory where static/archive will point.
722 # If this is an absolute path, all is fine. If someone calls this
723 # using a relative path, that is relative to src/platform/dev/.
724 # That use case is unmaintainable, but since applications use it
725 # with =./static, instead of a boolean flag, we'll make this relative
726 # to devserver_dir to keep these unbroken. For now.
727 archive_dir = options.archive_dir
728 if not os.path.isabs(archive_dir):
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700729 archive_dir = os.path.realpath(os.path.join(devserver_dir, archive_dir))
Zdenek Behan608f46c2011-02-19 00:47:16 +0100730 _PrepareToServeUpdatesOnly(archive_dir, static_dir)
Zdenek Behan6d93e552011-03-02 22:35:49 +0100731 static_dir = os.path.realpath(archive_dir)
Chris Sosa0356d3b2010-09-16 15:46:22 -0700732 serve_only = True
Chris Sosa0356d3b2010-09-16 15:46:22 -0700733
Don Garrettf90edf02010-11-16 17:36:14 -0800734 cache_dir = os.path.join(static_dir, 'cache')
Chris Sosadbc20082012-12-10 13:39:11 -0800735 # If our devserver is only supposed to serve payloads, we shouldn't be mucking
736 # with the cache at all. If the devserver hadn't previously generated a cache
737 # and is expected, the caller is using it wrong.
738 if serve_only:
739 # Extra check to make sure we're not being called incorrectly.
740 if (options.clear_cache or options.exit or options.pregenerate_update or
741 options.board or options.image):
742 parser.error('Incompatible flags detected for serve_only mode.')
Don Garrettf90edf02010-11-16 17:36:14 -0800743
Chris Sosadbc20082012-12-10 13:39:11 -0800744 elif os.path.exists(cache_dir):
745 _CleanCache(cache_dir, options.clear_cache)
Chris Sosa6b8c3742011-01-31 12:12:17 -0800746 else:
747 os.makedirs(cache_dir)
Don Garrettf90edf02010-11-16 17:36:14 -0800748
Chris Sosadbc20082012-12-10 13:39:11 -0800749 _Log('Using cache directory %s' % cache_dir)
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700750 _Log('Data dir is %s' % options.data_dir)
751 _Log('Source root is %s' % root_dir)
752 _Log('Serving from %s' % static_dir)
rtc@google.com21a5ca32009-11-04 18:23:23 +0000753
Chris Sosa6a3697f2013-01-29 16:44:43 -0800754 # We allow global use here to share with cherrypy classes.
755 # pylint: disable=W0603
Chris Sosacde6bf42012-05-31 18:36:39 -0700756 global updater
Andrew de los Reyes52620802010-04-12 13:40:07 -0700757 updater = autoupdate.Autoupdate(
758 root_dir=root_dir,
759 static_dir=static_dir,
Chris Sosa0356d3b2010-09-16 15:46:22 -0700760 serve_only=serve_only,
Andrew de los Reyes52620802010-04-12 13:40:07 -0700761 urlbase=options.urlbase,
762 test_image=options.test_image,
Chris Sosa5d342a22010-09-28 16:54:41 -0700763 forced_image=options.image,
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700764 payload_path=options.payload,
Don Garrett0ad09372010-12-06 16:20:30 -0800765 proxy_port=options.proxy_port,
Chris Sosa4136e692010-10-28 23:42:37 -0700766 src_image=options.src_image,
Chris Sosae67b78f2010-11-04 17:33:16 -0700767 vm=options.vm,
Chris Sosa08d55a22011-01-19 16:08:02 -0800768 board=options.board,
Chris Sosa0f1ec842011-02-14 16:33:22 -0800769 copy_to_static_root=not options.exit,
770 private_key=options.private_key,
Satoru Takabayashid733cbe2011-11-15 09:36:32 -0800771 critical_update=options.critical_update,
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700772 remote_payload=options.remote_payload,
Gilad Arnolda564b4b2012-10-04 10:32:44 -0700773 max_updates=options.max_updates,
Gilad Arnold8318eac2012-10-04 12:52:23 -0700774 host_log=options.host_log,
Chris Sosa0f1ec842011-02-14 16:33:22 -0800775 )
Chris Sosa7c931362010-10-11 19:49:01 -0700776
Chris Sosa6a3697f2013-01-29 16:44:43 -0800777 if options.pregenerate_update:
778 updater.PreGenerateUpdate()
Chris Sosa0356d3b2010-09-16 15:46:22 -0700779
Don Garrett0c880e22010-11-17 18:13:37 -0800780 # If the command line requested after setup, it's time to do it.
781 if not options.exit:
Chris Sosa66e2d9c2012-07-11 14:14:14 -0700782 # Handle options that must be set globally in cherrypy.
Chris Sosa2f1c41e2012-07-10 14:32:33 -0700783 if options.production:
Chris Sosa66e2d9c2012-07-11 14:14:14 -0700784 cherrypy.config.update({'environment': 'production'})
785 if not options.logfile:
786 cherrypy.config.update({'log.screen': True})
787 else:
788 cherrypy.config.update({'log.error_file': options.logfile,
789 'log.access_file': options.logfile})
Chris Sosa2f1c41e2012-07-10 14:32:33 -0700790
Don Garrett0c880e22010-11-17 18:13:37 -0800791 cherrypy.quickstart(DevServerRoot(), config=_GetConfig(options))
Chris Sosacde6bf42012-05-31 18:36:39 -0700792
793
794if __name__ == '__main__':
795 main()