blob: 5fa105c23908df783c8bdfa14b2fb654b4afe7c7 [file] [log] [blame]
Chris Sosa7c931362010-10-11 19:49:01 -07001#!/usr/bin/python
2
Chris Sosa781ba6d2012-04-11 12:44:43 -07003# Copyright (c) 2009-2012 The Chromium OS Authors. All rights reserved.
rtc@google.comded22402009-10-26 22:36:21 +00004# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
Chris Sosa7c931362010-10-11 19:49:01 -07007"""A CherryPy-based webserver to host images and build packages."""
8
Chris Sosadbc20082012-12-10 13:39:11 -08009import cherrypy
Gilad Arnold55a2a372012-10-02 09:46:32 -070010import json
Sean O'Connor14b6a0a2010-03-20 23:23:48 -070011import optparse
rtc@google.comded22402009-10-26 22:36:21 +000012import os
Scott Zawalski4647ce62012-01-03 17:17:28 -050013import re
Mandeep Singh Baines38dcdda2012-12-07 17:55:33 -080014import socket
chocobo@google.com4dc25812009-10-27 23:46:26 +000015import sys
Chris Masone816e38c2012-05-02 12:22:36 -070016import subprocess
17import tempfile
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -070018import types
rtc@google.comded22402009-10-26 22:36:21 +000019
Chris Sosa0356d3b2010-09-16 15:46:22 -070020import autoupdate
Gilad Arnoldc65330c2012-09-20 15:17:48 -070021import common_util
Chris Sosa47a7d4e2012-03-28 11:26:55 -070022import downloader
Gilad Arnoldc65330c2012-09-20 15:17:48 -070023import log_util
24
25
26# Module-local log function.
Chris Sosa6a3697f2013-01-29 16:44:43 -080027def _Log(message, *args):
28 return log_util.LogWithTag('DEVSERVER', message, *args)
Chris Sosa0356d3b2010-09-16 15:46:22 -070029
Frank Farzan40160872011-12-12 18:39:18 -080030
Chris Sosa417e55d2011-01-25 16:40:48 -080031CACHED_ENTRIES = 12
Don Garrettf90edf02010-11-16 17:36:14 -080032
Chris Sosa0356d3b2010-09-16 15:46:22 -070033# Sets up global to share between classes.
rtc@google.com21a5ca32009-11-04 18:23:23 +000034updater = None
rtc@google.comded22402009-10-26 22:36:21 +000035
Frank Farzan40160872011-12-12 18:39:18 -080036
Chris Sosa9164ca32012-03-28 11:04:50 -070037class DevServerError(Exception):
Chris Sosa47a7d4e2012-03-28 11:26:55 -070038 """Exception class used by this module."""
39 pass
40
41
Scott Zawalski4647ce62012-01-03 17:17:28 -050042def _LeadingWhiteSpaceCount(string):
43 """Count the amount of leading whitespace in a string.
44
45 Args:
46 string: The string to count leading whitespace in.
47 Returns:
48 number of white space chars before characters start.
49 """
50 matched = re.match('^\s+', string)
51 if matched:
52 return len(matched.group())
53
54 return 0
55
56
57def _PrintDocStringAsHTML(func):
58 """Make a functions docstring somewhat HTML style.
59
60 Args:
61 func: The function to return the docstring from.
62 Returns:
63 A string that is somewhat formated for a web browser.
64 """
65 # TODO(scottz): Make this parse Args/Returns in a prettier way.
66 # Arguments could be bolded and indented etc.
67 html_doc = []
68 for line in func.__doc__.splitlines():
69 leading_space = _LeadingWhiteSpaceCount(line)
70 if leading_space > 0:
Chris Sosa47a7d4e2012-03-28 11:26:55 -070071 line = ' ' * leading_space + line
Scott Zawalski4647ce62012-01-03 17:17:28 -050072
73 html_doc.append('<BR>%s' % line)
74
75 return '\n'.join(html_doc)
76
77
Chris Sosa7c931362010-10-11 19:49:01 -070078def _GetConfig(options):
79 """Returns the configuration for the devserver."""
Mandeep Singh Baines38dcdda2012-12-07 17:55:33 -080080
81 # On a system with IPv6 not compiled into the kernel,
82 # AF_INET6 sockets will return a socket.error exception.
83 # On such systems, fall-back to IPv4.
84 socket_host = '::'
85 try:
86 socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
87 except socket.error:
88 socket_host = '0.0.0.0'
89
Chris Sosa7c931362010-10-11 19:49:01 -070090 base_config = { 'global':
91 { 'server.log_request_headers': True,
92 'server.protocol_version': 'HTTP/1.1',
Mandeep Singh Baines38dcdda2012-12-07 17:55:33 -080093 'server.socket_host': socket_host,
Chris Sosa7c931362010-10-11 19:49:01 -070094 'server.socket_port': int(options.port),
Chris Sosa374c62d2010-10-14 09:13:54 -070095 'response.timeout': 6000,
Chris Sosa6fe23942012-07-02 15:44:46 -070096 'request.show_tracebacks': True,
Chris Sosa72333d12012-06-13 11:28:05 -070097 'server.socket_timeout': 60,
Zdenek Behan1347a312011-02-10 03:59:17 +010098 'tools.staticdir.root':
99 os.path.dirname(os.path.abspath(sys.argv[0])),
Chris Sosa7c931362010-10-11 19:49:01 -0700100 },
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700101 '/api':
102 {
103 # Gets rid of cherrypy parsing post file for args.
104 'request.process_request_body': False,
105 },
Chris Sosaa1ef0102010-10-21 16:22:35 -0700106 '/build':
107 {
108 'response.timeout': 100000,
109 },
Chris Sosa7c931362010-10-11 19:49:01 -0700110 '/update':
111 {
112 # Gets rid of cherrypy parsing post file for args.
113 'request.process_request_body': False,
Chris Sosaf65f4b92010-10-21 15:57:51 -0700114 'response.timeout': 10000,
Chris Sosa7c931362010-10-11 19:49:01 -0700115 },
116 # Sets up the static dir for file hosting.
117 '/static':
118 { 'tools.staticdir.dir': 'static',
119 'tools.staticdir.on': True,
Chris Sosaf65f4b92010-10-21 15:57:51 -0700120 'response.timeout': 10000,
Chris Sosa7c931362010-10-11 19:49:01 -0700121 },
122 }
Chris Sosa5f118ef2012-07-12 11:37:50 -0700123 if options.production:
Chris Sosad1ea86b2012-07-12 13:35:37 -0700124 base_config['global'].update({'server.thread_pool': 75})
Scott Zawalski1c5e7cd2012-02-27 13:12:52 -0500125
Chris Sosa7c931362010-10-11 19:49:01 -0700126 return base_config
rtc@google.com64244662009-11-12 00:52:08 +0000127
Darin Petkove17164a2010-08-11 13:24:41 -0700128
Zdenek Behan608f46c2011-02-19 00:47:16 +0100129def _PrepareToServeUpdatesOnly(image_dir, static_dir):
Chris Sosa0356d3b2010-09-16 15:46:22 -0700130 """Sets up symlink to image_dir for serving purposes."""
131 assert os.path.exists(image_dir), '%s must exist.' % image_dir
132 # If we're serving out of an archived build dir (e.g. a
133 # buildbot), prepare this webserver's magic 'static/' dir with a
134 # link to the build archive.
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700135 _Log('Preparing autoupdate for "serve updates only" mode.')
Zdenek Behan608f46c2011-02-19 00:47:16 +0100136 if os.path.lexists('%s/archive' % static_dir):
137 if image_dir != os.readlink('%s/archive' % static_dir):
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700138 _Log('removing stale symlink to %s' % image_dir)
Zdenek Behan608f46c2011-02-19 00:47:16 +0100139 os.unlink('%s/archive' % static_dir)
140 os.symlink(image_dir, '%s/archive' % static_dir)
Chris Sosacde6bf42012-05-31 18:36:39 -0700141
Chris Sosa0356d3b2010-09-16 15:46:22 -0700142 else:
Zdenek Behan608f46c2011-02-19 00:47:16 +0100143 os.symlink(image_dir, '%s/archive' % static_dir)
Chris Sosacde6bf42012-05-31 18:36:39 -0700144
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700145 _Log('archive dir: %s ready to be used to serve images.' % image_dir)
Chris Sosa7c931362010-10-11 19:49:01 -0700146
147
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700148def _GetRecursiveMemberObject(root, member_list):
149 """Returns an object corresponding to a nested member list.
150
151 Args:
152 root: the root object to search
153 member_list: list of nested members to search
154 Returns:
155 An object corresponding to the member name list; None otherwise.
156 """
157 for member in member_list:
158 next_root = root.__class__.__dict__.get(member)
159 if not next_root:
160 return None
161 root = next_root
162 return root
163
164
165def _IsExposed(name):
166 """Returns True iff |name| has an `exposed' attribute and it is set."""
167 return hasattr(name, 'exposed') and name.exposed
168
169
Gilad Arnold748c8322012-10-12 09:51:35 -0700170def _GetExposedMethod(root, nested_member, ignored=None):
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700171 """Returns a CherryPy-exposed method, if such exists.
172
173 Args:
174 root: the root object for searching
175 nested_member: a slash-joined path to the nested member
176 ignored: method paths to be ignored
177 Returns:
178 A function object corresponding to the path defined by |member_list| from
179 the |root| object, if the function is exposed and not ignored; None
180 otherwise.
181 """
Gilad Arnold748c8322012-10-12 09:51:35 -0700182 method = (not (ignored and nested_member in ignored) and
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700183 _GetRecursiveMemberObject(root, nested_member.split('/')))
184 if (method and type(method) == types.FunctionType and _IsExposed(method)):
185 return method
186
187
Gilad Arnold748c8322012-10-12 09:51:35 -0700188def _FindExposedMethods(root, prefix, unlisted=None):
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700189 """Finds exposed CherryPy methods.
190
191 Args:
192 root: the root object for searching
193 prefix: slash-joined chain of members leading to current object
194 unlisted: URLs to be excluded regardless of their exposed status
195 Returns:
196 List of exposed URLs that are not unlisted.
197 """
198 method_list = []
199 for member in sorted(root.__class__.__dict__.keys()):
200 prefixed_member = prefix + '/' + member if prefix else member
Gilad Arnold748c8322012-10-12 09:51:35 -0700201 if unlisted and prefixed_member in unlisted:
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700202 continue
203 member_obj = root.__class__.__dict__[member]
204 if _IsExposed(member_obj):
205 if type(member_obj) == types.FunctionType:
206 method_list.append(prefixed_member)
207 else:
208 method_list += _FindExposedMethods(
209 member_obj, prefixed_member, unlisted)
210 return method_list
211
212
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700213class ApiRoot(object):
214 """RESTful API for Dev Server information."""
215 exposed = True
216
217 @cherrypy.expose
218 def hostinfo(self, ip):
219 """Returns a JSON dictionary containing information about the given ip.
220
Gilad Arnold1b908392012-10-05 11:36:27 -0700221 Args:
222 ip: address of host whose info is requested
223 Returns:
224 A JSON dictionary containing all or some of the following fields:
225 last_event_type (int): last update event type received
226 last_event_status (int): last update event status received
227 last_known_version (string): last known version reported in update ping
228 forced_update_label (string): update label to force next update ping to
229 use, set by setnextupdate
230 See the OmahaEvent class in update_engine/omaha_request_action.h for
231 event type and status code definitions. If the ip does not exist an empty
232 string is returned.
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700233
Gilad Arnold1b908392012-10-05 11:36:27 -0700234 Example URL:
235 http://myhost/api/hostinfo?ip=192.168.1.5
236 """
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700237 return updater.HandleHostInfoPing(ip)
238
239 @cherrypy.expose
Gilad Arnold286a0062012-01-12 13:47:02 -0800240 def hostlog(self, ip):
Gilad Arnold1b908392012-10-05 11:36:27 -0700241 """Returns a JSON object containing a log of host event.
242
243 Args:
244 ip: address of host whose event log is requested, or `all'
245 Returns:
246 A JSON encoded list (log) of dictionaries (events), each of which
247 containing a `timestamp' and other event fields, as described under
248 /api/hostinfo.
249
250 Example URL:
251 http://myhost/api/hostlog?ip=192.168.1.5
252 """
Gilad Arnold286a0062012-01-12 13:47:02 -0800253 return updater.HandleHostLogPing(ip)
254
255 @cherrypy.expose
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700256 def setnextupdate(self, ip):
257 """Allows the response to the next update ping from a host to be set.
258
259 Takes the IP of the host and an update label as normally provided to the
Gilad Arnold1b908392012-10-05 11:36:27 -0700260 /update command.
261 """
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700262 body_length = int(cherrypy.request.headers['Content-Length'])
263 label = cherrypy.request.rfile.read(body_length)
264
265 if label:
266 label = label.strip()
267 if label:
268 return updater.HandleSetUpdatePing(ip, label)
269 raise cherrypy.HTTPError(400, 'No label provided.')
270
271
Gilad Arnold55a2a372012-10-02 09:46:32 -0700272 @cherrypy.expose
273 def fileinfo(self, *path_args):
274 """Returns information about a given staged file.
275
276 Args:
277 path_args: path to the file inside the server's static staging directory
278 Returns:
279 A JSON encoded dictionary with information about the said file, which may
280 contain the following keys/values:
Gilad Arnold1b908392012-10-05 11:36:27 -0700281 size (int): the file size in bytes
282 sha1 (string): a base64 encoded SHA1 hash
283 sha256 (string): a base64 encoded SHA256 hash
284
285 Example URL:
286 http://myhost/api/fileinfo/some/path/to/file
Gilad Arnold55a2a372012-10-02 09:46:32 -0700287 """
288 file_path = os.path.join(updater.static_dir, *path_args)
289 if not os.path.exists(file_path):
290 raise DevServerError('file not found: %s' % file_path)
291 try:
292 file_size = os.path.getsize(file_path)
293 file_sha1 = common_util.GetFileSha1(file_path)
294 file_sha256 = common_util.GetFileSha256(file_path)
295 except os.error, e:
296 raise DevServerError('failed to get info for file %s: %s' %
297 (file_path, str(e)))
298 return json.dumps(
299 {'size': file_size, 'sha1': file_sha1, 'sha256': file_sha256})
300
Chris Sosa76e44b92013-01-31 12:11:38 -0800301
David Rochberg7c79a812011-01-19 14:24:45 -0500302class DevServerRoot(object):
Chris Sosa7c931362010-10-11 19:49:01 -0700303 """The Root Class for the Dev Server.
304
305 CherryPy works as follows:
306 For each method in this class, cherrpy interprets root/path
307 as a call to an instance of DevServerRoot->method_name. For example,
308 a call to http://myhost/build will call build. CherryPy automatically
309 parses http args and places them as keyword arguments in each method.
310 For paths http://myhost/update/dir1/dir2, you can use *args so that
311 cherrypy uses the update method and puts the extra paths in args.
312 """
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700313 # Method names that should not be listed on the index page.
314 _UNLISTED_METHODS = ['index', 'doc']
315
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700316 api = ApiRoot()
Chris Sosa7c931362010-10-11 19:49:01 -0700317
David Rochberg7c79a812011-01-19 14:24:45 -0500318 def __init__(self):
Nick Sanders7dcaa2e2011-08-04 15:20:41 -0700319 self._builder = None
Chris Sosa76e44b92013-01-31 12:11:38 -0800320 self._download_lock_dict = common_util.LockDict()
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700321 self._downloader_dict = {}
David Rochberg7c79a812011-01-19 14:24:45 -0500322
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700323 @cherrypy.expose
David Rochberg7c79a812011-01-19 14:24:45 -0500324 def build(self, board, pkg, **kwargs):
Chris Sosa7c931362010-10-11 19:49:01 -0700325 """Builds the package specified."""
Nick Sanders7dcaa2e2011-08-04 15:20:41 -0700326 import builder
327 if self._builder is None:
328 self._builder = builder.Builder()
David Rochberg7c79a812011-01-19 14:24:45 -0500329 return self._builder.Build(board, pkg, kwargs)
Chris Sosa7c931362010-10-11 19:49:01 -0700330
Chris Sosacde6bf42012-05-31 18:36:39 -0700331 @staticmethod
332 def _canonicalize_archive_url(archive_url):
333 """Canonicalizes archive_url strings.
334
335 Raises:
336 DevserverError: if archive_url is not set.
337 """
338 if archive_url:
Chris Sosa76e44b92013-01-31 12:11:38 -0800339 if not archive_url.startswith('gs://'):
340 raise DevServerError("Archive URL isn't from Google Storage.")
341
Chris Sosacde6bf42012-05-31 18:36:39 -0700342 return archive_url.rstrip('/')
343 else:
344 raise DevServerError("Must specify an archive_url in the request")
345
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700346 @cherrypy.expose
Frank Farzanbcb571e2012-01-03 11:48:17 -0800347 def download(self, **kwargs):
348 """Downloads and archives full/delta payloads from Google Storage.
349
Chris Sosa76e44b92013-01-31 12:11:38 -0800350 THIS METHOD IS DEPRECATED: use stage(..., artifacts=...) instead.
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700351 This methods downloads artifacts. It may download artifacts in the
352 background in which case a caller should call wait_for_status to get
353 the status of the background artifact downloads. They should use the same
354 args passed to download.
355
Frank Farzanbcb571e2012-01-03 11:48:17 -0800356 Args:
357 archive_url: Google Storage URL for the build.
358
359 Example URL:
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700360 http://myhost/download?archive_url=gs://chromeos-image-archive/
361 x86-generic/R17-1208.0.0-a1-b338
Frank Farzanbcb571e2012-01-03 11:48:17 -0800362 """
Chris Sosa76e44b92013-01-31 12:11:38 -0800363 return self.stage(archive_url=kwargs.get('archive_url'),
364 artifacts='full_payload,test_suites,stateful')
365
366 @cherrypy.expose
367 def stage(self, **kwargs):
368 """Downloads and caches the artifacts from Google Storage URL.
369
370 Downloads and caches the artifacts Google Storage URL. Returns once these
371 have been downloaded on the devserver. A call to this will attempt to cache
372 non-specified artifacts in the background for the given from the given URL
373 following the principle of spatial locality. Spatial locality of different
374 artifacts is explicitly defined in the build_artifact module.
375
376 These artifacts will then be available from the static/ sub-directory of
377 the devserver.
378
379 Args:
380 archive_url: Google Storage URL for the build.
381 artifacts: Comma separated list of artifacts to download.
382
383 Example:
384 To download the autotest and test suites tarballs:
385 http://devserver_url:<port>/stage?archive_url=gs://your_url/path&
386 artifacts=autotest,test_suites
387 To download the full update payload:
388 http://devserver_url:<port>/stage?archive_url=gs://your_url/path&
389 artifacts=full_payload
390
391 For both these examples, one could find these artifacts at:
392 http://devserver_url:<port>/static/archive/<relative_path>*
393
394 Note for this example, relative path is the archive_url stripped of its
395 basename i.e. path/ in the examples above. Specific example:
396
397 gs://chromeos-image-archive/x86-mario-release/R26-3920.0.0
398
399 Will get staged to:
400
401 http://devserver_url:<port>/static/archive/x86-mario-release/R26-3920.0.0
402 """
Chris Sosacde6bf42012-05-31 18:36:39 -0700403 archive_url = self._canonicalize_archive_url(kwargs.get('archive_url'))
Chris Sosa76e44b92013-01-31 12:11:38 -0800404 artifacts = kwargs.get('artifacts', '')
405 if not artifacts:
406 raise DevServerError('No artifacts specified.')
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700407
Chris Sosa76e44b92013-01-31 12:11:38 -0800408 downloader.Downloader(updater.static_dir, archive_url).Download(
409 artifacts.split(','))
410 return 'Success'
Chris Sosacde6bf42012-05-31 18:36:39 -0700411
412 @cherrypy.expose
413 def wait_for_status(self, **kwargs):
414 """Waits for background artifacts to be downloaded from Google Storage.
415
Chris Sosa76e44b92013-01-31 12:11:38 -0800416 THIS METHOD IS DEPRECATED: use stage(..., artifacts=...) instead.
Chris Sosacde6bf42012-05-31 18:36:39 -0700417 Args:
418 archive_url: Google Storage URL for the build.
419
420 Example URL:
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700421 http://myhost/wait_for_status?archive_url=gs://chromeos-image-archive/
422 x86-generic/R17-1208.0.0-a1-b338
Chris Sosacde6bf42012-05-31 18:36:39 -0700423 """
Chris Sosa76e44b92013-01-31 12:11:38 -0800424 return self.stage(archive_url=kwargs.get('archive_url'),
425 artifacts='full_payload,test_suites,autotest,stateful')
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700426
427 @cherrypy.expose
Chris Masone816e38c2012-05-02 12:22:36 -0700428 def stage_debug(self, **kwargs):
429 """Downloads and stages debug symbol payloads from Google Storage.
430
Chris Sosa76e44b92013-01-31 12:11:38 -0800431 THIS METHOD IS DEPRECATED: use stage(..., artifacts=...) instead.
432 This methods downloads the debug symbol build artifact
433 synchronously, and then stages it for use by symbolicate_dump.
Chris Masone816e38c2012-05-02 12:22:36 -0700434
435 Args:
436 archive_url: Google Storage URL for the build.
437
438 Example URL:
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700439 http://myhost/stage_debug?archive_url=gs://chromeos-image-archive/
440 x86-generic/R17-1208.0.0-a1-b338
Chris Masone816e38c2012-05-02 12:22:36 -0700441 """
Chris Sosa76e44b92013-01-31 12:11:38 -0800442 return self.stage(archive_url=kwargs.get('archive_url'),
443 artifacts='symbols')
Chris Masone816e38c2012-05-02 12:22:36 -0700444
445 @cherrypy.expose
Chris Sosa76e44b92013-01-31 12:11:38 -0800446 def symbolicate_dump(self, minidump, **kwargs):
Chris Masone816e38c2012-05-02 12:22:36 -0700447 """Symbolicates a minidump using pre-downloaded symbols, returns it.
448
449 Callers will need to POST to this URL with a body of MIME-type
450 "multipart/form-data".
451 The body should include a single argument, 'minidump', containing the
452 binary-formatted minidump to symbolicate.
453
Chris Masone816e38c2012-05-02 12:22:36 -0700454 Args:
Chris Sosa76e44b92013-01-31 12:11:38 -0800455 archive_url: Google Storage URL for the build.
Chris Masone816e38c2012-05-02 12:22:36 -0700456 minidump: The binary minidump file to symbolicate.
457 """
Chris Sosa76e44b92013-01-31 12:11:38 -0800458 # Ensure the symbols have been staged.
459 archive_url = self._canonicalize_archive_url(kwargs.get('archive_url'))
460 if self.stage(archive_url=archive_url, artifacts='symbols') != 'Success':
461 raise DevServerError('Failed to stage symbols for %s' % archive_url)
462
Chris Masone816e38c2012-05-02 12:22:36 -0700463 to_return = ''
464 with tempfile.NamedTemporaryFile() as local:
465 while True:
466 data = minidump.file.read(8192)
467 if not data:
468 break
469 local.write(data)
Chris Sosa76e44b92013-01-31 12:11:38 -0800470
Chris Masone816e38c2012-05-02 12:22:36 -0700471 local.flush()
Chris Sosa76e44b92013-01-31 12:11:38 -0800472
473 symbols_directory = os.path.join(downloader.Downloader.GetBuildDir(
474 updater.static_dir, archive_url), 'debug', 'breakpad')
475
476 stackwalk = subprocess.Popen(
477 ['minidump_stackwalk', local.name, symbols_directory],
478 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
479
Chris Masone816e38c2012-05-02 12:22:36 -0700480 to_return, error_text = stackwalk.communicate()
481 if stackwalk.returncode != 0:
482 raise DevServerError("Can't generate stack trace: %s (rc=%d)" % (
483 error_text, stackwalk.returncode))
484
485 return to_return
486
487 @cherrypy.expose
Scott Zawalski16954532012-03-20 15:31:36 -0400488 def latestbuild(self, **params):
489 """Return a string representing the latest build for a given target.
490
491 Args:
492 target: The build target, typically a combination of the board and the
493 type of build e.g. x86-mario-release.
494 milestone: The milestone to filter builds on. E.g. R16. Optional, if not
495 provided the latest RXX build will be returned.
496 Returns:
497 A string representation of the latest build if one exists, i.e.
498 R19-1993.0.0-a1-b1480.
499 An empty string if no latest could be found.
500 """
501 if not params:
502 return _PrintDocStringAsHTML(self.latestbuild)
503
504 if 'target' not in params:
505 raise cherrypy.HTTPError('500 Internal Server Error',
506 'Error: target= is required!')
507 try:
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700508 return common_util.GetLatestBuildVersion(
Scott Zawalski16954532012-03-20 15:31:36 -0400509 updater.static_dir, params['target'],
510 milestone=params.get('milestone'))
Gilad Arnold17fe03d2012-10-02 10:05:01 -0700511 except common_util.CommonUtilError as errmsg:
Scott Zawalski16954532012-03-20 15:31:36 -0400512 raise cherrypy.HTTPError('500 Internal Server Error', str(errmsg))
513
514 @cherrypy.expose
Scott Zawalski84a39c92012-01-13 15:12:42 -0500515 def controlfiles(self, **params):
Scott Zawalski4647ce62012-01-03 17:17:28 -0500516 """Return a control file or a list of all known control files.
517
518 Example URL:
519 To List all control files:
Scott Zawalski84a39c92012-01-13 15:12:42 -0500520 http://dev-server/controlfiles?board=x86-alex-release&build=R18-1514.0.0
Scott Zawalski4647ce62012-01-03 17:17:28 -0500521 To return the contents of a path:
Scott Zawalski84a39c92012-01-13 15:12:42 -0500522 http://dev-server/controlfiles?board=x86-alex-release&build=R18-1514.0.0&control_path=client/sleeptest/control
Scott Zawalski4647ce62012-01-03 17:17:28 -0500523
524 Args:
Scott Zawalski84a39c92012-01-13 15:12:42 -0500525 build: The build i.e. x86-alex-release/R18-1514.0.0-a1-b1450.
Scott Zawalski4647ce62012-01-03 17:17:28 -0500526 control_path: If you want the contents of a control file set this
527 to the path. E.g. client/site_tests/sleeptest/control
528 Optional, if not provided return a list of control files is returned.
529 Returns:
530 Contents of a control file if control_path is provided.
531 A list of control files if no control_path is provided.
532 """
Scott Zawalski4647ce62012-01-03 17:17:28 -0500533 if not params:
534 return _PrintDocStringAsHTML(self.controlfiles)
535
Scott Zawalski84a39c92012-01-13 15:12:42 -0500536 if 'build' not in params:
Scott Zawalski4647ce62012-01-03 17:17:28 -0500537 raise cherrypy.HTTPError('500 Internal Server Error',
Scott Zawalski84a39c92012-01-13 15:12:42 -0500538 'Error: build= is required!')
Scott Zawalski4647ce62012-01-03 17:17:28 -0500539
540 if 'control_path' not in params:
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700541 return common_util.GetControlFileList(
542 updater.static_dir, params['build'])
Scott Zawalski4647ce62012-01-03 17:17:28 -0500543 else:
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700544 return common_util.GetControlFile(
545 updater.static_dir, params['build'], params['control_path'])
Frank Farzan40160872011-12-12 18:39:18 -0800546
547 @cherrypy.expose
Gilad Arnold6f99b982012-09-12 10:49:40 -0700548 def stage_images(self, **kwargs):
549 """Downloads and stages a Chrome OS image from Google Storage.
550
Chris Sosa76e44b92013-01-31 12:11:38 -0800551 THIS METHOD IS DEPRECATED: use stage(..., artifacts=...) instead.
Gilad Arnold6f99b982012-09-12 10:49:40 -0700552 This method downloads a zipped archive from a specified GS location, then
553 extracts and stages the specified list of images and stages them under
Chris Sosa76e44b92013-01-31 12:11:38 -0800554 static/BOARD/BUILD/. Download is synchronous.
Gilad Arnold6f99b982012-09-12 10:49:40 -0700555
556 Args:
557 archive_url: Google Storage URL for the build.
558 image_types: comma-separated list of images to download, may include
559 'test', 'recovery', and 'base'
560
561 Example URL:
562 http://myhost/stage_images?archive_url=gs://chromeos-image-archive/
563 x86-generic/R17-1208.0.0-a1-b338&image_types=test,base
564 """
Gilad Arnold6f99b982012-09-12 10:49:40 -0700565 image_types = kwargs.get('image_types').split(',')
Chris Sosa76e44b92013-01-31 12:11:38 -0800566 image_types_list = [image + '_image' for image in image_types]
567 self.stage(archive_url=kwargs.get('archive_url'), artifacts=','.join(
568 image_types_list))
Gilad Arnold6f99b982012-09-12 10:49:40 -0700569
570 @cherrypy.expose
Chris Sosa7c931362010-10-11 19:49:01 -0700571 def index(self):
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700572 """Presents a welcome message and documentation links."""
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700573 return ('Welcome to the Dev Server!<br>\n'
574 '<br>\n'
575 'Here are the available methods, click for documentation:<br>\n'
576 '<br>\n'
577 '%s' %
578 '<br>\n'.join(
579 [('<a href=doc/%s>%s</a>' % (name, name))
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700580 for name in _FindExposedMethods(
581 self, '', unlisted=self._UNLISTED_METHODS)]))
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700582
583 @cherrypy.expose
584 def doc(self, *args):
585 """Shows the documentation for available methods / URLs.
586
587 Example:
588 http://myhost/doc/update
589 """
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700590 name = '/'.join(args)
591 method = _GetExposedMethod(self, name)
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700592 if not method:
593 raise DevServerError("No exposed method named `%s'" % name)
594 if not method.__doc__:
595 raise DevServerError("No documentation for exposed method `%s'" % name)
596 return '<pre>\n%s</pre>' % method.__doc__
Chris Sosa7c931362010-10-11 19:49:01 -0700597
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700598 @cherrypy.expose
Chris Sosa7c931362010-10-11 19:49:01 -0700599 def update(self, *args):
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700600 """Handles an update check from a Chrome OS client.
601
602 The HTTP request should contain the standard Omaha-style XML blob. The URL
603 line may contain an additional intermediate path to the update payload.
604
605 Example:
606 http://myhost/update/optional/path/to/payload
607 """
Chris Sosa7c931362010-10-11 19:49:01 -0700608 label = '/'.join(args)
Gilad Arnold286a0062012-01-12 13:47:02 -0800609 body_length = int(cherrypy.request.headers.get('Content-Length', 0))
Chris Sosa7c931362010-10-11 19:49:01 -0700610 data = cherrypy.request.rfile.read(body_length)
611 return updater.HandleUpdatePing(data, label)
612
Chris Sosa0356d3b2010-09-16 15:46:22 -0700613
Chris Sosadbc20082012-12-10 13:39:11 -0800614def _CleanCache(cache_dir, wipe):
615 """Wipes any excess cached items in the cache_dir.
616
617 Args:
618 cache_dir: the directory we are wiping from.
619 wipe: If True, wipe all the contents -- not just the excess.
620 """
621 if wipe:
622 # Clear the cache and exit on error.
623 cmd = 'rm -rf %s/*' % cache_dir
624 if os.system(cmd) != 0:
625 _Log('Failed to clear the cache with %s' % cmd)
626 sys.exit(1)
627 else:
628 # Clear all but the last N cached updates
629 cmd = ('cd %s; ls -tr | head --lines=-%d | xargs rm -rf' %
630 (cache_dir, CACHED_ENTRIES))
631 if os.system(cmd) != 0:
632 _Log('Failed to clean up old delta cache files with %s' % cmd)
633 sys.exit(1)
634
635
Chris Sosacde6bf42012-05-31 18:36:39 -0700636def main():
Sean O'Connor14b6a0a2010-03-20 23:23:48 -0700637 usage = 'usage: %prog [options]'
Gilad Arnold286a0062012-01-12 13:47:02 -0800638 parser = optparse.OptionParser(usage=usage)
Gilad Arnold9714d9b2012-10-04 10:09:42 -0700639 parser.add_option('--archive_dir',
640 metavar='PATH',
Chris Sosadbc20082012-12-10 13:39:11 -0800641 help='Enables serve-only mode. Serves archived builds only')
Gilad Arnold9714d9b2012-10-04 10:09:42 -0700642 parser.add_option('--board',
643 help='when pre-generating update, board for latest image')
644 parser.add_option('--clear_cache',
Satoru Takabayashid733cbe2011-11-15 09:36:32 -0800645 action='store_true', default=False,
Gilad Arnold9714d9b2012-10-04 10:09:42 -0700646 help='clear out all cached updates and exit')
647 parser.add_option('--critical_update',
648 action='store_true', default=False,
649 help='present update payload as critical')
650 parser.add_option('--data_dir',
651 metavar='PATH',
652 default=os.path.dirname(os.path.abspath(sys.argv[0])),
653 help='writable directory where static lives')
654 parser.add_option('--exit',
655 action='store_true',
656 help='do not start server (yet pregenerate/clear cache)')
Gilad Arnold9714d9b2012-10-04 10:09:42 -0700657 parser.add_option('--for_vm',
658 dest='vm', action='store_true',
659 help='update is for a vm image')
Gilad Arnold8318eac2012-10-04 12:52:23 -0700660 parser.add_option('--host_log',
661 action='store_true', default=False,
662 help='record history of host update events (/api/hostlog)')
Gilad Arnold9714d9b2012-10-04 10:09:42 -0700663 parser.add_option('--image',
664 metavar='FILE',
Chris Sosadbc20082012-12-10 13:39:11 -0800665 help='Force update using this image. Can only be used when '
666 'not in serve-only mode as it is used to generate a '
667 'payload.')
Gilad Arnold9714d9b2012-10-04 10:09:42 -0700668 parser.add_option('--logfile',
669 metavar='PATH',
670 help='log output to this file instead of stdout')
Gilad Arnolda564b4b2012-10-04 10:32:44 -0700671 parser.add_option('--max_updates',
Chris Sosa76e44b92013-01-31 12:11:38 -0800672 metavar='NUM', default= -1, type='int',
Gilad Arnolda564b4b2012-10-04 10:32:44 -0700673 help='maximum number of update checks handled positively '
674 '(default: unlimited)')
Gilad Arnold9714d9b2012-10-04 10:09:42 -0700675 parser.add_option('-p', '--pregenerate_update',
676 action='store_true', default=False,
Chris Sosadbc20082012-12-10 13:39:11 -0800677 help='pre-generate update payload. Can only be used when '
678 'not in serve-only mode as it is used to generate a '
679 'payload.')
Gilad Arnold9714d9b2012-10-04 10:09:42 -0700680 parser.add_option('--payload',
681 metavar='PATH',
682 help='use update payload from specified directory')
683 parser.add_option('--port',
684 default=8080, type='int',
685 help='port for the dev server to use (default: 8080)')
686 parser.add_option('--private_key',
687 metavar='PATH', default=None,
688 help='path to the private key in pem format')
689 parser.add_option('--production',
690 action='store_true', default=False,
691 help='have the devserver use production values')
692 parser.add_option('--proxy_port',
693 metavar='PORT', default=None, type='int',
694 help='port to have the client connect to (testing support)')
695 parser.add_option('--remote_payload',
696 action='store_true', default=False,
697 help='Payload is being served from a remote machine')
698 parser.add_option('--src_image',
699 metavar='PATH', default='',
700 help='source image for generating delta updates from')
701 parser.add_option('-t', '--test_image',
702 action='store_true',
703 help='whether or not to use test images')
704 parser.add_option('-u', '--urlbase',
705 metavar='URL',
706 help='base URL for update images, other than the devserver')
Chris Sosa7c931362010-10-11 19:49:01 -0700707 (options, _) = parser.parse_args()
rtc@google.com21a5ca32009-11-04 18:23:23 +0000708
Chris Sosa7c931362010-10-11 19:49:01 -0700709 devserver_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
710 root_dir = os.path.realpath('%s/../..' % devserver_dir)
Chris Sosa0356d3b2010-09-16 15:46:22 -0700711 serve_only = False
712
Zdenek Behan608f46c2011-02-19 00:47:16 +0100713 static_dir = os.path.realpath('%s/static' % options.data_dir)
714 os.system('mkdir -p %s' % static_dir)
715
Sean O'Connor14b6a0a2010-03-20 23:23:48 -0700716 if options.archive_dir:
Zdenek Behan608f46c2011-02-19 00:47:16 +0100717 # TODO(zbehan) Remove legacy support:
718 # archive_dir is the directory where static/archive will point.
719 # If this is an absolute path, all is fine. If someone calls this
720 # using a relative path, that is relative to src/platform/dev/.
721 # That use case is unmaintainable, but since applications use it
722 # with =./static, instead of a boolean flag, we'll make this relative
723 # to devserver_dir to keep these unbroken. For now.
724 archive_dir = options.archive_dir
725 if not os.path.isabs(archive_dir):
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700726 archive_dir = os.path.realpath(os.path.join(devserver_dir, archive_dir))
Zdenek Behan608f46c2011-02-19 00:47:16 +0100727 _PrepareToServeUpdatesOnly(archive_dir, static_dir)
Zdenek Behan6d93e552011-03-02 22:35:49 +0100728 static_dir = os.path.realpath(archive_dir)
Chris Sosa0356d3b2010-09-16 15:46:22 -0700729 serve_only = True
Chris Sosa0356d3b2010-09-16 15:46:22 -0700730
Don Garrettf90edf02010-11-16 17:36:14 -0800731 cache_dir = os.path.join(static_dir, 'cache')
Chris Sosadbc20082012-12-10 13:39:11 -0800732 # If our devserver is only supposed to serve payloads, we shouldn't be mucking
733 # with the cache at all. If the devserver hadn't previously generated a cache
734 # and is expected, the caller is using it wrong.
735 if serve_only:
736 # Extra check to make sure we're not being called incorrectly.
737 if (options.clear_cache or options.exit or options.pregenerate_update or
738 options.board or options.image):
739 parser.error('Incompatible flags detected for serve_only mode.')
Don Garrettf90edf02010-11-16 17:36:14 -0800740
Chris Sosadbc20082012-12-10 13:39:11 -0800741 elif os.path.exists(cache_dir):
742 _CleanCache(cache_dir, options.clear_cache)
Chris Sosa6b8c3742011-01-31 12:12:17 -0800743 else:
744 os.makedirs(cache_dir)
Don Garrettf90edf02010-11-16 17:36:14 -0800745
Chris Sosadbc20082012-12-10 13:39:11 -0800746 _Log('Using cache directory %s' % cache_dir)
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700747 _Log('Data dir is %s' % options.data_dir)
748 _Log('Source root is %s' % root_dir)
749 _Log('Serving from %s' % static_dir)
rtc@google.com21a5ca32009-11-04 18:23:23 +0000750
Chris Sosa6a3697f2013-01-29 16:44:43 -0800751 # We allow global use here to share with cherrypy classes.
752 # pylint: disable=W0603
Chris Sosacde6bf42012-05-31 18:36:39 -0700753 global updater
Andrew de los Reyes52620802010-04-12 13:40:07 -0700754 updater = autoupdate.Autoupdate(
755 root_dir=root_dir,
756 static_dir=static_dir,
Chris Sosa0356d3b2010-09-16 15:46:22 -0700757 serve_only=serve_only,
Andrew de los Reyes52620802010-04-12 13:40:07 -0700758 urlbase=options.urlbase,
759 test_image=options.test_image,
Chris Sosa5d342a22010-09-28 16:54:41 -0700760 forced_image=options.image,
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700761 payload_path=options.payload,
Don Garrett0ad09372010-12-06 16:20:30 -0800762 proxy_port=options.proxy_port,
Chris Sosa4136e692010-10-28 23:42:37 -0700763 src_image=options.src_image,
Chris Sosae67b78f2010-11-04 17:33:16 -0700764 vm=options.vm,
Chris Sosa08d55a22011-01-19 16:08:02 -0800765 board=options.board,
Chris Sosa0f1ec842011-02-14 16:33:22 -0800766 copy_to_static_root=not options.exit,
767 private_key=options.private_key,
Satoru Takabayashid733cbe2011-11-15 09:36:32 -0800768 critical_update=options.critical_update,
Gilad Arnold0c9c8602012-10-02 23:58:58 -0700769 remote_payload=options.remote_payload,
Gilad Arnolda564b4b2012-10-04 10:32:44 -0700770 max_updates=options.max_updates,
Gilad Arnold8318eac2012-10-04 12:52:23 -0700771 host_log=options.host_log,
Chris Sosa0f1ec842011-02-14 16:33:22 -0800772 )
Chris Sosa7c931362010-10-11 19:49:01 -0700773
Chris Sosa6a3697f2013-01-29 16:44:43 -0800774 if options.pregenerate_update:
775 updater.PreGenerateUpdate()
Chris Sosa0356d3b2010-09-16 15:46:22 -0700776
Don Garrett0c880e22010-11-17 18:13:37 -0800777 # If the command line requested after setup, it's time to do it.
778 if not options.exit:
Chris Sosa66e2d9c2012-07-11 14:14:14 -0700779 # Handle options that must be set globally in cherrypy.
Chris Sosa2f1c41e2012-07-10 14:32:33 -0700780 if options.production:
Chris Sosa66e2d9c2012-07-11 14:14:14 -0700781 cherrypy.config.update({'environment': 'production'})
782 if not options.logfile:
783 cherrypy.config.update({'log.screen': True})
784 else:
785 cherrypy.config.update({'log.error_file': options.logfile,
786 'log.access_file': options.logfile})
Chris Sosa2f1c41e2012-07-10 14:32:33 -0700787
Don Garrett0c880e22010-11-17 18:13:37 -0800788 cherrypy.quickstart(DevServerRoot(), config=_GetConfig(options))
Chris Sosacde6bf42012-05-31 18:36:39 -0700789
790
791if __name__ == '__main__':
792 main()