blob: e176699e5b0fa36e63150598a04f40b8bdde92ad [file] [log] [blame]
Chris Sosa7c931362010-10-11 19:49:01 -07001#!/usr/bin/python
2
Chris Sosa781ba6d2012-04-11 12:44:43 -07003# Copyright (c) 2009-2012 The Chromium OS Authors. All rights reserved.
rtc@google.comded22402009-10-26 22:36:21 +00004# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
Chris Sosa3ae4dc12013-03-29 11:47:00 -07007"""Chromium OS development server that can be used for all forms of update.
8
9This devserver can be used to perform system-wide autoupdate and update
10of specific portage packages on devices running Chromium OS derived operating
11systems. It mainly operates in two modes:
12
131) archive mode: In this mode, the devserver is configured to stage and
14serve artifacts from Google Storage using the credentials provided to it before
15it is run. The easiest way to understand this is that the devserver is
16functioning as a local cache for artifacts produced and uploaded by build
17servers. Users of this form of devserver can either download the artifacts
18from the devservers static directory OR use the update RPC to perform a
19system-wide autoupdate. Archive mode is always active.
20
212) artifact-generation mode: in this mode, the devserver will attempt to
22generate update payloads and build artifacts when requested. This mode only
23works in the Chromium OS chroot as it uses build tools only present in the
24chroot (emerge, cros_generate_update_payload, etc.). By default, when a device
25requests an update from this form of devserver, the devserver will attempt to
26discover if a more recent build of the board has been built by the developer
27and generate a payload that the requested system can autoupdate to. In addition,
28it accepts gmerge requests from devices that will stage the newest version of
29a particular package from a developer's chroot onto a requesting device. Note
30if archive_dir is specified, this mode is disabled.
31
32For example:
33gmerge gmerge -d <devserver_url>
34
35devserver will see if a newer package of gmerge is available. If gmerge is
36cros_work'd on, it will re-build gmerge. After this, gmerge will install that
37version of gmerge that the devserver just created/found.
38
39For autoupdates, there are many more advanced options that can help specify
40how to update and which payload to give to a requester.
41"""
42
Chris Sosa7c931362010-10-11 19:49:01 -070043
Gilad Arnold55a2a372012-10-02 09:46:32 -070044import json
Sean O'Connor14b6a0a2010-03-20 23:23:48 -070045import optparse
rtc@google.comded22402009-10-26 22:36:21 +000046import os
Scott Zawalski4647ce62012-01-03 17:17:28 -050047import re
Simran Basi4baad082013-02-14 13:39:18 -080048import shutil
Mandeep Singh Baines38dcdda2012-12-07 17:55:33 -080049import socket
Chris Masone816e38c2012-05-02 12:22:36 -070050import subprocess
J. Richard Barnette3d977b82013-04-23 11:05:19 -070051import sys
Chris Masone816e38c2012-05-02 12:22:36 -070052import tempfile
Dan Shi59ae7092013-06-04 14:37:27 -070053import threading
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -070054import types
J. Richard Barnette3d977b82013-04-23 11:05:19 -070055from logging import handlers
56
57import cherrypy
58import cherrypy._cplogging
rtc@google.comded22402009-10-26 22:36:21 +000059
Chris Sosa0356d3b2010-09-16 15:46:22 -070060import autoupdate
Gilad Arnoldc65330c2012-09-20 15:17:48 -070061import common_util
Chris Sosa47a7d4e2012-03-28 11:26:55 -070062import downloader
Gilad Arnoldc65330c2012-09-20 15:17:48 -070063import log_util
joychen3cb228e2013-06-12 12:13:13 -070064import xbuddy
Gilad Arnoldc65330c2012-09-20 15:17:48 -070065
Gilad Arnoldc65330c2012-09-20 15:17:48 -070066# Module-local log function.
Chris Sosa6a3697f2013-01-29 16:44:43 -080067def _Log(message, *args):
68 return log_util.LogWithTag('DEVSERVER', message, *args)
Chris Sosa0356d3b2010-09-16 15:46:22 -070069
Frank Farzan40160872011-12-12 18:39:18 -080070
Chris Sosa417e55d2011-01-25 16:40:48 -080071CACHED_ENTRIES = 12
Don Garrettf90edf02010-11-16 17:36:14 -080072
Simran Basi4baad082013-02-14 13:39:18 -080073TELEMETRY_FOLDER = 'telemetry_src'
74TELEMETRY_DEPS = ['dep-telemetry_dep.tar.bz2',
75 'dep-page_cycler_dep.tar.bz2',
Simran Basi0d078682013-03-22 16:40:04 -070076 'dep-chrome_test.tar.bz2',
77 'dep-perf_data_dep.tar.bz2']
Simran Basi4baad082013-02-14 13:39:18 -080078
Chris Sosa0356d3b2010-09-16 15:46:22 -070079# Sets up global to share between classes.
rtc@google.com21a5ca32009-11-04 18:23:23 +000080updater = None
rtc@google.comded22402009-10-26 22:36:21 +000081
J. Richard Barnette3d977b82013-04-23 11:05:19 -070082# Log rotation parameters. These settings correspond to once a week
J. Richard Barnette6dfa5342013-06-04 11:48:56 -070083# at midnight between Friday and Saturday, with about three months
84# of old logs kept for backup.
J. Richard Barnette3d977b82013-04-23 11:05:19 -070085#
86# For more, see the documentation for
87# logging.handlers.TimedRotatingFileHandler
J. Richard Barnette6dfa5342013-06-04 11:48:56 -070088_LOG_ROTATION_TIME = 'W4'
J. Richard Barnette3d977b82013-04-23 11:05:19 -070089_LOG_ROTATION_BACKUP = 13
90
Frank Farzan40160872011-12-12 18:39:18 -080091
Chris Sosa9164ca32012-03-28 11:04:50 -070092class DevServerError(Exception):
Chris Sosa47a7d4e2012-03-28 11:26:55 -070093 """Exception class used by this module."""
94 pass
95
96
Scott Zawalski4647ce62012-01-03 17:17:28 -050097def _LeadingWhiteSpaceCount(string):
98 """Count the amount of leading whitespace in a string.
99
100 Args:
101 string: The string to count leading whitespace in.
102 Returns:
103 number of white space chars before characters start.
104 """
105 matched = re.match('^\s+', string)
106 if matched:
107 return len(matched.group())
108
109 return 0
110
111
112def _PrintDocStringAsHTML(func):
113 """Make a functions docstring somewhat HTML style.
114
115 Args:
116 func: The function to return the docstring from.
117 Returns:
118 A string that is somewhat formated for a web browser.
119 """
120 # TODO(scottz): Make this parse Args/Returns in a prettier way.
121 # Arguments could be bolded and indented etc.
122 html_doc = []
123 for line in func.__doc__.splitlines():
124 leading_space = _LeadingWhiteSpaceCount(line)
125 if leading_space > 0:
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700126 line = '&nbsp;' * leading_space + line
Scott Zawalski4647ce62012-01-03 17:17:28 -0500127
128 html_doc.append('<BR>%s' % line)
129
130 return '\n'.join(html_doc)
131
132
Chris Sosa7c931362010-10-11 19:49:01 -0700133def _GetConfig(options):
134 """Returns the configuration for the devserver."""
Mandeep Singh Baines38dcdda2012-12-07 17:55:33 -0800135
136 # On a system with IPv6 not compiled into the kernel,
137 # AF_INET6 sockets will return a socket.error exception.
138 # On such systems, fall-back to IPv4.
139 socket_host = '::'
140 try:
141 socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
142 except socket.error:
143 socket_host = '0.0.0.0'
144
Chris Sosa7c931362010-10-11 19:49:01 -0700145 base_config = { 'global':
146 { 'server.log_request_headers': True,
147 'server.protocol_version': 'HTTP/1.1',
Mandeep Singh Baines38dcdda2012-12-07 17:55:33 -0800148 'server.socket_host': socket_host,
Chris Sosa7c931362010-10-11 19:49:01 -0700149 'server.socket_port': int(options.port),
Chris Sosa374c62d2010-10-14 09:13:54 -0700150 'response.timeout': 6000,
Chris Sosa6fe23942012-07-02 15:44:46 -0700151 'request.show_tracebacks': True,
Chris Sosa72333d12012-06-13 11:28:05 -0700152 'server.socket_timeout': 60,
joychenecc02aa2013-07-17 18:27:35 -0700153 'server.thread_pool': 2,
Chris Sosa7c931362010-10-11 19:49:01 -0700154 },
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700155 '/api':
156 {
157 # Gets rid of cherrypy parsing post file for args.
158 'request.process_request_body': False,
159 },
Chris Sosaa1ef0102010-10-21 16:22:35 -0700160 '/build':
161 {
162 'response.timeout': 100000,
163 },
Chris Sosa7c931362010-10-11 19:49:01 -0700164 '/update':
165 {
166 # Gets rid of cherrypy parsing post file for args.
167 'request.process_request_body': False,
Chris Sosaf65f4b92010-10-21 15:57:51 -0700168 'response.timeout': 10000,
Chris Sosa7c931362010-10-11 19:49:01 -0700169 },
170 # Sets up the static dir for file hosting.
171 '/static':
joychened64b222013-06-21 16:39:34 -0700172 { 'tools.staticdir.dir': options.static_dir,
Chris Sosa7c931362010-10-11 19:49:01 -0700173 'tools.staticdir.on': True,
Chris Sosaf65f4b92010-10-21 15:57:51 -0700174 'response.timeout': 10000,
Chris Sosa7c931362010-10-11 19:49:01 -0700175 },
176 }
Chris Sosa5f118ef2012-07-12 11:37:50 -0700177 if options.production:
Alex Miller93beca52013-07-30 19:25:09 -0700178 base_config['global'].update({'server.thread_pool': 150})
Scott Zawalski1c5e7cd2012-02-27 13:12:52 -0500179
Chris Sosa7c931362010-10-11 19:49:01 -0700180 return base_config
rtc@google.com64244662009-11-12 00:52:08 +0000181
Darin Petkove17164a2010-08-11 13:24:41 -0700182
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700183def _GetRecursiveMemberObject(root, member_list):
184 """Returns an object corresponding to a nested member list.
185
186 Args:
187 root: the root object to search
188 member_list: list of nested members to search
189 Returns:
190 An object corresponding to the member name list; None otherwise.
191 """
192 for member in member_list:
193 next_root = root.__class__.__dict__.get(member)
194 if not next_root:
195 return None
196 root = next_root
197 return root
198
199
200def _IsExposed(name):
201 """Returns True iff |name| has an `exposed' attribute and it is set."""
202 return hasattr(name, 'exposed') and name.exposed
203
204
Gilad Arnold748c8322012-10-12 09:51:35 -0700205def _GetExposedMethod(root, nested_member, ignored=None):
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700206 """Returns a CherryPy-exposed method, if such exists.
207
208 Args:
209 root: the root object for searching
210 nested_member: a slash-joined path to the nested member
211 ignored: method paths to be ignored
212 Returns:
213 A function object corresponding to the path defined by |member_list| from
214 the |root| object, if the function is exposed and not ignored; None
215 otherwise.
216 """
Gilad Arnold748c8322012-10-12 09:51:35 -0700217 method = (not (ignored and nested_member in ignored) and
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700218 _GetRecursiveMemberObject(root, nested_member.split('/')))
219 if (method and type(method) == types.FunctionType and _IsExposed(method)):
220 return method
221
222
Gilad Arnold748c8322012-10-12 09:51:35 -0700223def _FindExposedMethods(root, prefix, unlisted=None):
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700224 """Finds exposed CherryPy methods.
225
226 Args:
227 root: the root object for searching
228 prefix: slash-joined chain of members leading to current object
229 unlisted: URLs to be excluded regardless of their exposed status
230 Returns:
231 List of exposed URLs that are not unlisted.
232 """
233 method_list = []
234 for member in sorted(root.__class__.__dict__.keys()):
235 prefixed_member = prefix + '/' + member if prefix else member
Gilad Arnold748c8322012-10-12 09:51:35 -0700236 if unlisted and prefixed_member in unlisted:
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700237 continue
238 member_obj = root.__class__.__dict__[member]
239 if _IsExposed(member_obj):
240 if type(member_obj) == types.FunctionType:
241 method_list.append(prefixed_member)
242 else:
243 method_list += _FindExposedMethods(
244 member_obj, prefixed_member, unlisted)
245 return method_list
246
247
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700248class ApiRoot(object):
249 """RESTful API for Dev Server information."""
250 exposed = True
251
252 @cherrypy.expose
253 def hostinfo(self, ip):
254 """Returns a JSON dictionary containing information about the given ip.
255
Gilad Arnold1b908392012-10-05 11:36:27 -0700256 Args:
257 ip: address of host whose info is requested
258 Returns:
259 A JSON dictionary containing all or some of the following fields:
260 last_event_type (int): last update event type received
261 last_event_status (int): last update event status received
262 last_known_version (string): last known version reported in update ping
263 forced_update_label (string): update label to force next update ping to
264 use, set by setnextupdate
265 See the OmahaEvent class in update_engine/omaha_request_action.h for
266 event type and status code definitions. If the ip does not exist an empty
267 string is returned.
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700268
Gilad Arnold1b908392012-10-05 11:36:27 -0700269 Example URL:
270 http://myhost/api/hostinfo?ip=192.168.1.5
271 """
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700272 return updater.HandleHostInfoPing(ip)
273
274 @cherrypy.expose
Gilad Arnold286a0062012-01-12 13:47:02 -0800275 def hostlog(self, ip):
Gilad Arnold1b908392012-10-05 11:36:27 -0700276 """Returns a JSON object containing a log of host event.
277
278 Args:
279 ip: address of host whose event log is requested, or `all'
280 Returns:
281 A JSON encoded list (log) of dictionaries (events), each of which
282 containing a `timestamp' and other event fields, as described under
283 /api/hostinfo.
284
285 Example URL:
286 http://myhost/api/hostlog?ip=192.168.1.5
287 """
Gilad Arnold286a0062012-01-12 13:47:02 -0800288 return updater.HandleHostLogPing(ip)
289
290 @cherrypy.expose
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700291 def setnextupdate(self, ip):
292 """Allows the response to the next update ping from a host to be set.
293
294 Takes the IP of the host and an update label as normally provided to the
Gilad Arnold1b908392012-10-05 11:36:27 -0700295 /update command.
296 """
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700297 body_length = int(cherrypy.request.headers['Content-Length'])
298 label = cherrypy.request.rfile.read(body_length)
299
300 if label:
301 label = label.strip()
302 if label:
303 return updater.HandleSetUpdatePing(ip, label)
304 raise cherrypy.HTTPError(400, 'No label provided.')
305
306
Gilad Arnold55a2a372012-10-02 09:46:32 -0700307 @cherrypy.expose
308 def fileinfo(self, *path_args):
309 """Returns information about a given staged file.
310
311 Args:
312 path_args: path to the file inside the server's static staging directory
313 Returns:
314 A JSON encoded dictionary with information about the said file, which may
315 contain the following keys/values:
Gilad Arnold1b908392012-10-05 11:36:27 -0700316 size (int): the file size in bytes
317 sha1 (string): a base64 encoded SHA1 hash
318 sha256 (string): a base64 encoded SHA256 hash
319
320 Example URL:
321 http://myhost/api/fileinfo/some/path/to/file
Gilad Arnold55a2a372012-10-02 09:46:32 -0700322 """
323 file_path = os.path.join(updater.static_dir, *path_args)
324 if not os.path.exists(file_path):
325 raise DevServerError('file not found: %s' % file_path)
326 try:
327 file_size = os.path.getsize(file_path)
328 file_sha1 = common_util.GetFileSha1(file_path)
329 file_sha256 = common_util.GetFileSha256(file_path)
330 except os.error, e:
331 raise DevServerError('failed to get info for file %s: %s' %
Gilad Arnolde74b3812013-04-22 11:27:38 -0700332 (file_path, e))
333
334 is_delta = autoupdate.Autoupdate.IsDeltaFormatFile(file_path)
335
336 return json.dumps({
337 autoupdate.Autoupdate.SIZE_ATTR: file_size,
338 autoupdate.Autoupdate.SHA1_ATTR: file_sha1,
339 autoupdate.Autoupdate.SHA256_ATTR: file_sha256,
340 autoupdate.Autoupdate.ISDELTA_ATTR: is_delta
341 })
Gilad Arnold55a2a372012-10-02 09:46:32 -0700342
Chris Sosa76e44b92013-01-31 12:11:38 -0800343
David Rochberg7c79a812011-01-19 14:24:45 -0500344class DevServerRoot(object):
Chris Sosa7c931362010-10-11 19:49:01 -0700345 """The Root Class for the Dev Server.
346
347 CherryPy works as follows:
348 For each method in this class, cherrpy interprets root/path
349 as a call to an instance of DevServerRoot->method_name. For example,
350 a call to http://myhost/build will call build. CherryPy automatically
351 parses http args and places them as keyword arguments in each method.
352 For paths http://myhost/update/dir1/dir2, you can use *args so that
353 cherrypy uses the update method and puts the extra paths in args.
354 """
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700355 # Method names that should not be listed on the index page.
356 _UNLISTED_METHODS = ['index', 'doc']
357
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700358 api = ApiRoot()
Chris Sosa7c931362010-10-11 19:49:01 -0700359
Dan Shi59ae7092013-06-04 14:37:27 -0700360 # Number of threads that devserver is staging images.
361 _staging_thread_count = 0
362 # Lock used to lock increasing/decreasing count.
363 _staging_thread_count_lock = threading.Lock()
364
joychen3cb228e2013-06-12 12:13:13 -0700365 def __init__(self, _xbuddy):
Nick Sanders7dcaa2e2011-08-04 15:20:41 -0700366 self._builder = None
Simran Basi4baad082013-02-14 13:39:18 -0800367 self._telemetry_lock_dict = common_util.LockDict()
joychen3cb228e2013-06-12 12:13:13 -0700368 self._xbuddy = _xbuddy
David Rochberg7c79a812011-01-19 14:24:45 -0500369
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700370 @cherrypy.expose
David Rochberg7c79a812011-01-19 14:24:45 -0500371 def build(self, board, pkg, **kwargs):
Chris Sosa7c931362010-10-11 19:49:01 -0700372 """Builds the package specified."""
Nick Sanders7dcaa2e2011-08-04 15:20:41 -0700373 import builder
374 if self._builder is None:
375 self._builder = builder.Builder()
David Rochberg7c79a812011-01-19 14:24:45 -0500376 return self._builder.Build(board, pkg, kwargs)
Chris Sosa7c931362010-10-11 19:49:01 -0700377
Chris Sosacde6bf42012-05-31 18:36:39 -0700378 @staticmethod
379 def _canonicalize_archive_url(archive_url):
380 """Canonicalizes archive_url strings.
381
382 Raises:
383 DevserverError: if archive_url is not set.
384 """
385 if archive_url:
Chris Sosa76e44b92013-01-31 12:11:38 -0800386 if not archive_url.startswith('gs://'):
387 raise DevServerError("Archive URL isn't from Google Storage.")
388
Chris Sosacde6bf42012-05-31 18:36:39 -0700389 return archive_url.rstrip('/')
390 else:
391 raise DevServerError("Must specify an archive_url in the request")
392
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700393 @cherrypy.expose
Frank Farzanbcb571e2012-01-03 11:48:17 -0800394 def download(self, **kwargs):
395 """Downloads and archives full/delta payloads from Google Storage.
396
Chris Sosa76e44b92013-01-31 12:11:38 -0800397 THIS METHOD IS DEPRECATED: use stage(..., artifacts=...) instead.
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700398 This methods downloads artifacts. It may download artifacts in the
399 background in which case a caller should call wait_for_status to get
400 the status of the background artifact downloads. They should use the same
401 args passed to download.
402
Frank Farzanbcb571e2012-01-03 11:48:17 -0800403 Args:
404 archive_url: Google Storage URL for the build.
Dan Shif8eb0d12013-08-01 17:52:06 -0700405 artifacts: Comma separated list of artifacts to download.
Frank Farzanbcb571e2012-01-03 11:48:17 -0800406
407 Example URL:
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700408 http://myhost/download?archive_url=gs://chromeos-image-archive/
Dan Shif8eb0d12013-08-01 17:52:06 -0700409 x86-generic/R17-1208.0.0-a1-b338&artifacts=full_payload,test_suites,
410 stateful
Frank Farzanbcb571e2012-01-03 11:48:17 -0800411 """
Dan Shif8eb0d12013-08-01 17:52:06 -0700412 async = kwargs.get('async', False)
Chris Sosa76e44b92013-01-31 12:11:38 -0800413 return self.stage(archive_url=kwargs.get('archive_url'),
Dan Shif8eb0d12013-08-01 17:52:06 -0700414 artifacts=kwargs.get('artifacts'),
415 async=async)
Chris Sosa76e44b92013-01-31 12:11:38 -0800416
Dan Shif8eb0d12013-08-01 17:52:06 -0700417 @cherrypy.expose
418 def is_staged(self, **kwargs):
419 """Check if artifacts have been downloaded.
420
421 @param archive_url: Google Storage URL for the build.
422 @param artifacts: Comma separated list of artifacts to download.
423 @returns: True of all artifacts are staged.
424
425 Example:
426 To check if autotest and test_suites are staged:
427 http://devserver_url:<port>/is_staged?archive_url=gs://your_url/path&
428 artifacts=autotest,test_suites
429 """
430 archive_url = self._canonicalize_archive_url(kwargs.get('archive_url'))
431 artifacts = kwargs.get('artifacts', '')
432 if not artifacts:
433 raise DevServerError('No artifacts specified.')
434 return str(downloader.Downloader(updater.static_dir, archive_url).IsStaged(
435 artifacts.split(',')))
Dan Shi59ae7092013-06-04 14:37:27 -0700436
Chris Sosa76e44b92013-01-31 12:11:38 -0800437 @cherrypy.expose
438 def stage(self, **kwargs):
439 """Downloads and caches the artifacts from Google Storage URL.
440
441 Downloads and caches the artifacts Google Storage URL. Returns once these
442 have been downloaded on the devserver. A call to this will attempt to cache
443 non-specified artifacts in the background for the given from the given URL
444 following the principle of spatial locality. Spatial locality of different
445 artifacts is explicitly defined in the build_artifact module.
446
447 These artifacts will then be available from the static/ sub-directory of
448 the devserver.
449
450 Args:
451 archive_url: Google Storage URL for the build.
452 artifacts: Comma separated list of artifacts to download.
Dan Shif8eb0d12013-08-01 17:52:06 -0700453 async: True to return without waiting for download to complete.
Chris Sosa76e44b92013-01-31 12:11:38 -0800454
455 Example:
456 To download the autotest and test suites tarballs:
457 http://devserver_url:<port>/stage?archive_url=gs://your_url/path&
458 artifacts=autotest,test_suites
459 To download the full update payload:
460 http://devserver_url:<port>/stage?archive_url=gs://your_url/path&
461 artifacts=full_payload
462
463 For both these examples, one could find these artifacts at:
joychened64b222013-06-21 16:39:34 -0700464 http://devserver_url:<port>/static/<relative_path>*
Chris Sosa76e44b92013-01-31 12:11:38 -0800465
466 Note for this example, relative path is the archive_url stripped of its
467 basename i.e. path/ in the examples above. Specific example:
468
469 gs://chromeos-image-archive/x86-mario-release/R26-3920.0.0
470
471 Will get staged to:
472
joychened64b222013-06-21 16:39:34 -0700473 http://devserver_url:<port>/static/x86-mario-release/R26-3920.0.0
Chris Sosa76e44b92013-01-31 12:11:38 -0800474 """
Chris Sosacde6bf42012-05-31 18:36:39 -0700475 archive_url = self._canonicalize_archive_url(kwargs.get('archive_url'))
Chris Sosa76e44b92013-01-31 12:11:38 -0800476 artifacts = kwargs.get('artifacts', '')
Dan Shif8eb0d12013-08-01 17:52:06 -0700477 async = kwargs.get('async', False)
Chris Sosa76e44b92013-01-31 12:11:38 -0800478 if not artifacts:
479 raise DevServerError('No artifacts specified.')
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700480
Dan Shi59ae7092013-06-04 14:37:27 -0700481 with DevServerRoot._staging_thread_count_lock:
482 DevServerRoot._staging_thread_count += 1
483 try:
Dan Shif8eb0d12013-08-01 17:52:06 -0700484 downloader.Downloader(updater.static_dir,
485 archive_url).Download(artifacts.split(','), async=async)
Dan Shi59ae7092013-06-04 14:37:27 -0700486 finally:
487 with DevServerRoot._staging_thread_count_lock:
488 DevServerRoot._staging_thread_count -= 1
Chris Sosa76e44b92013-01-31 12:11:38 -0800489 return 'Success'
Chris Sosacde6bf42012-05-31 18:36:39 -0700490
491 @cherrypy.expose
Simran Basi4baad082013-02-14 13:39:18 -0800492 def setup_telemetry(self, **kwargs):
493 """Extracts and sets up telemetry
494
495 This method goes through the telemetry deps packages, and stages them on
496 the devserver to be used by the drones and the telemetry tests.
497
498 Args:
499 archive_url: Google Storage URL for the build.
500
501 Returns:
502 Path to the source folder for the telemetry codebase once it is staged.
503 """
504 archive_url = kwargs.get('archive_url')
505 self.stage(archive_url=archive_url, artifacts='autotest')
506
507 build = '/'.join(downloader.Downloader.ParseUrl(archive_url))
508 build_path = os.path.join(updater.static_dir, build)
509 deps_path = os.path.join(build_path, 'autotest/packages')
510 telemetry_path = os.path.join(build_path, TELEMETRY_FOLDER)
511 src_folder = os.path.join(telemetry_path, 'src')
512
513 with self._telemetry_lock_dict.lock(telemetry_path):
514 if os.path.exists(src_folder):
515 # Telemetry is already fully stage return
516 return src_folder
517
518 common_util.MkDirP(telemetry_path)
519
520 # Copy over the required deps tar balls to the telemetry directory.
521 for dep in TELEMETRY_DEPS:
522 dep_path = os.path.join(deps_path, dep)
Simran Basi0d078682013-03-22 16:40:04 -0700523 if not os.path.exists(dep_path):
524 # This dep does not exist (could be new), do not extract it.
525 continue
Simran Basi4baad082013-02-14 13:39:18 -0800526 try:
527 common_util.ExtractTarball(dep_path, telemetry_path)
528 except common_util.CommonUtilError as e:
529 shutil.rmtree(telemetry_path)
530 raise DevServerError(str(e))
531
532 # By default all the tarballs extract to test_src but some parts of
533 # the telemetry code specifically hardcoded to exist inside of 'src'.
534 test_src = os.path.join(telemetry_path, 'test_src')
535 try:
536 shutil.move(test_src, src_folder)
537 except shutil.Error:
538 # This can occur if src_folder already exists. Remove and retry move.
539 shutil.rmtree(src_folder)
540 raise DevServerError('Failure in telemetry setup for build %s. Appears'
541 ' that the test_src to src move failed.' % build)
542
543 return src_folder
544
545 @cherrypy.expose
Chris Sosacde6bf42012-05-31 18:36:39 -0700546 def wait_for_status(self, **kwargs):
547 """Waits for background artifacts to be downloaded from Google Storage.
548
Chris Sosa76e44b92013-01-31 12:11:38 -0800549 THIS METHOD IS DEPRECATED: use stage(..., artifacts=...) instead.
Chris Sosacde6bf42012-05-31 18:36:39 -0700550 Args:
551 archive_url: Google Storage URL for the build.
552
553 Example URL:
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700554 http://myhost/wait_for_status?archive_url=gs://chromeos-image-archive/
555 x86-generic/R17-1208.0.0-a1-b338
Chris Sosacde6bf42012-05-31 18:36:39 -0700556 """
Chris Sosa76e44b92013-01-31 12:11:38 -0800557 return self.stage(archive_url=kwargs.get('archive_url'),
Dan Shif8eb0d12013-08-01 17:52:06 -0700558 artifacts=kwargs.get('artifacts'))
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700559
560 @cherrypy.expose
Chris Masone816e38c2012-05-02 12:22:36 -0700561 def stage_debug(self, **kwargs):
562 """Downloads and stages debug symbol payloads from Google Storage.
563
Chris Sosa76e44b92013-01-31 12:11:38 -0800564 THIS METHOD IS DEPRECATED: use stage(..., artifacts=...) instead.
565 This methods downloads the debug symbol build artifact
566 synchronously, and then stages it for use by symbolicate_dump.
Chris Masone816e38c2012-05-02 12:22:36 -0700567
568 Args:
569 archive_url: Google Storage URL for the build.
570
571 Example URL:
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700572 http://myhost/stage_debug?archive_url=gs://chromeos-image-archive/
573 x86-generic/R17-1208.0.0-a1-b338
Chris Masone816e38c2012-05-02 12:22:36 -0700574 """
Chris Sosa76e44b92013-01-31 12:11:38 -0800575 return self.stage(archive_url=kwargs.get('archive_url'),
576 artifacts='symbols')
Chris Masone816e38c2012-05-02 12:22:36 -0700577
578 @cherrypy.expose
Chris Sosa76e44b92013-01-31 12:11:38 -0800579 def symbolicate_dump(self, minidump, **kwargs):
Chris Masone816e38c2012-05-02 12:22:36 -0700580 """Symbolicates a minidump using pre-downloaded symbols, returns it.
581
582 Callers will need to POST to this URL with a body of MIME-type
583 "multipart/form-data".
584 The body should include a single argument, 'minidump', containing the
585 binary-formatted minidump to symbolicate.
586
Chris Masone816e38c2012-05-02 12:22:36 -0700587 Args:
Chris Sosa76e44b92013-01-31 12:11:38 -0800588 archive_url: Google Storage URL for the build.
Chris Masone816e38c2012-05-02 12:22:36 -0700589 minidump: The binary minidump file to symbolicate.
590 """
Chris Sosa76e44b92013-01-31 12:11:38 -0800591 # Ensure the symbols have been staged.
592 archive_url = self._canonicalize_archive_url(kwargs.get('archive_url'))
593 if self.stage(archive_url=archive_url, artifacts='symbols') != 'Success':
594 raise DevServerError('Failed to stage symbols for %s' % archive_url)
595
Chris Masone816e38c2012-05-02 12:22:36 -0700596 to_return = ''
597 with tempfile.NamedTemporaryFile() as local:
598 while True:
599 data = minidump.file.read(8192)
600 if not data:
601 break
602 local.write(data)
Chris Sosa76e44b92013-01-31 12:11:38 -0800603
Chris Masone816e38c2012-05-02 12:22:36 -0700604 local.flush()
Chris Sosa76e44b92013-01-31 12:11:38 -0800605
606 symbols_directory = os.path.join(downloader.Downloader.GetBuildDir(
607 updater.static_dir, archive_url), 'debug', 'breakpad')
608
609 stackwalk = subprocess.Popen(
610 ['minidump_stackwalk', local.name, symbols_directory],
611 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
612
Chris Masone816e38c2012-05-02 12:22:36 -0700613 to_return, error_text = stackwalk.communicate()
614 if stackwalk.returncode != 0:
615 raise DevServerError("Can't generate stack trace: %s (rc=%d)" % (
616 error_text, stackwalk.returncode))
617
618 return to_return
619
620 @cherrypy.expose
Scott Zawalski16954532012-03-20 15:31:36 -0400621 def latestbuild(self, **params):
622 """Return a string representing the latest build for a given target.
623
624 Args:
625 target: The build target, typically a combination of the board and the
626 type of build e.g. x86-mario-release.
627 milestone: The milestone to filter builds on. E.g. R16. Optional, if not
628 provided the latest RXX build will be returned.
629 Returns:
630 A string representation of the latest build if one exists, i.e.
631 R19-1993.0.0-a1-b1480.
632 An empty string if no latest could be found.
633 """
634 if not params:
635 return _PrintDocStringAsHTML(self.latestbuild)
636
637 if 'target' not in params:
638 raise cherrypy.HTTPError('500 Internal Server Error',
639 'Error: target= is required!')
640 try:
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700641 return common_util.GetLatestBuildVersion(
Scott Zawalski16954532012-03-20 15:31:36 -0400642 updater.static_dir, params['target'],
643 milestone=params.get('milestone'))
Gilad Arnold17fe03d2012-10-02 10:05:01 -0700644 except common_util.CommonUtilError as errmsg:
Scott Zawalski16954532012-03-20 15:31:36 -0400645 raise cherrypy.HTTPError('500 Internal Server Error', str(errmsg))
646
647 @cherrypy.expose
Scott Zawalski84a39c92012-01-13 15:12:42 -0500648 def controlfiles(self, **params):
Scott Zawalski4647ce62012-01-03 17:17:28 -0500649 """Return a control file or a list of all known control files.
650
651 Example URL:
652 To List all control files:
beepsbd337242013-07-09 22:44:06 -0700653 http://dev-server/controlfiles?suite_name=&build=daisy_spring-release/R29-4279.0.0
654 To List all control files for, say, the bvt suite:
655 http://dev-server/controlfiles?suite_name=bvt&build=daisy_spring-release/R29-4279.0.0
Scott Zawalski4647ce62012-01-03 17:17:28 -0500656 To return the contents of a path:
Scott Zawalski84a39c92012-01-13 15:12:42 -0500657 http://dev-server/controlfiles?board=x86-alex-release&build=R18-1514.0.0&control_path=client/sleeptest/control
Scott Zawalski4647ce62012-01-03 17:17:28 -0500658
659 Args:
Scott Zawalski84a39c92012-01-13 15:12:42 -0500660 build: The build i.e. x86-alex-release/R18-1514.0.0-a1-b1450.
Scott Zawalski4647ce62012-01-03 17:17:28 -0500661 control_path: If you want the contents of a control file set this
662 to the path. E.g. client/site_tests/sleeptest/control
663 Optional, if not provided return a list of control files is returned.
beepsbd337242013-07-09 22:44:06 -0700664 suite_name: If control_path is not specified but a suite_name is
665 specified, list the control files belonging to that suite instead of
666 all control files. The empty string for suite_name will list all control
667 files for the build.
Scott Zawalski4647ce62012-01-03 17:17:28 -0500668 Returns:
669 Contents of a control file if control_path is provided.
670 A list of control files if no control_path is provided.
671 """
Scott Zawalski4647ce62012-01-03 17:17:28 -0500672 if not params:
673 return _PrintDocStringAsHTML(self.controlfiles)
674
Scott Zawalski84a39c92012-01-13 15:12:42 -0500675 if 'build' not in params:
Scott Zawalski4647ce62012-01-03 17:17:28 -0500676 raise cherrypy.HTTPError('500 Internal Server Error',
Scott Zawalski84a39c92012-01-13 15:12:42 -0500677 'Error: build= is required!')
Scott Zawalski4647ce62012-01-03 17:17:28 -0500678
679 if 'control_path' not in params:
beepsbd337242013-07-09 22:44:06 -0700680 if 'suite_name' in params and params['suite_name']:
681 return common_util.GetControlFileListForSuite(
682 updater.static_dir, params['build'], params['suite_name'])
683 else:
684 return common_util.GetControlFileList(
685 updater.static_dir, params['build'])
Scott Zawalski4647ce62012-01-03 17:17:28 -0500686 else:
Gilad Arnoldc65330c2012-09-20 15:17:48 -0700687 return common_util.GetControlFile(
688 updater.static_dir, params['build'], params['control_path'])
Frank Farzan40160872011-12-12 18:39:18 -0800689
690 @cherrypy.expose
Gilad Arnold6f99b982012-09-12 10:49:40 -0700691 def stage_images(self, **kwargs):
692 """Downloads and stages a Chrome OS image from Google Storage.
693
Chris Sosa76e44b92013-01-31 12:11:38 -0800694 THIS METHOD IS DEPRECATED: use stage(..., artifacts=...) instead.
Gilad Arnold6f99b982012-09-12 10:49:40 -0700695 This method downloads a zipped archive from a specified GS location, then
696 extracts and stages the specified list of images and stages them under
Chris Sosa76e44b92013-01-31 12:11:38 -0800697 static/BOARD/BUILD/. Download is synchronous.
Gilad Arnold6f99b982012-09-12 10:49:40 -0700698
699 Args:
700 archive_url: Google Storage URL for the build.
701 image_types: comma-separated list of images to download, may include
702 'test', 'recovery', and 'base'
703
704 Example URL:
705 http://myhost/stage_images?archive_url=gs://chromeos-image-archive/
706 x86-generic/R17-1208.0.0-a1-b338&image_types=test,base
707 """
Gilad Arnold6f99b982012-09-12 10:49:40 -0700708 image_types = kwargs.get('image_types').split(',')
Chris Sosa76e44b92013-01-31 12:11:38 -0800709 image_types_list = [image + '_image' for image in image_types]
710 self.stage(archive_url=kwargs.get('archive_url'), artifacts=','.join(
711 image_types_list))
Gilad Arnold6f99b982012-09-12 10:49:40 -0700712
713 @cherrypy.expose
joycheneaf4cfc2013-07-02 08:38:57 -0700714 def xbuddy(self, *args, **kwargs):
715 """The full xBuddy call, returns resource specified by path_parts.
joychen3cb228e2013-06-12 12:13:13 -0700716
717 Args:
joycheneaf4cfc2013-07-02 08:38:57 -0700718 path_parts: the path following xbuddy/ in the call url is split into the
719 components of the path.
720 The path can be understood as a build_id/artifact, build_id is
721 composed of "board/version"
722
723 path_parts[0], the board, is the familiar board name, optionally
724 suffixed.
725 path_parts[1], the version, can be the google storage version
726 number, and may also be any of a number of xBuddy defined version
727 aliases that will be translated into the latest built image that
728 fits the description. defaults to latest.
729 path_parts[2], the artifact, is one of a number of image or artifact
730 aliases used by xbuddy, defined in xbuddy:ALIASES. Defaults to test
731
732 Kwargs:
joychen3cb228e2013-06-12 12:13:13 -0700733 return_dir: {true|false}
734 if set to true, returns the url to the update.gz
735 instead.
736
737 Example URL:
joycheneaf4cfc2013-07-02 08:38:57 -0700738 http://host:port/xbuddy/x86-generic/R26-4000.0.0/test
joychen3cb228e2013-06-12 12:13:13 -0700739 or
joycheneaf4cfc2013-07-02 08:38:57 -0700740 http://host:port/xbuddy/x86-generic/R26-4000.0.0/test?return_dir=true
joychen3cb228e2013-06-12 12:13:13 -0700741
742 Returns:
743 A redirect to the image or update file on the devserver.
744 e.g. http://host:port/static/archive/x86-generic-release/
745 R26-4000.0.0/chromium-test-image.bin
746 or if return_dir is True, return path to the folder where
747 image or update file is
748 http://host:port/static/x86-generic-release/R26-4000.0.0/
749 """
750 boolean_string = kwargs.get('return_dir')
751 return_dir = xbuddy.XBuddy.ParseBoolean(boolean_string)
joycheneaf4cfc2013-07-02 08:38:57 -0700752 return_url = self._xbuddy.Get(args,
joychen3cb228e2013-06-12 12:13:13 -0700753 return_dir)
754 if return_dir:
joycheneaf4cfc2013-07-02 08:38:57 -0700755 directory = os.path.join(cherrypy.request.base, return_url)
756 _Log("Directory requested, returning: %s", directory)
757 return directory
joychen3cb228e2013-06-12 12:13:13 -0700758 else:
joycheneaf4cfc2013-07-02 08:38:57 -0700759 return_url = '/' + return_url
760 _Log("Payload requested, returning: %s", return_url)
joychen3cb228e2013-06-12 12:13:13 -0700761 raise cherrypy.HTTPRedirect(return_url, 302)
762
763 @cherrypy.expose
764 def xbuddy_list(self):
765 """Lists the currently available images & time since last access.
766
767 @return: A string representation of a list of tuples
768 [(build_id, time since last access),...]
769 """
770 return self._xbuddy.List()
771
772 @cherrypy.expose
773 def xbuddy_capacity(self):
774 """Returns the number of images cached by xBuddy.
775
776 @return: Capacity of this devserver.
777 """
778 return self._xbuddy.Capacity()
779
780 @cherrypy.expose
Chris Sosa7c931362010-10-11 19:49:01 -0700781 def index(self):
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700782 """Presents a welcome message and documentation links."""
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700783 return ('Welcome to the Dev Server!<br>\n'
784 '<br>\n'
785 'Here are the available methods, click for documentation:<br>\n'
786 '<br>\n'
787 '%s' %
788 '<br>\n'.join(
789 [('<a href=doc/%s>%s</a>' % (name, name))
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700790 for name in _FindExposedMethods(
791 self, '', unlisted=self._UNLISTED_METHODS)]))
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700792
793 @cherrypy.expose
794 def doc(self, *args):
795 """Shows the documentation for available methods / URLs.
796
797 Example:
798 http://myhost/doc/update
799 """
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700800 name = '/'.join(args)
801 method = _GetExposedMethod(self, name)
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700802 if not method:
803 raise DevServerError("No exposed method named `%s'" % name)
804 if not method.__doc__:
805 raise DevServerError("No documentation for exposed method `%s'" % name)
806 return '<pre>\n%s</pre>' % method.__doc__
Chris Sosa7c931362010-10-11 19:49:01 -0700807
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700808 @cherrypy.expose
Chris Sosa7c931362010-10-11 19:49:01 -0700809 def update(self, *args):
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700810 """Handles an update check from a Chrome OS client.
811
812 The HTTP request should contain the standard Omaha-style XML blob. The URL
813 line may contain an additional intermediate path to the update payload.
814
joychenb0dfe552013-07-30 10:02:06 -0700815 Paths that can be handled by xbuddy are formatted:
816 http://myhost/update/xbuddy/board/version
817
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700818 Example:
819 http://myhost/update/optional/path/to/payload
820 """
joychen346531c2013-07-24 16:55:56 -0700821 if len(args) > 0 and args[0] == 'xbuddy':
822 # Interpret the rest of the path as an xbuddy path
823 label, found = self._xbuddy.Translate(args[1:] + ('full_payload',))
824 if not found:
825 _Log("Update payload not found for %s, xBuddy looking it up.", label)
826 else:
827 label = '/'.join(args)
828
829 _Log('Update label: %s', label)
Gilad Arnold286a0062012-01-12 13:47:02 -0800830 body_length = int(cherrypy.request.headers.get('Content-Length', 0))
Chris Sosa7c931362010-10-11 19:49:01 -0700831 data = cherrypy.request.rfile.read(body_length)
832 return updater.HandleUpdatePing(data, label)
833
Chris Sosa0356d3b2010-09-16 15:46:22 -0700834
Dan Shif5ce2de2013-04-25 16:06:32 -0700835 @cherrypy.expose
836 def check_health(self):
837 """Collect the health status of devserver to see if it's ready for staging.
838
839 @return: A JSON dictionary containing all or some of the following fields:
Dan Shi59ae7092013-06-04 14:37:27 -0700840 free_disk (int): free disk space in GB
841 staging_thread_count (int): number of devserver threads currently
842 staging an image
Dan Shif5ce2de2013-04-25 16:06:32 -0700843 """
844 # Get free disk space.
845 stat = os.statvfs(updater.static_dir)
846 free_disk = stat.f_bsize * stat.f_bavail / 1000000000
847
848 return json.dumps({
849 'free_disk': free_disk,
Dan Shi59ae7092013-06-04 14:37:27 -0700850 'staging_thread_count': DevServerRoot._staging_thread_count,
Dan Shif5ce2de2013-04-25 16:06:32 -0700851 })
852
853
Chris Sosadbc20082012-12-10 13:39:11 -0800854def _CleanCache(cache_dir, wipe):
855 """Wipes any excess cached items in the cache_dir.
856
857 Args:
858 cache_dir: the directory we are wiping from.
859 wipe: If True, wipe all the contents -- not just the excess.
860 """
861 if wipe:
862 # Clear the cache and exit on error.
863 cmd = 'rm -rf %s/*' % cache_dir
864 if os.system(cmd) != 0:
865 _Log('Failed to clear the cache with %s' % cmd)
866 sys.exit(1)
867 else:
868 # Clear all but the last N cached updates
869 cmd = ('cd %s; ls -tr | head --lines=-%d | xargs rm -rf' %
870 (cache_dir, CACHED_ENTRIES))
871 if os.system(cmd) != 0:
872 _Log('Failed to clean up old delta cache files with %s' % cmd)
873 sys.exit(1)
874
875
Chris Sosa3ae4dc12013-03-29 11:47:00 -0700876def _AddTestingOptions(parser):
877 group = optparse.OptionGroup(
878 parser, 'Advanced Testing Options', 'These are used by test scripts and '
879 'developers writing integration tests utilizing the devserver. They are '
880 'not intended to be really used outside the scope of someone '
881 'knowledgable about the test.')
882 group.add_option('--exit',
883 action='store_true',
884 help='do not start the server (yet pregenerate/clear cache)')
885 group.add_option('--host_log',
886 action='store_true', default=False,
887 help='record history of host update events (/api/hostlog)')
888 group.add_option('--max_updates',
889 metavar='NUM', default= -1, type='int',
890 help='maximum number of update checks handled positively '
891 '(default: unlimited)')
892 group.add_option('--private_key',
893 metavar='PATH', default=None,
894 help='path to the private key in pem format. If this is set '
895 'the devserver will generate update payloads that are '
896 'signed with this key.')
897 group.add_option('--proxy_port',
898 metavar='PORT', default=None, type='int',
899 help='port to have the client connect to -- basically the '
900 'devserver lies to the update to tell it to get the payload '
901 'from a different port that will proxy the request back to '
902 'the devserver. The proxy must be managed outside the '
903 'devserver.')
904 group.add_option('--remote_payload',
905 action='store_true', default=False,
906 help='Payload is being served from a remote machine')
907 group.add_option('-u', '--urlbase',
908 metavar='URL',
909 help='base URL for update images, other than the '
910 'devserver. Use in conjunction with remote_payload.')
911 parser.add_option_group(group)
912
913
914def _AddUpdateOptions(parser):
915 group = optparse.OptionGroup(
916 parser, 'Autoupdate Options', 'These options can be used to change '
917 'how the devserver either generates or serve update payloads. Please '
918 'note that all of these option affect how a payload is generated and so '
919 'do not work in archive-only mode.')
920 group.add_option('--board',
921 help='By default the devserver will create an update '
922 'payload from the latest image built for the board '
923 'a device that is requesting an update has. When we '
924 'pre-generate an update (see below) and we do not specify '
925 'another update_type option like image or payload, the '
926 'devserver needs to know the board to generate the latest '
927 'image for. This is that board.')
928 group.add_option('--critical_update',
929 action='store_true', default=False,
930 help='Present update payload as critical')
931 group.add_option('--for_vm',
932 dest='vm', action='store_true',
933 help='DEPRECATED: see no_patch_kernel.')
934 group.add_option('--image',
935 metavar='FILE',
936 help='Generate and serve an update using this image to any '
937 'device that requests an update.')
938 group.add_option('--no_patch_kernel',
939 dest='patch_kernel', action='store_false', default=True,
940 help='When generating an update payload, do not patch the '
941 'kernel with kernel verification blob from the stateful '
942 'partition.')
943 group.add_option('--payload',
944 metavar='PATH',
945 help='use the update payload from specified directory '
946 '(update.gz).')
947 group.add_option('-p', '--pregenerate_update',
948 action='store_true', default=False,
949 help='pre-generate the update payload before accepting '
950 'update requests. Useful to help debug payload generation '
951 'issues quickly. Also if an update payload will take a '
952 'long time to generate, a client may timeout if you do not'
953 'pregenerate the update.')
954 group.add_option('--src_image',
955 metavar='PATH', default='',
956 help='If specified, delta updates will be generated using '
957 'this image as the source image. Delta updates are when '
958 'you are updating from a "source image" to a another '
959 'image.')
960 parser.add_option_group(group)
961
962
963def _AddProductionOptions(parser):
964 group = optparse.OptionGroup(
965 parser, 'Advanced Server Options', 'These options can be used to changed '
966 'for advanced server behavior.')
Chris Sosa3ae4dc12013-03-29 11:47:00 -0700967 group.add_option('--archive_dir',
968 metavar='PATH',
joychened64b222013-06-21 16:39:34 -0700969 help='To be deprecated.')
Chris Sosa3ae4dc12013-03-29 11:47:00 -0700970 group.add_option('--clear_cache',
971 action='store_true', default=False,
972 help='At startup, removes all cached entries from the'
973 'devserver\'s cache.')
974 group.add_option('--logfile',
975 metavar='PATH',
976 help='log output to this file instead of stdout')
977 group.add_option('--production',
978 action='store_true', default=False,
979 help='have the devserver use production values when '
980 'starting up. This includes using more threads and '
981 'performing less logging.')
982 parser.add_option_group(group)
983
984
J. Richard Barnette3d977b82013-04-23 11:05:19 -0700985def _MakeLogHandler(logfile):
986 """Create a LogHandler instance used to log all messages."""
987 hdlr_cls = handlers.TimedRotatingFileHandler
988 hdlr = hdlr_cls(logfile, when=_LOG_ROTATION_TIME,
989 backupCount=_LOG_ROTATION_BACKUP)
990 # The cherrypy documentation says to use the _cplogging module for
991 # this, even though it's named as a private module.
992 # pylint: disable=W0212
993 hdlr.setFormatter(cherrypy._cplogging.logfmt)
994 return hdlr
995
996
Chris Sosacde6bf42012-05-31 18:36:39 -0700997def main():
Chris Sosa3ae4dc12013-03-29 11:47:00 -0700998 usage = '\n\n'.join(['usage: %prog [options]', __doc__])
Gilad Arnold286a0062012-01-12 13:47:02 -0800999 parser = optparse.OptionParser(usage=usage)
joychened64b222013-06-21 16:39:34 -07001000
1001 # get directory that the devserver is run from
1002 devserver_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
1003 default_archive_dir = '%s/static' % devserver_dir
1004 parser.add_option('--static_dir',
Gilad Arnold9714d9b2012-10-04 10:09:42 -07001005 metavar='PATH',
joychened64b222013-06-21 16:39:34 -07001006 default=default_archive_dir,
1007 help='writable static directory')
Gilad Arnold9714d9b2012-10-04 10:09:42 -07001008 parser.add_option('--port',
1009 default=8080, type='int',
1010 help='port for the dev server to use (default: 8080)')
Gilad Arnold9714d9b2012-10-04 10:09:42 -07001011 parser.add_option('-t', '--test_image',
1012 action='store_true',
Chris Sosa3ae4dc12013-03-29 11:47:00 -07001013 help='If set, look for the chromiumos_test_image.bin file '
1014 'when generating update payloads rather than the '
1015 'chromiumos_image.bin which is the default.')
joychen5260b9a2013-07-16 14:48:01 -07001016 parser.add_option('-x', '--xbuddy_manage_builds',
1017 action='store_true',
1018 default=False,
1019 help='If set, allow xbuddy to manage images in'
1020 'build/images.')
Chris Sosa3ae4dc12013-03-29 11:47:00 -07001021 _AddProductionOptions(parser)
1022 _AddUpdateOptions(parser)
1023 _AddTestingOptions(parser)
Chris Sosa7c931362010-10-11 19:49:01 -07001024 (options, _) = parser.parse_args()
rtc@google.com21a5ca32009-11-04 18:23:23 +00001025
J. Richard Barnette3d977b82013-04-23 11:05:19 -07001026 # Handle options that must be set globally in cherrypy. Do this
1027 # work up front, because calls to _Log() below depend on this
1028 # initialization.
1029 if options.production:
1030 cherrypy.config.update({'environment': 'production'})
1031 if not options.logfile:
1032 cherrypy.config.update({'log.screen': True})
1033 else:
1034 cherrypy.config.update({'log.error_file': '',
1035 'log.access_file': ''})
1036 hdlr = _MakeLogHandler(options.logfile)
1037 # Pylint can't seem to process these two calls properly
1038 # pylint: disable=E1101
1039 cherrypy.log.access_log.addHandler(hdlr)
1040 cherrypy.log.error_log.addHandler(hdlr)
1041 # pylint: enable=E1101
1042
Chris Sosa7c931362010-10-11 19:49:01 -07001043 root_dir = os.path.realpath('%s/../..' % devserver_dir)
Chris Sosa0356d3b2010-09-16 15:46:22 -07001044
J. Richard Barnette3d977b82013-04-23 11:05:19 -07001045 # TODO(sosa): Remove after deprecation.
Chris Sosa3ae4dc12013-03-29 11:47:00 -07001046 if options.vm:
1047 options.patch_kernel = False
1048
joychened64b222013-06-21 16:39:34 -07001049 # set static_dir, from which everything will be served
Sean O'Connor14b6a0a2010-03-20 23:23:48 -07001050 if options.archive_dir:
joychened64b222013-06-21 16:39:34 -07001051 # TODO(joyc) To be deprecated
Zdenek Behan608f46c2011-02-19 00:47:16 +01001052 archive_dir = options.archive_dir
1053 if not os.path.isabs(archive_dir):
joychened64b222013-06-21 16:39:34 -07001054 archive_dir = os.path.join(devserver_dir, archive_dir)
1055 options.static_dir = os.path.realpath(archive_dir)
joychened64b222013-06-21 16:39:34 -07001056 else:
1057 options.static_dir = os.path.realpath(options.static_dir)
Chris Sosa0356d3b2010-09-16 15:46:22 -07001058
joychened64b222013-06-21 16:39:34 -07001059 cache_dir = os.path.join(options.static_dir, 'cache')
J. Richard Barnette3d977b82013-04-23 11:05:19 -07001060 # If our devserver is only supposed to serve payloads, we shouldn't be
1061 # mucking with the cache at all. If the devserver hadn't previously
1062 # generated a cache and is expected, the caller is using it wrong.
joychen7c2054a2013-07-25 11:14:07 -07001063 if os.path.exists(cache_dir):
Chris Sosadbc20082012-12-10 13:39:11 -08001064 _CleanCache(cache_dir, options.clear_cache)
Chris Sosa6b8c3742011-01-31 12:12:17 -08001065 else:
1066 os.makedirs(cache_dir)
Don Garrettf90edf02010-11-16 17:36:14 -08001067
Chris Sosadbc20082012-12-10 13:39:11 -08001068 _Log('Using cache directory %s' % cache_dir)
Gilad Arnoldc65330c2012-09-20 15:17:48 -07001069 _Log('Source root is %s' % root_dir)
joychened64b222013-06-21 16:39:34 -07001070 _Log('Serving from %s' % options.static_dir)
rtc@google.com21a5ca32009-11-04 18:23:23 +00001071
Chris Sosa6a3697f2013-01-29 16:44:43 -08001072 # We allow global use here to share with cherrypy classes.
1073 # pylint: disable=W0603
Chris Sosacde6bf42012-05-31 18:36:39 -07001074 global updater
Andrew de los Reyes52620802010-04-12 13:40:07 -07001075 updater = autoupdate.Autoupdate(
1076 root_dir=root_dir,
joychened64b222013-06-21 16:39:34 -07001077 static_dir=options.static_dir,
Andrew de los Reyes52620802010-04-12 13:40:07 -07001078 urlbase=options.urlbase,
1079 test_image=options.test_image,
Chris Sosa5d342a22010-09-28 16:54:41 -07001080 forced_image=options.image,
Gilad Arnold0c9c8602012-10-02 23:58:58 -07001081 payload_path=options.payload,
Don Garrett0ad09372010-12-06 16:20:30 -08001082 proxy_port=options.proxy_port,
Chris Sosa4136e692010-10-28 23:42:37 -07001083 src_image=options.src_image,
Chris Sosa3ae4dc12013-03-29 11:47:00 -07001084 patch_kernel=options.patch_kernel,
Chris Sosa08d55a22011-01-19 16:08:02 -08001085 board=options.board,
Chris Sosa0f1ec842011-02-14 16:33:22 -08001086 copy_to_static_root=not options.exit,
1087 private_key=options.private_key,
Satoru Takabayashid733cbe2011-11-15 09:36:32 -08001088 critical_update=options.critical_update,
Gilad Arnold0c9c8602012-10-02 23:58:58 -07001089 remote_payload=options.remote_payload,
Gilad Arnolda564b4b2012-10-04 10:32:44 -07001090 max_updates=options.max_updates,
Gilad Arnold8318eac2012-10-04 12:52:23 -07001091 host_log=options.host_log,
Chris Sosa0f1ec842011-02-14 16:33:22 -08001092 )
Chris Sosa7c931362010-10-11 19:49:01 -07001093
Chris Sosa6a3697f2013-01-29 16:44:43 -08001094 if options.pregenerate_update:
1095 updater.PreGenerateUpdate()
Chris Sosa0356d3b2010-09-16 15:46:22 -07001096
J. Richard Barnette3d977b82013-04-23 11:05:19 -07001097 if options.exit:
1098 return
Chris Sosa2f1c41e2012-07-10 14:32:33 -07001099
joychen5260b9a2013-07-16 14:48:01 -07001100 _xbuddy = xbuddy.XBuddy(options.xbuddy_manage_builds,
joychenb0dfe552013-07-30 10:02:06 -07001101 options.board,
joychen5260b9a2013-07-16 14:48:01 -07001102 root_dir=root_dir,
joychenb0dfe552013-07-30 10:02:06 -07001103 static_dir=options.static_dir,
1104 )
joychen3cb228e2013-06-12 12:13:13 -07001105 dev_server = DevServerRoot(_xbuddy)
1106
1107 cherrypy.quickstart(dev_server, config=_GetConfig(options))
Chris Sosacde6bf42012-05-31 18:36:39 -07001108
1109
1110if __name__ == '__main__':
1111 main()