blob: 5e6cc0f06e34ae464ded301fd840acea115aa777 [file] [log] [blame]
Gabe Black3b567202015-09-23 14:07:59 -07001#!/usr/bin/python2
Chris Sosa7c931362010-10-11 19:49:01 -07002
Chris Sosa781ba6d2012-04-11 12:44:43 -07003# Copyright (c) 2009-2012 The Chromium OS Authors. All rights reserved.
rtc@google.comded22402009-10-26 22:36:21 +00004# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
Chris Sosa3ae4dc12013-03-29 11:47:00 -07007"""Chromium OS development server that can be used for all forms of update.
8
9This devserver can be used to perform system-wide autoupdate and update
10of specific portage packages on devices running Chromium OS derived operating
11systems. It mainly operates in two modes:
12
131) archive mode: In this mode, the devserver is configured to stage and
14serve artifacts from Google Storage using the credentials provided to it before
15it is run. The easiest way to understand this is that the devserver is
16functioning as a local cache for artifacts produced and uploaded by build
17servers. Users of this form of devserver can either download the artifacts
18from the devservers static directory OR use the update RPC to perform a
19system-wide autoupdate. Archive mode is always active.
20
212) artifact-generation mode: in this mode, the devserver will attempt to
22generate update payloads and build artifacts when requested. This mode only
23works in the Chromium OS chroot as it uses build tools only present in the
24chroot (emerge, cros_generate_update_payload, etc.). By default, when a device
25requests an update from this form of devserver, the devserver will attempt to
26discover if a more recent build of the board has been built by the developer
27and generate a payload that the requested system can autoupdate to. In addition,
28it accepts gmerge requests from devices that will stage the newest version of
joychen84d13772013-08-06 09:17:23 -070029a particular package from a developer's chroot onto a requesting device.
Chris Sosa3ae4dc12013-03-29 11:47:00 -070030
31For example:
32gmerge gmerge -d <devserver_url>
33
34devserver will see if a newer package of gmerge is available. If gmerge is
35cros_work'd on, it will re-build gmerge. After this, gmerge will install that
36version of gmerge that the devserver just created/found.
37
38For autoupdates, there are many more advanced options that can help specify
39how to update and which payload to give to a requester.
40"""
41
Gabe Black3b567202015-09-23 14:07:59 -070042from __future__ import print_function
Chris Sosa7c931362010-10-11 19:49:01 -070043
Gilad Arnold55a2a372012-10-02 09:46:32 -070044import json
Sean O'Connor14b6a0a2010-03-20 23:23:48 -070045import optparse
rtc@google.comded22402009-10-26 22:36:21 +000046import os
Scott Zawalski4647ce62012-01-03 17:17:28 -050047import re
Simran Basi4baad082013-02-14 13:39:18 -080048import shutil
xixuan52c2fba2016-05-20 17:02:48 -070049import signal
Mandeep Singh Baines38dcdda2012-12-07 17:55:33 -080050import socket
Chris Masone816e38c2012-05-02 12:22:36 -070051import subprocess
J. Richard Barnette3d977b82013-04-23 11:05:19 -070052import sys
Chris Masone816e38c2012-05-02 12:22:36 -070053import tempfile
Dan Shi59ae7092013-06-04 14:37:27 -070054import threading
Dan Shiafd0e492015-05-27 14:23:51 -070055import time
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -070056import types
J. Richard Barnette3d977b82013-04-23 11:05:19 -070057from logging import handlers
58
59import cherrypy
Chris Sosa855b8932013-08-21 13:24:55 -070060from cherrypy import _cplogging as cplogging
61from cherrypy.process import plugins
rtc@google.comded22402009-10-26 22:36:21 +000062
Chris Sosa0356d3b2010-09-16 15:46:22 -070063import autoupdate
Dan Shi2f136862016-02-11 15:38:38 -080064import artifact_info
Chris Sosa75490802013-09-30 17:21:45 -070065import build_artifact
Gilad Arnold11fbef42014-02-10 11:04:13 -080066import cherrypy_ext
Gilad Arnoldc65330c2012-09-20 15:17:48 -070067import common_util
Simran Basief83d6a2014-08-28 14:32:01 -070068import devserver_constants
Chris Sosa47a7d4e2012-03-28 11:26:55 -070069import downloader
Chris Sosa7cd23202013-10-15 17:22:57 -070070import gsutil_util
Gilad Arnoldc65330c2012-09-20 15:17:48 -070071import log_util
joychen3cb228e2013-06-12 12:13:13 -070072import xbuddy
Gilad Arnoldc65330c2012-09-20 15:17:48 -070073
Gilad Arnoldc65330c2012-09-20 15:17:48 -070074# Module-local log function.
Chris Sosa6a3697f2013-01-29 16:44:43 -080075def _Log(message, *args):
76 return log_util.LogWithTag('DEVSERVER', message, *args)
Chris Sosa0356d3b2010-09-16 15:46:22 -070077
Dan Shiafd0e492015-05-27 14:23:51 -070078try:
79 import psutil
80except ImportError:
81 # Ignore psutil import failure. This is for backwards compatibility, so
82 # "cros flash" can still update duts with build without psutil installed.
83 # The reason is that, during cros flash, local devserver code is copied over
84 # to DUT, and devserver will be running inside DUT to stage the build.
85 _Log('Python module psutil is not installed, devserver load data will not be '
86 'collected')
87 psutil = None
Dan Shi94dcbe82015-06-08 20:51:13 -070088except OSError as e:
89 # Ignore error like following. psutil may not work properly in builder. Ignore
90 # the error as load information of devserver is not used in builder.
91 # OSError: [Errno 2] No such file or directory: '/dev/pts/0'
92 _Log('psutil is failed to be imported, error: %s. devserver load data will '
93 'not be collected.', e)
94 psutil = None
95
xixuanac89ce82016-11-30 16:48:20 -080096# Use try-except to skip unneccesary import for simple use case, eg. running
97# devserver on host.
98try:
99 import cros_update
100 import cros_update_progress
101except ImportError as e:
102 _Log('cros_update cannot be imported: %r', e)
103 cros_update = None
104 cros_update_progress = None
105
106# only import setup_chromite before chromite import.
107import setup_chromite # pylint: disable=unused-import
108try:
109 from chromite.lib.paygen import gspaths
110except ImportError as e:
111 _Log('chromite cannot be imported: %r', e)
112 gspaths = None
113
Dan Shi72b16132015-10-08 12:10:33 -0700114try:
115 import android_build
116except ImportError as e:
117 # Ignore android_build import failure. This is to support devserver running
118 # inside a ChromeOS device triggered by cros flash. Most ChromeOS test images
119 # do not have google-api-python-client module and they don't need to support
120 # Android updating, therefore, ignore the import failure here.
Dan Shi72b16132015-10-08 12:10:33 -0700121 android_build = None
Frank Farzan40160872011-12-12 18:39:18 -0800122
Chris Sosa417e55d2011-01-25 16:40:48 -0800123CACHED_ENTRIES = 12
Don Garrettf90edf02010-11-16 17:36:14 -0800124
Simran Basi4baad082013-02-14 13:39:18 -0800125TELEMETRY_FOLDER = 'telemetry_src'
126TELEMETRY_DEPS = ['dep-telemetry_dep.tar.bz2',
127 'dep-page_cycler_dep.tar.bz2',
Simran Basi0d078682013-03-22 16:40:04 -0700128 'dep-chrome_test.tar.bz2',
129 'dep-perf_data_dep.tar.bz2']
Simran Basi4baad082013-02-14 13:39:18 -0800130
Chris Sosa0356d3b2010-09-16 15:46:22 -0700131# Sets up global to share between classes.
rtc@google.com21a5ca32009-11-04 18:23:23 +0000132updater = None
rtc@google.comded22402009-10-26 22:36:21 +0000133
xixuan3d48bff2017-01-30 19:00:09 -0800134# Log rotation parameters. These settings correspond to twice a day once
135# devserver is started, with about two weeks (28 backup files) of old logs
136# kept for backup.
J. Richard Barnette3d977b82013-04-23 11:05:19 -0700137#
xixuan3d48bff2017-01-30 19:00:09 -0800138# For more, see the documentation in standard python library for
J. Richard Barnette3d977b82013-04-23 11:05:19 -0700139# logging.handlers.TimedRotatingFileHandler
xixuan3d48bff2017-01-30 19:00:09 -0800140_LOG_ROTATION_TIME = 'H'
141_LOG_ROTATION_INTERVAL = 12 # hours
142_LOG_ROTATION_BACKUP = 28 # backup counts
J. Richard Barnette3d977b82013-04-23 11:05:19 -0700143
Dan Shiafd0e492015-05-27 14:23:51 -0700144# Number of seconds between the collection of disk and network IO counters.
145STATS_INTERVAL = 10.0
Frank Farzan40160872011-12-12 18:39:18 -0800146
xixuan52c2fba2016-05-20 17:02:48 -0700147# Auto-update parameters
148
149# Error msg for missing key in CrOS auto-update.
150KEY_ERROR_MSG = 'Key Error in cmd %s: %s= is required'
151
152# Command of running auto-update.
153AUTO_UPDATE_CMD = '/usr/bin/python -u %s -d %s -b %s --static_dir %s'
154
155
Chris Sosa9164ca32012-03-28 11:04:50 -0700156class DevServerError(Exception):
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700157 """Exception class used by this module."""
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700158
159
Dan Shiafd0e492015-05-27 14:23:51 -0700160def require_psutil():
Gabe Black3b567202015-09-23 14:07:59 -0700161 """Decorator for functions require psutil to run."""
Dan Shiafd0e492015-05-27 14:23:51 -0700162 def deco_require_psutil(func):
163 """Wrapper of the decorator function.
164
Gabe Black3b567202015-09-23 14:07:59 -0700165 Args:
166 func: function to be called.
Dan Shiafd0e492015-05-27 14:23:51 -0700167 """
168 def func_require_psutil(*args, **kwargs):
169 """Decorator for functions require psutil to run.
170
171 If psutil is not installed, skip calling the function.
172
Gabe Black3b567202015-09-23 14:07:59 -0700173 Args:
174 *args: arguments for function to be called.
175 **kwargs: keyword arguments for function to be called.
Dan Shiafd0e492015-05-27 14:23:51 -0700176 """
177 if psutil:
178 return func(*args, **kwargs)
179 else:
180 _Log('Python module psutil is not installed. Function call %s is '
181 'skipped.' % func)
182 return func_require_psutil
183 return deco_require_psutil
184
185
Gabe Black3b567202015-09-23 14:07:59 -0700186def _canonicalize_archive_url(archive_url):
187 """Canonicalizes archive_url strings.
188
189 Raises:
190 DevserverError: if archive_url is not set.
191 """
192 if archive_url:
193 if not archive_url.startswith('gs://'):
194 raise DevServerError("Archive URL isn't from Google Storage (%s) ." %
195 archive_url)
196
197 return archive_url.rstrip('/')
198 else:
199 raise DevServerError("Must specify an archive_url in the request")
200
201
202def _canonicalize_local_path(local_path):
203 """Canonicalizes |local_path| strings.
204
205 Raises:
206 DevserverError: if |local_path| is not set.
207 """
208 # Restrict staging of local content to only files within the static
209 # directory.
210 local_path = os.path.abspath(local_path)
211 if not local_path.startswith(updater.static_dir):
212 raise DevServerError('Local path %s must be a subdirectory of the static'
213 ' directory: %s' % (local_path, updater.static_dir))
214
215 return local_path.rstrip('/')
216
217
218def _get_artifacts(kwargs):
219 """Returns a tuple of named and file artifacts given the stage rpc kwargs.
220
221 Raises:
222 DevserverError if no artifacts would be returned.
223 """
224 artifacts = kwargs.get('artifacts')
225 files = kwargs.get('files')
226 if not artifacts and not files:
227 raise DevServerError('No artifacts specified.')
228
229 # Note we NEED to coerce files to a string as we get raw unicode from
230 # cherrypy and we treat files as strings elsewhere in the code.
231 return (str(artifacts).split(',') if artifacts else [],
232 str(files).split(',') if files else [])
233
234
Dan Shi61305df2015-10-26 16:52:35 -0700235def _is_android_build_request(kwargs):
236 """Check if a devserver call is for Android build, based on the arguments.
237
238 This method exams the request's arguments (os_type) to determine if the
239 request is for Android build. If os_type is set to `android`, returns True.
240 If os_type is not set or has other values, returns False.
241
242 Args:
243 kwargs: Keyword arguments for the request.
244
245 Returns:
246 True if the request is for Android build. False otherwise.
247 """
248 os_type = kwargs.get('os_type', None)
249 return os_type == 'android'
250
251
Gabe Black3b567202015-09-23 14:07:59 -0700252def _get_downloader(kwargs):
253 """Returns the downloader based on passed in arguments.
254
255 Args:
256 kwargs: Keyword arguments for the request.
257 """
258 local_path = kwargs.get('local_path')
259 if local_path:
260 local_path = _canonicalize_local_path(local_path)
261
262 dl = None
263 if local_path:
264 dl = downloader.LocalDownloader(updater.static_dir, local_path)
265
Dan Shi61305df2015-10-26 16:52:35 -0700266 if not _is_android_build_request(kwargs):
Gabe Black3b567202015-09-23 14:07:59 -0700267 archive_url = kwargs.get('archive_url')
268 if not archive_url and not local_path:
269 raise DevServerError('Requires archive_url or local_path to be '
270 'specified.')
271 if archive_url and local_path:
272 raise DevServerError('archive_url and local_path can not both be '
273 'specified.')
274 if not dl:
275 archive_url = _canonicalize_archive_url(archive_url)
276 dl = downloader.GoogleStorageDownloader(updater.static_dir, archive_url)
277 elif not dl:
278 target = kwargs.get('target', None)
Dan Shi72b16132015-10-08 12:10:33 -0700279 branch = kwargs.get('branch', None)
Dan Shi61305df2015-10-26 16:52:35 -0700280 build_id = kwargs.get('build_id', None)
281 if not target or not branch or not build_id:
Dan Shi72b16132015-10-08 12:10:33 -0700282 raise DevServerError(
Dan Shi61305df2015-10-26 16:52:35 -0700283 'target, branch, build ID must all be specified for downloading '
284 'Android build.')
Dan Shi72b16132015-10-08 12:10:33 -0700285 dl = downloader.AndroidBuildDownloader(updater.static_dir, branch, build_id,
286 target)
Gabe Black3b567202015-09-23 14:07:59 -0700287
288 return dl
289
290
291def _get_downloader_and_factory(kwargs):
292 """Returns the downloader and artifact factory based on passed in arguments.
293
294 Args:
295 kwargs: Keyword arguments for the request.
296 """
297 artifacts, files = _get_artifacts(kwargs)
298 dl = _get_downloader(kwargs)
299
300 if (isinstance(dl, downloader.GoogleStorageDownloader) or
301 isinstance(dl, downloader.LocalDownloader)):
302 factory_class = build_artifact.ChromeOSArtifactFactory
Dan Shi72b16132015-10-08 12:10:33 -0700303 elif isinstance(dl, downloader.AndroidBuildDownloader):
Gabe Black3b567202015-09-23 14:07:59 -0700304 factory_class = build_artifact.AndroidArtifactFactory
305 else:
306 raise DevServerError('Unrecognized value for downloader type: %s' %
307 type(dl))
308
309 factory = factory_class(dl.GetBuildDir(), artifacts, files, dl.GetBuild())
310
311 return dl, factory
312
313
Scott Zawalski4647ce62012-01-03 17:17:28 -0500314def _LeadingWhiteSpaceCount(string):
315 """Count the amount of leading whitespace in a string.
316
317 Args:
318 string: The string to count leading whitespace in.
Don Garrettf84631a2014-01-07 18:21:26 -0800319
Scott Zawalski4647ce62012-01-03 17:17:28 -0500320 Returns:
321 number of white space chars before characters start.
322 """
Gabe Black3b567202015-09-23 14:07:59 -0700323 matched = re.match(r'^\s+', string)
Scott Zawalski4647ce62012-01-03 17:17:28 -0500324 if matched:
325 return len(matched.group())
326
327 return 0
328
329
330def _PrintDocStringAsHTML(func):
331 """Make a functions docstring somewhat HTML style.
332
333 Args:
334 func: The function to return the docstring from.
Don Garrettf84631a2014-01-07 18:21:26 -0800335
Scott Zawalski4647ce62012-01-03 17:17:28 -0500336 Returns:
337 A string that is somewhat formated for a web browser.
338 """
339 # TODO(scottz): Make this parse Args/Returns in a prettier way.
340 # Arguments could be bolded and indented etc.
341 html_doc = []
342 for line in func.__doc__.splitlines():
343 leading_space = _LeadingWhiteSpaceCount(line)
344 if leading_space > 0:
Chris Sosa47a7d4e2012-03-28 11:26:55 -0700345 line = '&nbsp;' * leading_space + line
Scott Zawalski4647ce62012-01-03 17:17:28 -0500346
347 html_doc.append('<BR>%s' % line)
348
349 return '\n'.join(html_doc)
350
351
Simran Basief83d6a2014-08-28 14:32:01 -0700352def _GetUpdateTimestampHandler(static_dir):
353 """Returns a handler to update directory staged.timestamp.
354
355 This handler resets the stage.timestamp whenever static content is accessed.
356
357 Args:
358 static_dir: Directory from which static content is being staged.
359
360 Returns:
361 A cherrypy handler to update the timestamp of accessed content.
362 """
363 def UpdateTimestampHandler():
364 if not '404' in cherrypy.response.status:
365 build_match = re.match(devserver_constants.STAGED_BUILD_REGEX,
366 cherrypy.request.path_info)
367 if build_match:
368 build_dir = os.path.join(static_dir, build_match.group('build'))
369 downloader.Downloader.TouchTimestampForStaged(build_dir)
370 return UpdateTimestampHandler
371
372
Chris Sosa7c931362010-10-11 19:49:01 -0700373def _GetConfig(options):
374 """Returns the configuration for the devserver."""
Mandeep Singh Baines38dcdda2012-12-07 17:55:33 -0800375
Mandeep Singh Baines38dcdda2012-12-07 17:55:33 -0800376 socket_host = '::'
Yu-Ju Hongc8d4af32013-11-12 15:14:26 -0800377 # Fall back to IPv4 when python is not configured with IPv6.
378 if not socket.has_ipv6:
Mandeep Singh Baines38dcdda2012-12-07 17:55:33 -0800379 socket_host = '0.0.0.0'
380
Simran Basief83d6a2014-08-28 14:32:01 -0700381 # Adds the UpdateTimestampHandler to cherrypy's tools. This tools executes
382 # on the on_end_resource hook. This hook is called once processing is
383 # complete and the response is ready to be returned.
384 cherrypy.tools.update_timestamp = cherrypy.Tool(
385 'on_end_resource', _GetUpdateTimestampHandler(options.static_dir))
386
Gabe Black3b567202015-09-23 14:07:59 -0700387 base_config = {'global':
388 {'server.log_request_headers': True,
389 'server.protocol_version': 'HTTP/1.1',
390 'server.socket_host': socket_host,
391 'server.socket_port': int(options.port),
392 'response.timeout': 6000,
393 'request.show_tracebacks': True,
394 'server.socket_timeout': 60,
395 'server.thread_pool': 2,
396 'engine.autoreload.on': False,
397 },
398 '/api':
399 {
400 # Gets rid of cherrypy parsing post file for args.
401 'request.process_request_body': False,
402 },
403 '/build':
404 {'response.timeout': 100000,
405 },
406 '/update':
407 {
408 # Gets rid of cherrypy parsing post file for args.
409 'request.process_request_body': False,
410 'response.timeout': 10000,
411 },
412 # Sets up the static dir for file hosting.
413 '/static':
414 {'tools.staticdir.dir': options.static_dir,
415 'tools.staticdir.on': True,
416 'response.timeout': 10000,
417 'tools.update_timestamp.on': True,
418 },
419 }
Chris Sosa5f118ef2012-07-12 11:37:50 -0700420 if options.production:
Alex Miller93beca52013-07-30 19:25:09 -0700421 base_config['global'].update({'server.thread_pool': 150})
Chris Sosa7cd23202013-10-15 17:22:57 -0700422 # TODO(sosa): Do this more cleanly.
423 gsutil_util.GSUTIL_ATTEMPTS = 5
Scott Zawalski1c5e7cd2012-02-27 13:12:52 -0500424
Chris Sosa7c931362010-10-11 19:49:01 -0700425 return base_config
rtc@google.com64244662009-11-12 00:52:08 +0000426
Darin Petkove17164a2010-08-11 13:24:41 -0700427
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700428def _GetRecursiveMemberObject(root, member_list):
429 """Returns an object corresponding to a nested member list.
430
431 Args:
432 root: the root object to search
433 member_list: list of nested members to search
Don Garrettf84631a2014-01-07 18:21:26 -0800434
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700435 Returns:
436 An object corresponding to the member name list; None otherwise.
437 """
438 for member in member_list:
439 next_root = root.__class__.__dict__.get(member)
440 if not next_root:
441 return None
442 root = next_root
443 return root
444
445
446def _IsExposed(name):
447 """Returns True iff |name| has an `exposed' attribute and it is set."""
448 return hasattr(name, 'exposed') and name.exposed
449
450
Gilad Arnold748c8322012-10-12 09:51:35 -0700451def _GetExposedMethod(root, nested_member, ignored=None):
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700452 """Returns a CherryPy-exposed method, if such exists.
453
454 Args:
455 root: the root object for searching
456 nested_member: a slash-joined path to the nested member
457 ignored: method paths to be ignored
Don Garrettf84631a2014-01-07 18:21:26 -0800458
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700459 Returns:
460 A function object corresponding to the path defined by |member_list| from
461 the |root| object, if the function is exposed and not ignored; None
462 otherwise.
463 """
Gilad Arnold748c8322012-10-12 09:51:35 -0700464 method = (not (ignored and nested_member in ignored) and
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700465 _GetRecursiveMemberObject(root, nested_member.split('/')))
Gabe Black3b567202015-09-23 14:07:59 -0700466 if method and type(method) == types.FunctionType and _IsExposed(method):
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700467 return method
468
469
Gilad Arnold748c8322012-10-12 09:51:35 -0700470def _FindExposedMethods(root, prefix, unlisted=None):
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700471 """Finds exposed CherryPy methods.
472
473 Args:
474 root: the root object for searching
475 prefix: slash-joined chain of members leading to current object
476 unlisted: URLs to be excluded regardless of their exposed status
Don Garrettf84631a2014-01-07 18:21:26 -0800477
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700478 Returns:
479 List of exposed URLs that are not unlisted.
480 """
481 method_list = []
482 for member in sorted(root.__class__.__dict__.keys()):
483 prefixed_member = prefix + '/' + member if prefix else member
Gilad Arnold748c8322012-10-12 09:51:35 -0700484 if unlisted and prefixed_member in unlisted:
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -0700485 continue
486 member_obj = root.__class__.__dict__[member]
487 if _IsExposed(member_obj):
488 if type(member_obj) == types.FunctionType:
489 method_list.append(prefixed_member)
490 else:
491 method_list += _FindExposedMethods(
492 member_obj, prefixed_member, unlisted)
493 return method_list
494
495
xixuan52c2fba2016-05-20 17:02:48 -0700496def _check_base_args_for_auto_update(kwargs):
xixuanac89ce82016-11-30 16:48:20 -0800497 """Check basic args required for auto-update.
498
499 Args:
500 kwargs: the parameters to be checked.
501
502 Raises:
503 DevServerHTTPError if required parameters don't exist in kwargs.
504 """
xixuan52c2fba2016-05-20 17:02:48 -0700505 if 'host_name' not in kwargs:
506 raise common_util.DevServerHTTPError(KEY_ERROR_MSG % 'host_name')
507
508 if 'build_name' not in kwargs:
509 raise common_util.DevServerHTTPError(KEY_ERROR_MSG % 'build_name')
510
511
512def _parse_boolean_arg(kwargs, key):
xixuanac89ce82016-11-30 16:48:20 -0800513 """Parse boolean arg from kwargs.
514
515 Args:
516 kwargs: the parameters to be checked.
517 key: the key to be parsed.
518
519 Returns:
520 The boolean value of kwargs[key], or False if key doesn't exist in kwargs.
521
522 Raises:
523 DevServerHTTPError if kwargs[key] is not a boolean variable.
524 """
xixuan52c2fba2016-05-20 17:02:48 -0700525 if key in kwargs:
526 if kwargs[key] == 'True':
527 return True
528 elif kwargs[key] == 'False':
529 return False
530 else:
531 raise common_util.DevServerHTTPError(
532 'The value for key %s is not boolean.' % key)
533 else:
534 return False
535
xixuan447ad9d2017-02-28 14:46:20 -0800536
xixuanac89ce82016-11-30 16:48:20 -0800537def _parse_string_arg(kwargs, key):
538 """Parse string arg from kwargs.
539
540 Args:
541 kwargs: the parameters to be checked.
542 key: the key to be parsed.
543
544 Returns:
545 The string value of kwargs[key], or None if key doesn't exist in kwargs.
546 """
547 if key in kwargs:
548 return kwargs[key]
549 else:
550 return None
551
xixuan447ad9d2017-02-28 14:46:20 -0800552
xixuanac89ce82016-11-30 16:48:20 -0800553def _build_uri_from_build_name(build_name):
554 """Get build url from a given build name.
555
556 Args:
557 build_name: the build name to be parsed, whose format is
558 'board/release_version'.
559
560 Returns:
561 The release_archive_url on Google Storage for this build name.
562 """
563 return gspaths.ChromeosReleases.BuildUri(
564 cros_update.STABLE_BUILD_CHANNEL, build_name.split('/')[0],
565 build_name.split('/')[1])
xixuan52c2fba2016-05-20 17:02:48 -0700566
xixuan447ad9d2017-02-28 14:46:20 -0800567
568def _clear_process(host_name, pid):
569 """Clear AU process for given hostname and pid.
570
571 This clear includes:
572 1. kill process if it's alive.
573 2. delete the track status file of this process.
574 3. delete the executing log file of this process.
575
576 Args:
577 host_name: the host to execute auto-update.
578 pid: the background auto-update process id.
579 """
580 if cros_update_progress.IsProcessAlive(pid):
581 os.killpg(int(pid), signal.SIGKILL)
582
583 cros_update_progress.DelTrackStatusFile(host_name, pid)
584 cros_update_progress.DelExecuteLogFile(host_name, pid)
585
586
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700587class ApiRoot(object):
588 """RESTful API for Dev Server information."""
589 exposed = True
590
591 @cherrypy.expose
592 def hostinfo(self, ip):
593 """Returns a JSON dictionary containing information about the given ip.
594
Gilad Arnold1b908392012-10-05 11:36:27 -0700595 Args:
596 ip: address of host whose info is requested
Don Garrettf84631a2014-01-07 18:21:26 -0800597
Gilad Arnold1b908392012-10-05 11:36:27 -0700598 Returns:
599 A JSON dictionary containing all or some of the following fields:
600 last_event_type (int): last update event type received
601 last_event_status (int): last update event status received
602 last_known_version (string): last known version reported in update ping
603 forced_update_label (string): update label to force next update ping to
604 use, set by setnextupdate
605 See the OmahaEvent class in update_engine/omaha_request_action.h for
606 event type and status code definitions. If the ip does not exist an empty
607 string is returned.
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700608
Gilad Arnold1b908392012-10-05 11:36:27 -0700609 Example URL:
610 http://myhost/api/hostinfo?ip=192.168.1.5
611 """
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700612 return updater.HandleHostInfoPing(ip)
613
614 @cherrypy.expose
Gilad Arnold286a0062012-01-12 13:47:02 -0800615 def hostlog(self, ip):
Gilad Arnold1b908392012-10-05 11:36:27 -0700616 """Returns a JSON object containing a log of host event.
617
618 Args:
619 ip: address of host whose event log is requested, or `all'
Don Garrettf84631a2014-01-07 18:21:26 -0800620
Gilad Arnold1b908392012-10-05 11:36:27 -0700621 Returns:
622 A JSON encoded list (log) of dictionaries (events), each of which
623 containing a `timestamp' and other event fields, as described under
624 /api/hostinfo.
625
626 Example URL:
627 http://myhost/api/hostlog?ip=192.168.1.5
628 """
Gilad Arnold286a0062012-01-12 13:47:02 -0800629 return updater.HandleHostLogPing(ip)
630
631 @cherrypy.expose
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700632 def setnextupdate(self, ip):
633 """Allows the response to the next update ping from a host to be set.
634
635 Takes the IP of the host and an update label as normally provided to the
Gilad Arnold1b908392012-10-05 11:36:27 -0700636 /update command.
637 """
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700638 body_length = int(cherrypy.request.headers['Content-Length'])
639 label = cherrypy.request.rfile.read(body_length)
640
641 if label:
642 label = label.strip()
643 if label:
644 return updater.HandleSetUpdatePing(ip, label)
Chris Sosa4b951602014-04-09 20:26:07 -0700645 raise common_util.DevServerHTTPError(400, 'No label provided.')
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700646
647
Gilad Arnold55a2a372012-10-02 09:46:32 -0700648 @cherrypy.expose
Don Garrettf84631a2014-01-07 18:21:26 -0800649 def fileinfo(self, *args):
Gilad Arnold55a2a372012-10-02 09:46:32 -0700650 """Returns information about a given staged file.
651
652 Args:
Don Garrettf84631a2014-01-07 18:21:26 -0800653 args: path to the file inside the server's static staging directory
654
Gilad Arnold55a2a372012-10-02 09:46:32 -0700655 Returns:
656 A JSON encoded dictionary with information about the said file, which may
657 contain the following keys/values:
Gilad Arnold1b908392012-10-05 11:36:27 -0700658 size (int): the file size in bytes
659 sha1 (string): a base64 encoded SHA1 hash
660 sha256 (string): a base64 encoded SHA256 hash
661
662 Example URL:
663 http://myhost/api/fileinfo/some/path/to/file
Gilad Arnold55a2a372012-10-02 09:46:32 -0700664 """
Don Garrettf84631a2014-01-07 18:21:26 -0800665 file_path = os.path.join(updater.static_dir, *args)
Gilad Arnold55a2a372012-10-02 09:46:32 -0700666 if not os.path.exists(file_path):
667 raise DevServerError('file not found: %s' % file_path)
668 try:
669 file_size = os.path.getsize(file_path)
670 file_sha1 = common_util.GetFileSha1(file_path)
671 file_sha256 = common_util.GetFileSha256(file_path)
672 except os.error, e:
673 raise DevServerError('failed to get info for file %s: %s' %
Gilad Arnolde74b3812013-04-22 11:27:38 -0700674 (file_path, e))
675
676 is_delta = autoupdate.Autoupdate.IsDeltaFormatFile(file_path)
677
678 return json.dumps({
679 autoupdate.Autoupdate.SIZE_ATTR: file_size,
680 autoupdate.Autoupdate.SHA1_ATTR: file_sha1,
681 autoupdate.Autoupdate.SHA256_ATTR: file_sha256,
682 autoupdate.Autoupdate.ISDELTA_ATTR: is_delta
683 })
Gilad Arnold55a2a372012-10-02 09:46:32 -0700684
Chris Sosa76e44b92013-01-31 12:11:38 -0800685
David Rochberg7c79a812011-01-19 14:24:45 -0500686class DevServerRoot(object):
Chris Sosa7c931362010-10-11 19:49:01 -0700687 """The Root Class for the Dev Server.
688
689 CherryPy works as follows:
690 For each method in this class, cherrpy interprets root/path
691 as a call to an instance of DevServerRoot->method_name. For example,
692 a call to http://myhost/build will call build. CherryPy automatically
693 parses http args and places them as keyword arguments in each method.
694 For paths http://myhost/update/dir1/dir2, you can use *args so that
695 cherrypy uses the update method and puts the extra paths in args.
696 """
Gilad Arnoldf8f769f2012-09-24 08:43:01 -0700697 # Method names that should not be listed on the index page.
698 _UNLISTED_METHODS = ['index', 'doc']
699
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700700 api = ApiRoot()
Chris Sosa7c931362010-10-11 19:49:01 -0700701
Dan Shi59ae7092013-06-04 14:37:27 -0700702 # Number of threads that devserver is staging images.
703 _staging_thread_count = 0
704 # Lock used to lock increasing/decreasing count.
705 _staging_thread_count_lock = threading.Lock()
706
Dan Shiafd0e492015-05-27 14:23:51 -0700707 @require_psutil()
708 def _refresh_io_stats(self):
709 """A call running in a thread to update IO stats periodically."""
710 prev_disk_io_counters = psutil.disk_io_counters()
711 prev_network_io_counters = psutil.net_io_counters()
712 prev_read_time = time.time()
713 while True:
714 time.sleep(STATS_INTERVAL)
715 now = time.time()
716 interval = now - prev_read_time
717 prev_read_time = now
718 # Disk IO is for all disks.
719 disk_io_counters = psutil.disk_io_counters()
720 network_io_counters = psutil.net_io_counters()
721
722 self.disk_read_bytes_per_sec = (
723 disk_io_counters.read_bytes -
724 prev_disk_io_counters.read_bytes)/interval
725 self.disk_write_bytes_per_sec = (
726 disk_io_counters.write_bytes -
727 prev_disk_io_counters.write_bytes)/interval
728 prev_disk_io_counters = disk_io_counters
729
730 self.network_sent_bytes_per_sec = (
731 network_io_counters.bytes_sent -
732 prev_network_io_counters.bytes_sent)/interval
733 self.network_recv_bytes_per_sec = (
734 network_io_counters.bytes_recv -
735 prev_network_io_counters.bytes_recv)/interval
736 prev_network_io_counters = network_io_counters
737
738 @require_psutil()
739 def _start_io_stat_thread(self):
Gabe Black3b567202015-09-23 14:07:59 -0700740 """Start the thread to collect IO stats."""
Dan Shiafd0e492015-05-27 14:23:51 -0700741 thread = threading.Thread(target=self._refresh_io_stats)
742 thread.daemon = True
743 thread.start()
744
joychen3cb228e2013-06-12 12:13:13 -0700745 def __init__(self, _xbuddy):
Nick Sanders7dcaa2e2011-08-04 15:20:41 -0700746 self._builder = None
Simran Basi4baad082013-02-14 13:39:18 -0800747 self._telemetry_lock_dict = common_util.LockDict()
joychen3cb228e2013-06-12 12:13:13 -0700748 self._xbuddy = _xbuddy
David Rochberg7c79a812011-01-19 14:24:45 -0500749
Dan Shiafd0e492015-05-27 14:23:51 -0700750 # Cache of disk IO stats, a thread refresh the stats every 10 seconds.
751 # lock is not used for these variables as the only thread writes to these
752 # variables is _refresh_io_stats.
753 self.disk_read_bytes_per_sec = 0
754 self.disk_write_bytes_per_sec = 0
755 # Cache of network IO stats.
756 self.network_sent_bytes_per_sec = 0
757 self.network_recv_bytes_per_sec = 0
758 self._start_io_stat_thread()
759
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700760 @cherrypy.expose
David Rochberg7c79a812011-01-19 14:24:45 -0500761 def build(self, board, pkg, **kwargs):
Chris Sosa7c931362010-10-11 19:49:01 -0700762 """Builds the package specified."""
Nick Sanders7dcaa2e2011-08-04 15:20:41 -0700763 import builder
764 if self._builder is None:
765 self._builder = builder.Builder()
David Rochberg7c79a812011-01-19 14:24:45 -0500766 return self._builder.Build(board, pkg, kwargs)
Chris Sosa7c931362010-10-11 19:49:01 -0700767
Dale Curtisc9aaf3a2011-08-09 15:47:40 -0700768 @cherrypy.expose
Dan Shif8eb0d12013-08-01 17:52:06 -0700769 def is_staged(self, **kwargs):
770 """Check if artifacts have been downloaded.
771
Chris Sosa6b0c6172013-08-05 17:01:33 -0700772 async: True to return without waiting for download to complete.
773 artifacts: Comma separated list of named artifacts to download.
774 These are defined in artifact_info and have their implementation
775 in build_artifact.py.
776 files: Comma separated list of file artifacts to stage. These
777 will be available as is in the corresponding static directory with no
778 custom post-processing.
779
780 returns: True of all artifacts are staged.
Dan Shif8eb0d12013-08-01 17:52:06 -0700781
782 Example:
783 To check if autotest and test_suites are staged:
784 http://devserver_url:<port>/is_staged?archive_url=gs://your_url/path&
785 artifacts=autotest,test_suites
786 """
Gabe Black3b567202015-09-23 14:07:59 -0700787 dl, factory = _get_downloader_and_factory(kwargs)
Aviv Keshet57d18172016-06-18 20:39:09 -0700788 response = str(dl.IsStaged(factory))
789 _Log('Responding to is_staged %s request with %r', kwargs, response)
790 return response
Dan Shi59ae7092013-06-04 14:37:27 -0700791
Chris Sosa76e44b92013-01-31 12:11:38 -0800792 @cherrypy.expose
Prashanth Ba06d2d22014-03-07 15:35:19 -0800793 def list_image_dir(self, **kwargs):
794 """Take an archive url and list the contents in its staged directory.
795
796 Args:
797 kwargs:
798 archive_url: Google Storage URL for the build.
799
800 Example:
801 To list the contents of where this devserver should have staged
802 gs://image-archive/<board>-release/<build> call:
803 http://devserver_url:<port>/list_image_dir?archive_url=<gs://..>
804
805 Returns:
806 A string with information about the contents of the image directory.
807 """
Gabe Black3b567202015-09-23 14:07:59 -0700808 dl = _get_downloader(kwargs)
Prashanth Ba06d2d22014-03-07 15:35:19 -0800809 try:
Gabe Black3b567202015-09-23 14:07:59 -0700810 image_dir_contents = dl.ListBuildDir()
Prashanth Ba06d2d22014-03-07 15:35:19 -0800811 except build_artifact.ArtifactDownloadError as e:
812 return 'Cannot list the contents of staged artifacts. %s' % e
813 if not image_dir_contents:
Gabe Black3b567202015-09-23 14:07:59 -0700814 return '%s has not been staged on this devserver.' % dl.DescribeSource()
Prashanth Ba06d2d22014-03-07 15:35:19 -0800815 return image_dir_contents
816
817 @cherrypy.expose
Chris Sosa76e44b92013-01-31 12:11:38 -0800818 def stage(self, **kwargs):
Gabe Black3b567202015-09-23 14:07:59 -0700819 """Downloads and caches build artifacts.
Chris Sosa76e44b92013-01-31 12:11:38 -0800820
Gabe Black3b567202015-09-23 14:07:59 -0700821 Downloads and caches build artifacts, possibly from a Google Storage URL,
Dan Shi72b16132015-10-08 12:10:33 -0700822 or from Android's build server. Returns once these have been downloaded
Gabe Black3b567202015-09-23 14:07:59 -0700823 on the devserver. A call to this will attempt to cache non-specified
824 artifacts in the background for the given from the given URL following
825 the principle of spatial locality. Spatial locality of different
Chris Sosa76e44b92013-01-31 12:11:38 -0800826 artifacts is explicitly defined in the build_artifact module.
827
828 These artifacts will then be available from the static/ sub-directory of
829 the devserver.
830
831 Args:
832 archive_url: Google Storage URL for the build.
Simran Basi4243a862014-12-12 12:48:33 -0800833 local_path: Local path for the build.
Dan Shif8eb0d12013-08-01 17:52:06 -0700834 async: True to return without waiting for download to complete.
Chris Sosa6b0c6172013-08-05 17:01:33 -0700835 artifacts: Comma separated list of named artifacts to download.
836 These are defined in artifact_info and have their implementation
837 in build_artifact.py.
838 files: Comma separated list of files to stage. These
839 will be available as is in the corresponding static directory with no
840 custom post-processing.
Laurence Goodbyf5c958d2016-01-14 18:23:56 -0800841 clean: True to remove any previously staged artifacts first.
Chris Sosa76e44b92013-01-31 12:11:38 -0800842
843 Example:
844 To download the autotest and test suites tarballs:
845 http://devserver_url:<port>/stage?archive_url=gs://your_url/path&
846 artifacts=autotest,test_suites
847 To download the full update payload:
848 http://devserver_url:<port>/stage?archive_url=gs://your_url/path&
849 artifacts=full_payload
Chris Sosa6b0c6172013-08-05 17:01:33 -0700850 To download just a file called blah.bin:
851 http://devserver_url:<port>/stage?archive_url=gs://your_url/path&
852 files=blah.bin
Chris Sosa76e44b92013-01-31 12:11:38 -0800853
854 For both these examples, one could find these artifacts at:
joychened64b222013-06-21 16:39:34 -0700855 http://devserver_url:<port>/static/<relative_path>*
Chris Sosa76e44b92013-01-31 12:11:38 -0800856
857 Note for this example, relative path is the archive_url stripped of its
858 basename i.e. path/ in the examples above. Specific example:
859
860 gs://chromeos-image-archive/x86-mario-release/R26-3920.0.0
861
862 Will get staged to:
863
joychened64b222013-06-21 16:39:34 -0700864 http://devserver_url:<port>/static/x86-mario-release/R26-3920.0.0
Chris Sosa76e44b92013-01-31 12:11:38 -0800865 """
Gabe Black3b567202015-09-23 14:07:59 -0700866 dl, factory = _get_downloader_and_factory(kwargs)
867
Dan Shi59ae7092013-06-04 14:37:27 -0700868 with DevServerRoot._staging_thread_count_lock:
869 DevServerRoot._staging_thread_count += 1
870 try:
Laurence Goodbyf5c958d2016-01-14 18:23:56 -0800871 boolean_string = kwargs.get('clean')
872 clean = xbuddy.XBuddy.ParseBoolean(boolean_string)
873 if clean and os.path.exists(dl.GetBuildDir()):
874 _Log('Removing %s' % dl.GetBuildDir())
875 shutil.rmtree(dl.GetBuildDir())
Gabe Black3b567202015-09-23 14:07:59 -0700876 async = kwargs.get('async', False)
877 dl.Download(factory, async=async)
Dan Shi59ae7092013-06-04 14:37:27 -0700878 finally:
879 with DevServerRoot._staging_thread_count_lock:
880 DevServerRoot._staging_thread_count -= 1
Chris Sosa76e44b92013-01-31 12:11:38 -0800881 return 'Success'
Chris Sosacde6bf42012-05-31 18:36:39 -0700882
883 @cherrypy.expose
xixuan52c2fba2016-05-20 17:02:48 -0700884 def cros_au(self, **kwargs):
885 """Auto-update a CrOS DUT.
886
887 Args:
888 kwargs:
889 host_name: the hostname of the DUT to auto-update.
890 build_name: the build name for update the DUT.
891 force_update: Force an update even if the version installed is the
892 same. Default: False.
893 full_update: If True, do not run stateful update, directly force a full
894 reimage. If False, try stateful update first if the dut is already
895 installed with the same version.
896 async: Whether the auto_update function is ran in the background.
897
898 Returns:
899 A tuple includes two elements:
900 a boolean variable represents whether the auto-update process is
901 successfully started.
902 an integer represents the background auto-update process id.
903 """
904 _check_base_args_for_auto_update(kwargs)
905
906 host_name = kwargs['host_name']
907 build_name = kwargs['build_name']
908 force_update = _parse_boolean_arg(kwargs, 'force_update')
909 full_update = _parse_boolean_arg(kwargs, 'full_update')
910 async = _parse_boolean_arg(kwargs, 'async')
xixuanac89ce82016-11-30 16:48:20 -0800911 original_build = _parse_string_arg(kwargs, 'original_build')
David Haddock90e49442017-04-07 19:14:09 -0700912 payload_filename = _parse_string_arg(kwargs, 'payload_filename')
xixuan52c2fba2016-05-20 17:02:48 -0700913
914 if async:
915 path = os.path.dirname(os.path.abspath(__file__))
916 execute_file = os.path.join(path, 'cros_update.py')
917 args = (AUTO_UPDATE_CMD % (execute_file, host_name, build_name,
918 updater.static_dir))
xixuanac89ce82016-11-30 16:48:20 -0800919
920 # The original_build's format is like: link/3428.210.0
921 # The corresponding release_archive_url's format is like:
922 # gs://chromeos-releases/stable-channel/link/3428.210.0
923 if original_build:
924 release_archive_url = _build_uri_from_build_name(original_build)
925 # First staging the stateful.tgz synchronousely.
926 self.stage(files='stateful.tgz', async=False,
927 archive_url=release_archive_url)
928 args = ('%s --original_build %s' % (args, original_build))
929
xixuan52c2fba2016-05-20 17:02:48 -0700930 if force_update:
931 args = ('%s --force_update' % args)
932
933 if full_update:
934 args = ('%s --full_update' % args)
935
David Haddock90e49442017-04-07 19:14:09 -0700936 if payload_filename:
937 args = ('%s --payload_filename %s' % (args, payload_filename))
938
xixuan2a0970a2016-08-10 12:12:44 -0700939 p = subprocess.Popen([args], shell=True, preexec_fn=os.setsid)
940 pid = os.getpgid(p.pid)
xixuan52c2fba2016-05-20 17:02:48 -0700941
942 # Pre-write status in the track_status_file before the first call of
943 # 'get_au_status' to make sure that the track_status_file exists.
xixuan2a0970a2016-08-10 12:12:44 -0700944 progress_tracker = cros_update_progress.AUProgress(host_name, pid)
xixuan52c2fba2016-05-20 17:02:48 -0700945 progress_tracker.WriteStatus('CrOS update is just started.')
946
xixuan2a0970a2016-08-10 12:12:44 -0700947 return json.dumps((True, pid))
xixuan52c2fba2016-05-20 17:02:48 -0700948 else:
949 cros_update_trigger = cros_update.CrOSUpdateTrigger(
xixuanac89ce82016-11-30 16:48:20 -0800950 host_name, build_name, updater.static_dir, force_update=force_update,
951 full_update=full_update, original_build=original_build)
xixuan52c2fba2016-05-20 17:02:48 -0700952 cros_update_trigger.TriggerAU()
953
954 @cherrypy.expose
955 def get_au_status(self, **kwargs):
956 """Check if the auto-update task is finished.
957
958 It handles 4 cases:
959 1. If an error exists in the track_status_file, delete the track file and
960 raise it.
961 2. If cros-update process is finished, delete the file and return the
962 success result.
963 3. If the process is not running, delete the track file and raise an error
964 about 'the process is terminated due to unknown reason'.
965 4. If the track_status_file does not exist, kill the process if it exists,
966 and raise the IOError.
967
968 Args:
969 kwargs:
970 host_name: the hostname of the DUT to auto-update.
971 pid: the background process id of cros-update.
972
973 Returns:
xixuan28d99072016-10-06 12:24:16 -0700974 A dict with three elements:
xixuan52c2fba2016-05-20 17:02:48 -0700975 a boolean variable represents whether the auto-update process is
976 finished.
977 a string represents the current auto-update process status.
978 For example, 'Transfer Devserver/Stateful Update Package'.
xixuan28d99072016-10-06 12:24:16 -0700979 a detailed error message paragraph if there exists an Auto-Update
980 error, in which the last line shows the main exception. Empty
981 string otherwise.
xixuan52c2fba2016-05-20 17:02:48 -0700982 """
983 if 'host_name' not in kwargs:
984 raise common_util.DevServerHTTPError((KEY_ERROR_MSG % 'host_name'))
985
986 if 'pid' not in kwargs:
987 raise common_util.DevServerHTTPError((KEY_ERROR_MSG % 'pid'))
988
989 host_name = kwargs['host_name']
990 pid = kwargs['pid']
991 progress_tracker = cros_update_progress.AUProgress(host_name, pid)
992
xixuan28d99072016-10-06 12:24:16 -0700993 result_dict = {'finished': False, 'status': '', 'detailed_error_msg': ''}
xixuan52c2fba2016-05-20 17:02:48 -0700994 try:
995 result = progress_tracker.ReadStatus()
996 if result.startswith(cros_update_progress.ERROR_TAG):
xixuan28d99072016-10-06 12:24:16 -0700997 result_dict['detailed_error_msg'] = result[len(
998 cros_update_progress.ERROR_TAG):]
xixuan28681fd2016-11-23 11:13:56 -0800999 elif result == cros_update_progress.FINISHED:
xixuan28d99072016-10-06 12:24:16 -07001000 result_dict['finished'] = True
1001 result_dict['status'] = result
xixuan28681fd2016-11-23 11:13:56 -08001002 elif not cros_update_progress.IsProcessAlive(pid):
xixuan28d99072016-10-06 12:24:16 -07001003 result_dict['detailed_error_msg'] = (
1004 'Cros_update process terminated midway due to unknown reason. '
1005 'Last update status was %s' % result)
xixuan28681fd2016-11-23 11:13:56 -08001006 else:
1007 result_dict['status'] = result
1008 except IOError as e:
1009 if pid and cros_update_progress.IsProcessAlive(pid):
xixuan2a0970a2016-08-10 12:12:44 -07001010 os.killpg(int(pid), signal.SIGKILL)
xixuan52c2fba2016-05-20 17:02:48 -07001011
xixuan28681fd2016-11-23 11:13:56 -08001012 result_dict['detailed_error_msg'] = str(e)
1013
1014 return json.dumps(result_dict)
xixuan52c2fba2016-05-20 17:02:48 -07001015
1016 @cherrypy.expose
1017 def handler_cleanup(self, **kwargs):
xixuan3bc974e2016-10-18 17:21:43 -07001018 """Clean track status log and temp directory for CrOS auto-update process.
xixuan52c2fba2016-05-20 17:02:48 -07001019
1020 Args:
1021 kwargs:
1022 host_name: the hostname of the DUT to auto-update.
1023 pid: the background process id of cros-update.
1024 """
1025 if 'host_name' not in kwargs:
1026 raise common_util.DevServerHTTPError((KEY_ERROR_MSG % 'host_name'))
1027
1028 if 'pid' not in kwargs:
1029 raise common_util.DevServerHTTPError((KEY_ERROR_MSG % 'pid'))
1030
1031 host_name = kwargs['host_name']
1032 pid = kwargs['pid']
1033 cros_update_progress.DelTrackStatusFile(host_name, pid)
xixuan3bc974e2016-10-18 17:21:43 -07001034 cros_update_progress.DelAUTempDirectory(host_name, pid)
xixuan52c2fba2016-05-20 17:02:48 -07001035
1036 @cherrypy.expose
1037 def kill_au_proc(self, **kwargs):
1038 """Kill CrOS auto-update process using given process id.
1039
1040 Args:
1041 kwargs:
1042 host_name: Kill all the CrOS auto-update process of this host.
1043
1044 Returns:
1045 True if all processes are killed properly.
1046 """
1047 if 'host_name' not in kwargs:
1048 raise common_util.DevServerHTTPError((KEY_ERROR_MSG % 'host_name'))
1049
xixuan447ad9d2017-02-28 14:46:20 -08001050 cur_pid = kwargs.get('pid')
1051
xixuan52c2fba2016-05-20 17:02:48 -07001052 host_name = kwargs['host_name']
xixuan3bc974e2016-10-18 17:21:43 -07001053 track_log_list = cros_update_progress.GetAllTrackStatusFileByHostName(
1054 host_name)
xixuan52c2fba2016-05-20 17:02:48 -07001055 for log in track_log_list:
1056 # The track log's full path is: path/host_name_pid.log
1057 # Use splitext to remove file extension, then parse pid from the
1058 # filename.
1059 pid = os.path.splitext(os.path.basename(log))[0][len(host_name)+1:]
xixuan447ad9d2017-02-28 14:46:20 -08001060 _clear_process(host_name, pid)
xixuan52c2fba2016-05-20 17:02:48 -07001061
xixuan447ad9d2017-02-28 14:46:20 -08001062 if cur_pid:
1063 _clear_process(host_name, cur_pid)
xixuan52c2fba2016-05-20 17:02:48 -07001064
1065 return 'True'
1066
1067 @cherrypy.expose
1068 def collect_cros_au_log(self, **kwargs):
1069 """Collect CrOS auto-update log.
1070
1071 Args:
1072 kwargs:
1073 host_name: the hostname of the DUT to auto-update.
1074 pid: the background process id of cros-update.
1075
1076 Returns:
David Haddock9f459632017-05-11 14:45:46 -07001077 A dictionary containing the execute log file and any hostlog files.
xixuan52c2fba2016-05-20 17:02:48 -07001078 """
1079 if 'host_name' not in kwargs:
1080 raise common_util.DevServerHTTPError((KEY_ERROR_MSG % 'host_name'))
1081
1082 if 'pid' not in kwargs:
1083 raise common_util.DevServerHTTPError((KEY_ERROR_MSG % 'pid'))
1084
1085 host_name = kwargs['host_name']
1086 pid = kwargs['pid']
xixuan3bc974e2016-10-18 17:21:43 -07001087
1088 # Fetch the execute log recorded by cros_update_progress.
xixuan1bbfaba2016-10-13 17:53:22 -07001089 au_log = cros_update_progress.ReadExecuteLogFile(host_name, pid)
1090 cros_update_progress.DelExecuteLogFile(host_name, pid)
David Haddock9f459632017-05-11 14:45:46 -07001091 # Fetch the cros_au host_logs if they exist
1092 au_hostlogs = cros_update_progress.ReadAUHostLogFiles(host_name, pid)
1093 return json.dumps({'cros_au_log': au_log, 'host_logs': au_hostlogs})
xixuan1bbfaba2016-10-13 17:53:22 -07001094
xixuan52c2fba2016-05-20 17:02:48 -07001095 @cherrypy.expose
Dan Shi2f136862016-02-11 15:38:38 -08001096 def locate_file(self, **kwargs):
1097 """Get the path to the given file name.
1098
1099 This method looks up the given file name inside specified build artifacts.
1100 One use case is to help caller to locate an apk file inside a build
1101 artifact. The location of the apk file could be different based on the
1102 branch and target.
1103
1104 Args:
1105 file_name: Name of the file to look for.
1106 artifacts: A list of artifact names to search for the file.
1107
1108 Returns:
1109 Path to the file with the given name. It's relative to the folder for the
1110 build, e.g., DATA/priv-app/sl4a/sl4a.apk
Dan Shi2f136862016-02-11 15:38:38 -08001111 """
1112 dl, _ = _get_downloader_and_factory(kwargs)
1113 try:
Joe Brennan1691f8e2017-03-15 15:53:36 -07001114 file_name = kwargs['file_name']
Dan Shi2f136862016-02-11 15:38:38 -08001115 artifacts = kwargs['artifacts']
1116 except KeyError:
1117 raise DevServerError('`file_name` and `artifacts` are required to search '
1118 'for a file in build artifacts.')
1119 build_path = dl.GetBuildDir()
1120 for artifact in artifacts:
1121 # Get the unzipped folder of the artifact. If it's not defined in
1122 # ARTIFACT_UNZIP_FOLDER_MAP, assume the files are unzipped to the build
1123 # directory directly.
1124 folder = artifact_info.ARTIFACT_UNZIP_FOLDER_MAP.get(artifact, '')
1125 artifact_path = os.path.join(build_path, folder)
1126 for root, _, filenames in os.walk(artifact_path):
Joe Brennan1691f8e2017-03-15 15:53:36 -07001127 if file_name in set([f for f in filenames]):
Dan Shi2f136862016-02-11 15:38:38 -08001128 return os.path.relpath(os.path.join(root, file_name), build_path)
1129 raise DevServerError('File `%s` can not be found in artifacts: %s' %
1130 (file_name, artifacts))
1131
1132 @cherrypy.expose
Simran Basi4baad082013-02-14 13:39:18 -08001133 def setup_telemetry(self, **kwargs):
1134 """Extracts and sets up telemetry
1135
1136 This method goes through the telemetry deps packages, and stages them on
1137 the devserver to be used by the drones and the telemetry tests.
1138
1139 Args:
1140 archive_url: Google Storage URL for the build.
1141
1142 Returns:
1143 Path to the source folder for the telemetry codebase once it is staged.
1144 """
Gabe Black3b567202015-09-23 14:07:59 -07001145 dl = _get_downloader(kwargs)
Simran Basi4baad082013-02-14 13:39:18 -08001146
Gabe Black3b567202015-09-23 14:07:59 -07001147 build_path = dl.GetBuildDir()
Simran Basi4baad082013-02-14 13:39:18 -08001148 deps_path = os.path.join(build_path, 'autotest/packages')
1149 telemetry_path = os.path.join(build_path, TELEMETRY_FOLDER)
1150 src_folder = os.path.join(telemetry_path, 'src')
1151
1152 with self._telemetry_lock_dict.lock(telemetry_path):
1153 if os.path.exists(src_folder):
1154 # Telemetry is already fully stage return
1155 return src_folder
1156
1157 common_util.MkDirP(telemetry_path)
1158
1159 # Copy over the required deps tar balls to the telemetry directory.
1160 for dep in TELEMETRY_DEPS:
1161 dep_path = os.path.join(deps_path, dep)
Simran Basi0d078682013-03-22 16:40:04 -07001162 if not os.path.exists(dep_path):
1163 # This dep does not exist (could be new), do not extract it.
1164 continue
Simran Basi4baad082013-02-14 13:39:18 -08001165 try:
1166 common_util.ExtractTarball(dep_path, telemetry_path)
1167 except common_util.CommonUtilError as e:
1168 shutil.rmtree(telemetry_path)
1169 raise DevServerError(str(e))
1170
1171 # By default all the tarballs extract to test_src but some parts of
1172 # the telemetry code specifically hardcoded to exist inside of 'src'.
1173 test_src = os.path.join(telemetry_path, 'test_src')
1174 try:
1175 shutil.move(test_src, src_folder)
1176 except shutil.Error:
1177 # This can occur if src_folder already exists. Remove and retry move.
1178 shutil.rmtree(src_folder)
Gabe Black3b567202015-09-23 14:07:59 -07001179 raise DevServerError(
1180 'Failure in telemetry setup for build %s. Appears that the '
1181 'test_src to src move failed.' % dl.GetBuild())
Simran Basi4baad082013-02-14 13:39:18 -08001182
1183 return src_folder
1184
1185 @cherrypy.expose
Chris Sosa76e44b92013-01-31 12:11:38 -08001186 def symbolicate_dump(self, minidump, **kwargs):
Chris Masone816e38c2012-05-02 12:22:36 -07001187 """Symbolicates a minidump using pre-downloaded symbols, returns it.
1188
1189 Callers will need to POST to this URL with a body of MIME-type
1190 "multipart/form-data".
1191 The body should include a single argument, 'minidump', containing the
1192 binary-formatted minidump to symbolicate.
1193
Chris Masone816e38c2012-05-02 12:22:36 -07001194 Args:
Chris Sosa76e44b92013-01-31 12:11:38 -08001195 archive_url: Google Storage URL for the build.
Chris Masone816e38c2012-05-02 12:22:36 -07001196 minidump: The binary minidump file to symbolicate.
1197 """
Chris Sosa76e44b92013-01-31 12:11:38 -08001198 # Ensure the symbols have been staged.
Dan Shif08fe492016-10-04 14:39:25 -07001199 # Try debug.tar.xz first, then debug.tgz
1200 for artifact in (artifact_info.SYMBOLS_ONLY, artifact_info.SYMBOLS):
1201 kwargs['artifacts'] = artifact
1202 dl = _get_downloader(kwargs)
1203
1204 try:
1205 if self.stage(**kwargs) == 'Success':
1206 break
1207 except build_artifact.ArtifactDownloadError:
1208 continue
1209 else:
Gabe Black3b567202015-09-23 14:07:59 -07001210 raise DevServerError('Failed to stage symbols for %s' %
1211 dl.DescribeSource())
Chris Sosa76e44b92013-01-31 12:11:38 -08001212
Chris Masone816e38c2012-05-02 12:22:36 -07001213 to_return = ''
1214 with tempfile.NamedTemporaryFile() as local:
1215 while True:
1216 data = minidump.file.read(8192)
1217 if not data:
1218 break
1219 local.write(data)
Chris Sosa76e44b92013-01-31 12:11:38 -08001220
Chris Masone816e38c2012-05-02 12:22:36 -07001221 local.flush()
Chris Sosa76e44b92013-01-31 12:11:38 -08001222
Gabe Black3b567202015-09-23 14:07:59 -07001223 symbols_directory = os.path.join(dl.GetBuildDir(), 'debug', 'breakpad')
Chris Sosa76e44b92013-01-31 12:11:38 -08001224
xixuanab744382017-04-27 10:41:27 -07001225 # The location of minidump_stackwalk is defined in chromeos-admin.
Chris Sosa76e44b92013-01-31 12:11:38 -08001226 stackwalk = subprocess.Popen(
xixuanab744382017-04-27 10:41:27 -07001227 ['/usr/local/bin/minidump_stackwalk', local.name, symbols_directory],
Chris Sosa76e44b92013-01-31 12:11:38 -08001228 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
1229
Chris Masone816e38c2012-05-02 12:22:36 -07001230 to_return, error_text = stackwalk.communicate()
1231 if stackwalk.returncode != 0:
1232 raise DevServerError("Can't generate stack trace: %s (rc=%d)" % (
1233 error_text, stackwalk.returncode))
1234
1235 return to_return
1236
1237 @cherrypy.expose
Don Garrettf84631a2014-01-07 18:21:26 -08001238 def latestbuild(self, **kwargs):
Scott Zawalski16954532012-03-20 15:31:36 -04001239 """Return a string representing the latest build for a given target.
1240
1241 Args:
1242 target: The build target, typically a combination of the board and the
1243 type of build e.g. x86-mario-release.
1244 milestone: The milestone to filter builds on. E.g. R16. Optional, if not
1245 provided the latest RXX build will be returned.
Don Garrettf84631a2014-01-07 18:21:26 -08001246
Scott Zawalski16954532012-03-20 15:31:36 -04001247 Returns:
1248 A string representation of the latest build if one exists, i.e.
1249 R19-1993.0.0-a1-b1480.
1250 An empty string if no latest could be found.
1251 """
Don Garrettf84631a2014-01-07 18:21:26 -08001252 if not kwargs:
Scott Zawalski16954532012-03-20 15:31:36 -04001253 return _PrintDocStringAsHTML(self.latestbuild)
1254
Don Garrettf84631a2014-01-07 18:21:26 -08001255 if 'target' not in kwargs:
Chris Sosa4b951602014-04-09 20:26:07 -07001256 raise common_util.DevServerHTTPError(500, 'Error: target= is required!')
Dan Shi61305df2015-10-26 16:52:35 -07001257
1258 if _is_android_build_request(kwargs):
1259 branch = kwargs.get('branch', None)
1260 target = kwargs.get('target', None)
1261 if not target or not branch:
1262 raise DevServerError(
xixuan52c2fba2016-05-20 17:02:48 -07001263 'Both target and branch must be specified to query for the latest '
1264 'Android build.')
Dan Shi61305df2015-10-26 16:52:35 -07001265 return android_build.BuildAccessor.GetLatestBuildID(target, branch)
1266
Scott Zawalski16954532012-03-20 15:31:36 -04001267 try:
Gilad Arnoldc65330c2012-09-20 15:17:48 -07001268 return common_util.GetLatestBuildVersion(
Don Garrettf84631a2014-01-07 18:21:26 -08001269 updater.static_dir, kwargs['target'],
1270 milestone=kwargs.get('milestone'))
Gilad Arnold17fe03d2012-10-02 10:05:01 -07001271 except common_util.CommonUtilError as errmsg:
Chris Sosa4b951602014-04-09 20:26:07 -07001272 raise common_util.DevServerHTTPError(500, str(errmsg))
Scott Zawalski16954532012-03-20 15:31:36 -04001273
1274 @cherrypy.expose
xixuan7efd0002016-04-14 15:34:01 -07001275 def list_suite_controls(self, **kwargs):
1276 """Return a list of contents of all known control files.
1277
1278 Example URL:
1279 To List all control files' content:
1280 http://dev-server/list_suite_controls?suite_name=bvt&
1281 build=daisy_spring-release/R29-4279.0.0
1282
1283 Args:
1284 build: The build i.e. x86-alex-release/R18-1514.0.0-a1-b1450.
1285 suite_name: List the control files belonging to that suite.
1286
1287 Returns:
Dan Shia1cd6522016-04-18 16:07:21 -07001288 A dictionary of all control files's path to its content for given suite.
xixuan7efd0002016-04-14 15:34:01 -07001289 """
1290 if not kwargs:
1291 return _PrintDocStringAsHTML(self.controlfiles)
1292
1293 if 'build' not in kwargs:
1294 raise common_util.DevServerHTTPError(500, 'Error: build= is required!')
1295
1296 if 'suite_name' not in kwargs:
Dan Shia1cd6522016-04-18 16:07:21 -07001297 raise common_util.DevServerHTTPError(500,
1298 'Error: suite_name= is required!')
xixuan7efd0002016-04-14 15:34:01 -07001299
1300 control_file_list = [
1301 line.rstrip() for line in common_util.GetControlFileListForSuite(
1302 updater.static_dir, kwargs['build'],
1303 kwargs['suite_name']).splitlines()]
1304
Dan Shia1cd6522016-04-18 16:07:21 -07001305 control_file_content_dict = {}
xixuan7efd0002016-04-14 15:34:01 -07001306 for control_path in control_file_list:
Dan Shia1cd6522016-04-18 16:07:21 -07001307 control_file_content_dict[control_path] = (common_util.GetControlFile(
xixuan7efd0002016-04-14 15:34:01 -07001308 updater.static_dir, kwargs['build'], control_path))
1309
Dan Shia1cd6522016-04-18 16:07:21 -07001310 return json.dumps(control_file_content_dict)
xixuan7efd0002016-04-14 15:34:01 -07001311
1312 @cherrypy.expose
Don Garrettf84631a2014-01-07 18:21:26 -08001313 def controlfiles(self, **kwargs):
Scott Zawalski4647ce62012-01-03 17:17:28 -05001314 """Return a control file or a list of all known control files.
1315
1316 Example URL:
1317 To List all control files:
beepsbd337242013-07-09 22:44:06 -07001318 http://dev-server/controlfiles?suite_name=&build=daisy_spring-release/R29-4279.0.0
1319 To List all control files for, say, the bvt suite:
1320 http://dev-server/controlfiles?suite_name=bvt&build=daisy_spring-release/R29-4279.0.0
Scott Zawalski4647ce62012-01-03 17:17:28 -05001321 To return the contents of a path:
Scott Zawalski84a39c92012-01-13 15:12:42 -05001322 http://dev-server/controlfiles?board=x86-alex-release&build=R18-1514.0.0&control_path=client/sleeptest/control
Scott Zawalski4647ce62012-01-03 17:17:28 -05001323
1324 Args:
Scott Zawalski84a39c92012-01-13 15:12:42 -05001325 build: The build i.e. x86-alex-release/R18-1514.0.0-a1-b1450.
Scott Zawalski4647ce62012-01-03 17:17:28 -05001326 control_path: If you want the contents of a control file set this
1327 to the path. E.g. client/site_tests/sleeptest/control
1328 Optional, if not provided return a list of control files is returned.
beepsbd337242013-07-09 22:44:06 -07001329 suite_name: If control_path is not specified but a suite_name is
1330 specified, list the control files belonging to that suite instead of
1331 all control files. The empty string for suite_name will list all control
1332 files for the build.
Don Garrettf84631a2014-01-07 18:21:26 -08001333
Scott Zawalski4647ce62012-01-03 17:17:28 -05001334 Returns:
1335 Contents of a control file if control_path is provided.
1336 A list of control files if no control_path is provided.
1337 """
Don Garrettf84631a2014-01-07 18:21:26 -08001338 if not kwargs:
Scott Zawalski4647ce62012-01-03 17:17:28 -05001339 return _PrintDocStringAsHTML(self.controlfiles)
1340
Don Garrettf84631a2014-01-07 18:21:26 -08001341 if 'build' not in kwargs:
Chris Sosa4b951602014-04-09 20:26:07 -07001342 raise common_util.DevServerHTTPError(500, 'Error: build= is required!')
Scott Zawalski4647ce62012-01-03 17:17:28 -05001343
Don Garrettf84631a2014-01-07 18:21:26 -08001344 if 'control_path' not in kwargs:
1345 if 'suite_name' in kwargs and kwargs['suite_name']:
beepsbd337242013-07-09 22:44:06 -07001346 return common_util.GetControlFileListForSuite(
Don Garrettf84631a2014-01-07 18:21:26 -08001347 updater.static_dir, kwargs['build'], kwargs['suite_name'])
beepsbd337242013-07-09 22:44:06 -07001348 else:
1349 return common_util.GetControlFileList(
Don Garrettf84631a2014-01-07 18:21:26 -08001350 updater.static_dir, kwargs['build'])
Scott Zawalski4647ce62012-01-03 17:17:28 -05001351 else:
Gilad Arnoldc65330c2012-09-20 15:17:48 -07001352 return common_util.GetControlFile(
Don Garrettf84631a2014-01-07 18:21:26 -08001353 updater.static_dir, kwargs['build'], kwargs['control_path'])
Frank Farzan40160872011-12-12 18:39:18 -08001354
1355 @cherrypy.expose
Simran Basi99e63c02014-05-20 10:39:52 -07001356 def xbuddy_translate(self, *args, **kwargs):
Yu-Ju Hong1bdb7a92014-04-10 16:02:11 -07001357 """Translates an xBuddy path to a real path to artifact if it exists.
1358
1359 Args:
Simran Basi99e63c02014-05-20 10:39:52 -07001360 args: An xbuddy path in the form of {local|remote}/build_id/artifact.
1361 Local searches the devserver's static directory. Remote searches a
1362 Google Storage image archive.
1363
1364 Kwargs:
1365 image_dir: Google Storage image archive to search in if requesting a
1366 remote artifact. If none uses the default bucket.
Yu-Ju Hong1bdb7a92014-04-10 16:02:11 -07001367
1368 Returns:
Simran Basi99e63c02014-05-20 10:39:52 -07001369 String in the format of build_id/artifact as stored on the local server
1370 or in Google Storage.
Yu-Ju Hong1bdb7a92014-04-10 16:02:11 -07001371 """
Simran Basi99e63c02014-05-20 10:39:52 -07001372 build_id, filename = self._xbuddy.Translate(
Gabe Black3b567202015-09-23 14:07:59 -07001373 args, image_dir=kwargs.get('image_dir'))
Yu-Ju Hong1bdb7a92014-04-10 16:02:11 -07001374 response = os.path.join(build_id, filename)
1375 _Log('Path translation requested, returning: %s', response)
1376 return response
1377
1378 @cherrypy.expose
joycheneaf4cfc2013-07-02 08:38:57 -07001379 def xbuddy(self, *args, **kwargs):
1380 """The full xBuddy call, returns resource specified by path_parts.
joychen3cb228e2013-06-12 12:13:13 -07001381
1382 Args:
joycheneaf4cfc2013-07-02 08:38:57 -07001383 path_parts: the path following xbuddy/ in the call url is split into the
joychen121fc9b2013-08-02 14:30:30 -07001384 components of the path. The path can be understood as
1385 "{local|remote}/build_id/artifact" where build_id is composed of
1386 "board/version."
joycheneaf4cfc2013-07-02 08:38:57 -07001387
joychen121fc9b2013-08-02 14:30:30 -07001388 The first path element is optional, and can be "remote" or "local"
1389 If local (the default), devserver will not attempt to access Google
1390 Storage, and will only search the static directory for the files.
1391 If remote, devserver will try to obtain the artifact off GS if it's
1392 not found locally.
1393 The board is the familiar board name, optionally suffixed.
1394 The version can be the google storage version number, and may also be
1395 any of a number of xBuddy defined version aliases that will be
1396 translated into the latest built image that fits the description.
1397 Defaults to latest.
1398 The artifact is one of a number of image or artifact aliases used by
1399 xbuddy, defined in xbuddy:ALIASES. Defaults to test.
joycheneaf4cfc2013-07-02 08:38:57 -07001400
1401 Kwargs:
Yu-Ju Hong51495eb2013-12-12 17:08:43 -08001402 for_update: {true|false}
1403 if true, pregenerates the update payloads for the image,
1404 and returns the update uri to pass to the
1405 update_engine_client.
joychen3cb228e2013-06-12 12:13:13 -07001406 return_dir: {true|false}
1407 if set to true, returns the url to the update.gz
Yu-Ju Hong51495eb2013-12-12 17:08:43 -08001408 relative_path: {true|false}
1409 if set to true, returns the relative path to the payload
1410 directory from static_dir.
joychen3cb228e2013-06-12 12:13:13 -07001411 Example URL:
joycheneaf4cfc2013-07-02 08:38:57 -07001412 http://host:port/xbuddy/x86-generic/R26-4000.0.0/test
joychen3cb228e2013-06-12 12:13:13 -07001413 or
joycheneaf4cfc2013-07-02 08:38:57 -07001414 http://host:port/xbuddy/x86-generic/R26-4000.0.0/test?return_dir=true
joychen3cb228e2013-06-12 12:13:13 -07001415
1416 Returns:
Yu-Ju Hong51495eb2013-12-12 17:08:43 -08001417 If |for_update|, returns a redirect to the image or update file
1418 on the devserver. E.g.,
1419 http://host:port/static/archive/x86-generic-release/R26-4000.0.0/
1420 chromium-test-image.bin
1421 If |return_dir|, return a uri to the folder where the artifact is. E.g.,
1422 http://host:port/static/x86-generic-release/R26-4000.0.0/
1423 If |relative_path| is true, return a relative path the folder where the
1424 payloads are. E.g.,
1425 archive/x86-generic-release/R26-4000.0.0
joychen3cb228e2013-06-12 12:13:13 -07001426 """
Chris Sosa75490802013-09-30 17:21:45 -07001427 boolean_string = kwargs.get('for_update')
1428 for_update = xbuddy.XBuddy.ParseBoolean(boolean_string)
Yu-Ju Hong51495eb2013-12-12 17:08:43 -08001429 boolean_string = kwargs.get('return_dir')
1430 return_dir = xbuddy.XBuddy.ParseBoolean(boolean_string)
1431 boolean_string = kwargs.get('relative_path')
1432 relative_path = xbuddy.XBuddy.ParseBoolean(boolean_string)
joychen121fc9b2013-08-02 14:30:30 -07001433
Yu-Ju Hong51495eb2013-12-12 17:08:43 -08001434 if return_dir and relative_path:
Chris Sosa4b951602014-04-09 20:26:07 -07001435 raise common_util.DevServerHTTPError(
1436 500, 'Cannot specify both return_dir and relative_path')
Chris Sosa75490802013-09-30 17:21:45 -07001437
1438 # For updates, we optimize downloading of test images.
1439 file_name = None
1440 build_id = None
1441 if for_update:
1442 try:
Yu-Ju Hong1bdb7a92014-04-10 16:02:11 -07001443 build_id = self._xbuddy.StageTestArtifactsForUpdate(args)
Chris Sosa75490802013-09-30 17:21:45 -07001444 except build_artifact.ArtifactDownloadError:
1445 build_id = None
1446
1447 if not build_id:
1448 build_id, file_name = self._xbuddy.Get(args)
1449
Yu-Ju Hong51495eb2013-12-12 17:08:43 -08001450 if for_update:
1451 _Log('Payload generation triggered by request')
1452 # Forces payload to be in cache and symlinked into build_id dir.
Chris Sosa75490802013-09-30 17:21:45 -07001453 updater.GetUpdateForLabel(autoupdate.FORCED_UPDATE, build_id,
1454 image_name=file_name)
Yu-Ju Hong51495eb2013-12-12 17:08:43 -08001455
1456 response = None
1457 if return_dir:
1458 response = os.path.join(cherrypy.request.base, 'static', build_id)
1459 _Log('Directory requested, returning: %s', response)
1460 elif relative_path:
1461 response = build_id
1462 _Log('Relative path requested, returning: %s', response)
1463 elif for_update:
1464 response = os.path.join(cherrypy.request.base, 'update', build_id)
1465 _Log('Update URI requested, returning: %s', response)
joychen3cb228e2013-06-12 12:13:13 -07001466 else:
Yu-Ju Hong51495eb2013-12-12 17:08:43 -08001467 # Redirect to download the payload if no kwargs are set.
joychen121fc9b2013-08-02 14:30:30 -07001468 build_id = '/' + os.path.join('static', build_id, file_name)
Yu-Ju Hong51495eb2013-12-12 17:08:43 -08001469 _Log('Payload requested, returning: %s', build_id)
joychen121fc9b2013-08-02 14:30:30 -07001470 raise cherrypy.HTTPRedirect(build_id, 302)
joychen3cb228e2013-06-12 12:13:13 -07001471
Yu-Ju Hong51495eb2013-12-12 17:08:43 -08001472 return response
1473
joychen3cb228e2013-06-12 12:13:13 -07001474 @cherrypy.expose
1475 def xbuddy_list(self):
1476 """Lists the currently available images & time since last access.
1477
Gilad Arnold452fd272014-02-04 11:09:28 -08001478 Returns:
1479 A string representation of a list of tuples [(build_id, time since last
1480 access),...]
joychen3cb228e2013-06-12 12:13:13 -07001481 """
1482 return self._xbuddy.List()
1483
1484 @cherrypy.expose
1485 def xbuddy_capacity(self):
Gilad Arnold452fd272014-02-04 11:09:28 -08001486 """Returns the number of images cached by xBuddy."""
joychen3cb228e2013-06-12 12:13:13 -07001487 return self._xbuddy.Capacity()
1488
1489 @cherrypy.expose
Chris Sosa7c931362010-10-11 19:49:01 -07001490 def index(self):
Gilad Arnoldf8f769f2012-09-24 08:43:01 -07001491 """Presents a welcome message and documentation links."""
Gilad Arnoldf8f769f2012-09-24 08:43:01 -07001492 return ('Welcome to the Dev Server!<br>\n'
1493 '<br>\n'
1494 'Here are the available methods, click for documentation:<br>\n'
1495 '<br>\n'
1496 '%s' %
1497 '<br>\n'.join(
1498 [('<a href=doc/%s>%s</a>' % (name, name))
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -07001499 for name in _FindExposedMethods(
1500 self, '', unlisted=self._UNLISTED_METHODS)]))
Gilad Arnoldf8f769f2012-09-24 08:43:01 -07001501
1502 @cherrypy.expose
1503 def doc(self, *args):
1504 """Shows the documentation for available methods / URLs.
1505
1506 Example:
1507 http://myhost/doc/update
1508 """
Gilad Arnoldd5ebaaa2012-10-02 11:52:38 -07001509 name = '/'.join(args)
1510 method = _GetExposedMethod(self, name)
Gilad Arnoldf8f769f2012-09-24 08:43:01 -07001511 if not method:
1512 raise DevServerError("No exposed method named `%s'" % name)
1513 if not method.__doc__:
1514 raise DevServerError("No documentation for exposed method `%s'" % name)
1515 return '<pre>\n%s</pre>' % method.__doc__
Chris Sosa7c931362010-10-11 19:49:01 -07001516
Dale Curtisc9aaf3a2011-08-09 15:47:40 -07001517 @cherrypy.expose
Chris Sosa7c931362010-10-11 19:49:01 -07001518 def update(self, *args):
Gilad Arnoldf8f769f2012-09-24 08:43:01 -07001519 """Handles an update check from a Chrome OS client.
1520
1521 The HTTP request should contain the standard Omaha-style XML blob. The URL
1522 line may contain an additional intermediate path to the update payload.
1523
joychen121fc9b2013-08-02 14:30:30 -07001524 This request can be handled in one of 4 ways, depending on the devsever
1525 settings and intermediate path.
joychenb0dfe552013-07-30 10:02:06 -07001526
joychen121fc9b2013-08-02 14:30:30 -07001527 1. No intermediate path
1528 If no intermediate path is given, the default behavior is to generate an
1529 update payload from the latest test image locally built for the board
1530 specified in the xml. Devserver serves the generated payload.
1531
1532 2. Path explicitly invokes XBuddy
1533 If there is a path given, it can explicitly invoke xbuddy by prefixing it
1534 with 'xbuddy'. This path is then used to acquire an image binary for the
1535 devserver to generate an update payload from. Devserver then serves this
1536 payload.
1537
1538 3. Path is left for the devserver to interpret.
1539 If the path given doesn't explicitly invoke xbuddy, devserver will attempt
1540 to generate a payload from the test image in that directory and serve it.
1541
1542 4. The devserver is in a 'forced' mode. TO BE DEPRECATED
1543 This comes from the usage of --forced_payload or --image when starting the
1544 devserver. No matter what path (or no path) gets passed in, devserver will
1545 serve the update payload (--forced_payload) or generate an update payload
1546 from the image (--image).
1547
1548 Examples:
1549 1. No intermediate path
1550 update_engine_client --omaha_url=http://myhost/update
1551 This generates an update payload from the latest test image locally built
1552 for the board specified in the xml.
1553
1554 2. Explicitly invoke xbuddy
1555 update_engine_client --omaha_url=
1556 http://myhost/update/xbuddy/remote/board/version/dev
1557 This would go to GS to download the dev image for the board, from which
1558 the devserver would generate a payload to serve.
1559
1560 3. Give a path for devserver to interpret
1561 update_engine_client --omaha_url=http://myhost/update/some/random/path
1562 This would attempt, in order to:
1563 a) Generate an update from a test image binary if found in
1564 static_dir/some/random/path.
1565 b) Serve an update payload found in static_dir/some/random/path.
1566 c) Hope that some/random/path takes the form "board/version" and
1567 and attempt to download an update payload for that board/version
1568 from GS.
Gilad Arnoldf8f769f2012-09-24 08:43:01 -07001569 """
joychen121fc9b2013-08-02 14:30:30 -07001570 label = '/'.join(args)
Gilad Arnold286a0062012-01-12 13:47:02 -08001571 body_length = int(cherrypy.request.headers.get('Content-Length', 0))
Chris Sosa7c931362010-10-11 19:49:01 -07001572 data = cherrypy.request.rfile.read(body_length)
Chris Sosa7c931362010-10-11 19:49:01 -07001573
joychen121fc9b2013-08-02 14:30:30 -07001574 return updater.HandleUpdatePing(data, label)
Chris Sosa0356d3b2010-09-16 15:46:22 -07001575
Dan Shiafd0e492015-05-27 14:23:51 -07001576 @require_psutil()
1577 def _get_io_stats(self):
1578 """Get the IO stats as a dictionary.
1579
Gabe Black3b567202015-09-23 14:07:59 -07001580 Returns:
1581 A dictionary of IO stats collected by psutil.
Dan Shiafd0e492015-05-27 14:23:51 -07001582 """
1583 return {'disk_read_bytes_per_second': self.disk_read_bytes_per_sec,
1584 'disk_write_bytes_per_second': self.disk_write_bytes_per_sec,
1585 'disk_total_bytes_per_second': (self.disk_read_bytes_per_sec +
1586 self.disk_write_bytes_per_sec),
1587 'network_sent_bytes_per_second': self.network_sent_bytes_per_sec,
1588 'network_recv_bytes_per_second': self.network_recv_bytes_per_sec,
1589 'network_total_bytes_per_second': (self.network_sent_bytes_per_sec +
1590 self.network_recv_bytes_per_sec),
1591 'cpu_percent': psutil.cpu_percent(),}
1592
Dan Shi7247f9c2016-06-01 09:19:09 -07001593
1594 def _get_process_count(self, process_cmd_pattern):
1595 """Get the count of processes that match the given command pattern.
1596
1597 Args:
1598 process_cmd_pattern: The regex pattern of process command to match.
1599
1600 Returns:
1601 The count of processes that match the given command pattern.
1602 """
1603 try:
xixuanac89ce82016-11-30 16:48:20 -08001604 # Use Popen instead of check_output since the latter cannot run with old
1605 # python version (less than 2.7)
1606 proc = subprocess.Popen(
1607 'pgrep -fc "%s"' % process_cmd_pattern,
1608 stdout=subprocess.PIPE,
1609 stderr=subprocess.PIPE,
1610 shell=True)
1611 cmd_output, cmd_error = proc.communicate()
1612 if cmd_error:
1613 _Log('Error happened when getting process count: %s' % cmd_error)
1614
1615 return int(cmd_output)
Dan Shi7247f9c2016-06-01 09:19:09 -07001616 except subprocess.CalledProcessError:
1617 return 0
1618
1619
Dan Shif5ce2de2013-04-25 16:06:32 -07001620 @cherrypy.expose
1621 def check_health(self):
1622 """Collect the health status of devserver to see if it's ready for staging.
1623
Gilad Arnold452fd272014-02-04 11:09:28 -08001624 Returns:
1625 A JSON dictionary containing all or some of the following fields:
1626 free_disk (int): free disk space in GB
1627 staging_thread_count (int): number of devserver threads currently staging
1628 an image
Dan Shi7247f9c2016-06-01 09:19:09 -07001629 apache_client_count (int): count of Apache processes.
1630 telemetry_test_count (int): count of telemetry tests.
1631 gsutil_count (int): count of gsutil processes.
Dan Shif5ce2de2013-04-25 16:06:32 -07001632 """
1633 # Get free disk space.
1634 stat = os.statvfs(updater.static_dir)
1635 free_disk = stat.f_bsize * stat.f_bavail / 1000000000
Dan Shi7247f9c2016-06-01 09:19:09 -07001636 apache_client_count = self._get_process_count('apache')
1637 telemetry_test_count = self._get_process_count('python.*telemetry')
1638 gsutil_count = self._get_process_count('gsutil')
xixuan447ad9d2017-02-28 14:46:20 -08001639 au_process_count = len(cros_update_progress.GetAllRunningAUProcess())
Dan Shif5ce2de2013-04-25 16:06:32 -07001640
Dan Shiafd0e492015-05-27 14:23:51 -07001641 health_data = {
Dan Shif5ce2de2013-04-25 16:06:32 -07001642 'free_disk': free_disk,
Dan Shid76e6bb2016-01-28 22:28:51 -08001643 'staging_thread_count': DevServerRoot._staging_thread_count,
1644 'apache_client_count': apache_client_count,
Dan Shi7247f9c2016-06-01 09:19:09 -07001645 'telemetry_test_count': telemetry_test_count,
xixuan447ad9d2017-02-28 14:46:20 -08001646 'gsutil_count': gsutil_count,
1647 'au_process_count': au_process_count,
1648 }
Dan Shiafd0e492015-05-27 14:23:51 -07001649 health_data.update(self._get_io_stats() or {})
1650
1651 return json.dumps(health_data)
Dan Shif5ce2de2013-04-25 16:06:32 -07001652
1653
Chris Sosadbc20082012-12-10 13:39:11 -08001654def _CleanCache(cache_dir, wipe):
1655 """Wipes any excess cached items in the cache_dir.
1656
1657 Args:
1658 cache_dir: the directory we are wiping from.
1659 wipe: If True, wipe all the contents -- not just the excess.
1660 """
1661 if wipe:
1662 # Clear the cache and exit on error.
1663 cmd = 'rm -rf %s/*' % cache_dir
1664 if os.system(cmd) != 0:
1665 _Log('Failed to clear the cache with %s' % cmd)
1666 sys.exit(1)
1667 else:
1668 # Clear all but the last N cached updates
1669 cmd = ('cd %s; ls -tr | head --lines=-%d | xargs rm -rf' %
1670 (cache_dir, CACHED_ENTRIES))
1671 if os.system(cmd) != 0:
1672 _Log('Failed to clean up old delta cache files with %s' % cmd)
1673 sys.exit(1)
1674
1675
Chris Sosa3ae4dc12013-03-29 11:47:00 -07001676def _AddTestingOptions(parser):
1677 group = optparse.OptionGroup(
1678 parser, 'Advanced Testing Options', 'These are used by test scripts and '
1679 'developers writing integration tests utilizing the devserver. They are '
1680 'not intended to be really used outside the scope of someone '
1681 'knowledgable about the test.')
1682 group.add_option('--exit',
1683 action='store_true',
1684 help='do not start the server (yet pregenerate/clear cache)')
1685 group.add_option('--host_log',
1686 action='store_true', default=False,
1687 help='record history of host update events (/api/hostlog)')
1688 group.add_option('--max_updates',
Gabe Black3b567202015-09-23 14:07:59 -07001689 metavar='NUM', default=-1, type='int',
Chris Sosa3ae4dc12013-03-29 11:47:00 -07001690 help='maximum number of update checks handled positively '
1691 '(default: unlimited)')
1692 group.add_option('--private_key',
1693 metavar='PATH', default=None,
1694 help='path to the private key in pem format. If this is set '
1695 'the devserver will generate update payloads that are '
1696 'signed with this key.')
David Zeuthen52ccd012013-10-31 12:58:26 -07001697 group.add_option('--private_key_for_metadata_hash_signature',
1698 metavar='PATH', default=None,
1699 help='path to the private key in pem format. If this is set '
1700 'the devserver will sign the metadata hash with the given '
1701 'key and transmit in the Omaha-style XML response.')
1702 group.add_option('--public_key',
1703 metavar='PATH', default=None,
1704 help='path to the public key in pem format. If this is set '
1705 'the devserver will transmit a base64 encoded version of '
1706 'the content in the Omaha-style XML response.')
Chris Sosa3ae4dc12013-03-29 11:47:00 -07001707 group.add_option('--proxy_port',
1708 metavar='PORT', default=None, type='int',
1709 help='port to have the client connect to -- basically the '
1710 'devserver lies to the update to tell it to get the payload '
1711 'from a different port that will proxy the request back to '
1712 'the devserver. The proxy must be managed outside the '
1713 'devserver.')
1714 group.add_option('--remote_payload',
1715 action='store_true', default=False,
Chris Sosa4b951602014-04-09 20:26:07 -07001716 help='Payload is being served from a remote machine. With '
1717 'this setting enabled, this devserver instance serves as '
1718 'just an Omaha server instance. In this mode, the '
1719 'devserver enforces a few extra components of the Omaha '
Chris Sosafc715442014-04-09 20:45:23 -07001720 'protocol, such as hardware class, being sent.')
Chris Sosa3ae4dc12013-03-29 11:47:00 -07001721 group.add_option('-u', '--urlbase',
1722 metavar='URL',
Gabe Black3b567202015-09-23 14:07:59 -07001723 help='base URL for update images, other than the '
1724 'devserver. Use in conjunction with remote_payload.')
Chris Sosa3ae4dc12013-03-29 11:47:00 -07001725 parser.add_option_group(group)
1726
1727
1728def _AddUpdateOptions(parser):
1729 group = optparse.OptionGroup(
1730 parser, 'Autoupdate Options', 'These options can be used to change '
1731 'how the devserver either generates or serve update payloads. Please '
1732 'note that all of these option affect how a payload is generated and so '
1733 'do not work in archive-only mode.')
1734 group.add_option('--board',
1735 help='By default the devserver will create an update '
1736 'payload from the latest image built for the board '
1737 'a device that is requesting an update has. When we '
1738 'pre-generate an update (see below) and we do not specify '
1739 'another update_type option like image or payload, the '
1740 'devserver needs to know the board to generate the latest '
1741 'image for. This is that board.')
1742 group.add_option('--critical_update',
1743 action='store_true', default=False,
1744 help='Present update payload as critical')
Chris Sosa3ae4dc12013-03-29 11:47:00 -07001745 group.add_option('--image',
1746 metavar='FILE',
1747 help='Generate and serve an update using this image to any '
1748 'device that requests an update.')
Chris Sosa3ae4dc12013-03-29 11:47:00 -07001749 group.add_option('--payload',
1750 metavar='PATH',
1751 help='use the update payload from specified directory '
1752 '(update.gz).')
1753 group.add_option('-p', '--pregenerate_update',
1754 action='store_true', default=False,
1755 help='pre-generate the update payload before accepting '
1756 'update requests. Useful to help debug payload generation '
1757 'issues quickly. Also if an update payload will take a '
1758 'long time to generate, a client may timeout if you do not'
1759 'pregenerate the update.')
1760 group.add_option('--src_image',
1761 metavar='PATH', default='',
1762 help='If specified, delta updates will be generated using '
1763 'this image as the source image. Delta updates are when '
1764 'you are updating from a "source image" to a another '
1765 'image.')
1766 parser.add_option_group(group)
1767
1768
1769def _AddProductionOptions(parser):
1770 group = optparse.OptionGroup(
1771 parser, 'Advanced Server Options', 'These options can be used to changed '
1772 'for advanced server behavior.')
Chris Sosa3ae4dc12013-03-29 11:47:00 -07001773 group.add_option('--clear_cache',
1774 action='store_true', default=False,
1775 help='At startup, removes all cached entries from the'
1776 'devserver\'s cache.')
1777 group.add_option('--logfile',
1778 metavar='PATH',
1779 help='log output to this file instead of stdout')
Chris Sosa855b8932013-08-21 13:24:55 -07001780 group.add_option('--pidfile',
1781 metavar='PATH',
1782 help='path to output a pid file for the server.')
Gilad Arnold11fbef42014-02-10 11:04:13 -08001783 group.add_option('--portfile',
1784 metavar='PATH',
1785 help='path to output the port number being served on.')
Chris Sosa3ae4dc12013-03-29 11:47:00 -07001786 group.add_option('--production',
1787 action='store_true', default=False,
1788 help='have the devserver use production values when '
1789 'starting up. This includes using more threads and '
1790 'performing less logging.')
1791 parser.add_option_group(group)
1792
1793
Paul Hobbsef4e0702016-06-27 17:01:42 -07001794def MakeLogHandler(logfile):
J. Richard Barnette3d977b82013-04-23 11:05:19 -07001795 """Create a LogHandler instance used to log all messages."""
1796 hdlr_cls = handlers.TimedRotatingFileHandler
1797 hdlr = hdlr_cls(logfile, when=_LOG_ROTATION_TIME,
xixuan3d48bff2017-01-30 19:00:09 -08001798 interval=_LOG_ROTATION_INTERVAL,
J. Richard Barnette3d977b82013-04-23 11:05:19 -07001799 backupCount=_LOG_ROTATION_BACKUP)
Chris Sosa855b8932013-08-21 13:24:55 -07001800 hdlr.setFormatter(cplogging.logfmt)
J. Richard Barnette3d977b82013-04-23 11:05:19 -07001801 return hdlr
1802
1803
Chris Sosacde6bf42012-05-31 18:36:39 -07001804def main():
Chris Sosa3ae4dc12013-03-29 11:47:00 -07001805 usage = '\n\n'.join(['usage: %prog [options]', __doc__])
Gilad Arnold286a0062012-01-12 13:47:02 -08001806 parser = optparse.OptionParser(usage=usage)
joychened64b222013-06-21 16:39:34 -07001807
1808 # get directory that the devserver is run from
1809 devserver_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
joychen84d13772013-08-06 09:17:23 -07001810 default_static_dir = '%s/static' % devserver_dir
joychened64b222013-06-21 16:39:34 -07001811 parser.add_option('--static_dir',
Gilad Arnold9714d9b2012-10-04 10:09:42 -07001812 metavar='PATH',
joychen84d13772013-08-06 09:17:23 -07001813 default=default_static_dir,
joychened64b222013-06-21 16:39:34 -07001814 help='writable static directory')
Gilad Arnold9714d9b2012-10-04 10:09:42 -07001815 parser.add_option('--port',
1816 default=8080, type='int',
Gilad Arnoldaf696d12014-02-14 13:13:28 -08001817 help=('port for the dev server to use; if zero, binds to '
1818 'an arbitrary available port (default: 8080)'))
Gilad Arnold9714d9b2012-10-04 10:09:42 -07001819 parser.add_option('-t', '--test_image',
1820 action='store_true',
joychen121fc9b2013-08-02 14:30:30 -07001821 help='Deprecated.')
joychen5260b9a2013-07-16 14:48:01 -07001822 parser.add_option('-x', '--xbuddy_manage_builds',
1823 action='store_true',
1824 default=False,
1825 help='If set, allow xbuddy to manage images in'
1826 'build/images.')
Dan Shi72b16132015-10-08 12:10:33 -07001827 parser.add_option('-a', '--android_build_credential',
1828 default=None,
1829 help='Path to a json file which contains the credential '
1830 'needed to access Android builds.')
Chris Sosa3ae4dc12013-03-29 11:47:00 -07001831 _AddProductionOptions(parser)
1832 _AddUpdateOptions(parser)
1833 _AddTestingOptions(parser)
Chris Sosa7c931362010-10-11 19:49:01 -07001834 (options, _) = parser.parse_args()
rtc@google.com21a5ca32009-11-04 18:23:23 +00001835
J. Richard Barnette3d977b82013-04-23 11:05:19 -07001836 # Handle options that must be set globally in cherrypy. Do this
1837 # work up front, because calls to _Log() below depend on this
1838 # initialization.
1839 if options.production:
1840 cherrypy.config.update({'environment': 'production'})
1841 if not options.logfile:
1842 cherrypy.config.update({'log.screen': True})
1843 else:
1844 cherrypy.config.update({'log.error_file': '',
1845 'log.access_file': ''})
Paul Hobbsef4e0702016-06-27 17:01:42 -07001846 hdlr = MakeLogHandler(options.logfile)
J. Richard Barnette3d977b82013-04-23 11:05:19 -07001847 # Pylint can't seem to process these two calls properly
1848 # pylint: disable=E1101
1849 cherrypy.log.access_log.addHandler(hdlr)
1850 cherrypy.log.error_log.addHandler(hdlr)
1851 # pylint: enable=E1101
1852
joychened64b222013-06-21 16:39:34 -07001853 # set static_dir, from which everything will be served
joychen84d13772013-08-06 09:17:23 -07001854 options.static_dir = os.path.realpath(options.static_dir)
Chris Sosa0356d3b2010-09-16 15:46:22 -07001855
joychened64b222013-06-21 16:39:34 -07001856 cache_dir = os.path.join(options.static_dir, 'cache')
J. Richard Barnette3d977b82013-04-23 11:05:19 -07001857 # If our devserver is only supposed to serve payloads, we shouldn't be
1858 # mucking with the cache at all. If the devserver hadn't previously
1859 # generated a cache and is expected, the caller is using it wrong.
joychen7c2054a2013-07-25 11:14:07 -07001860 if os.path.exists(cache_dir):
Chris Sosadbc20082012-12-10 13:39:11 -08001861 _CleanCache(cache_dir, options.clear_cache)
Chris Sosa6b8c3742011-01-31 12:12:17 -08001862 else:
1863 os.makedirs(cache_dir)
Don Garrettf90edf02010-11-16 17:36:14 -08001864
Chris Sosadbc20082012-12-10 13:39:11 -08001865 _Log('Using cache directory %s' % cache_dir)
joychened64b222013-06-21 16:39:34 -07001866 _Log('Serving from %s' % options.static_dir)
rtc@google.com21a5ca32009-11-04 18:23:23 +00001867
joychen121fc9b2013-08-02 14:30:30 -07001868 _xbuddy = xbuddy.XBuddy(options.xbuddy_manage_builds,
1869 options.board,
joychen121fc9b2013-08-02 14:30:30 -07001870 static_dir=options.static_dir)
Chris Sosa75490802013-09-30 17:21:45 -07001871 if options.clear_cache and options.xbuddy_manage_builds:
1872 _xbuddy.CleanCache()
joychen121fc9b2013-08-02 14:30:30 -07001873
Chris Sosa6a3697f2013-01-29 16:44:43 -08001874 # We allow global use here to share with cherrypy classes.
1875 # pylint: disable=W0603
Chris Sosacde6bf42012-05-31 18:36:39 -07001876 global updater
Andrew de los Reyes52620802010-04-12 13:40:07 -07001877 updater = autoupdate.Autoupdate(
joychen121fc9b2013-08-02 14:30:30 -07001878 _xbuddy,
joychened64b222013-06-21 16:39:34 -07001879 static_dir=options.static_dir,
Andrew de los Reyes52620802010-04-12 13:40:07 -07001880 urlbase=options.urlbase,
Chris Sosa5d342a22010-09-28 16:54:41 -07001881 forced_image=options.image,
Gilad Arnold0c9c8602012-10-02 23:58:58 -07001882 payload_path=options.payload,
Don Garrett0ad09372010-12-06 16:20:30 -08001883 proxy_port=options.proxy_port,
Chris Sosa4136e692010-10-28 23:42:37 -07001884 src_image=options.src_image,
Chris Sosa08d55a22011-01-19 16:08:02 -08001885 board=options.board,
Chris Sosa0f1ec842011-02-14 16:33:22 -08001886 copy_to_static_root=not options.exit,
1887 private_key=options.private_key,
Gabe Black3b567202015-09-23 14:07:59 -07001888 private_key_for_metadata_hash_signature=(
1889 options.private_key_for_metadata_hash_signature),
David Zeuthen52ccd012013-10-31 12:58:26 -07001890 public_key=options.public_key,
Satoru Takabayashid733cbe2011-11-15 09:36:32 -08001891 critical_update=options.critical_update,
Gilad Arnold0c9c8602012-10-02 23:58:58 -07001892 remote_payload=options.remote_payload,
Gilad Arnolda564b4b2012-10-04 10:32:44 -07001893 max_updates=options.max_updates,
Gilad Arnold8318eac2012-10-04 12:52:23 -07001894 host_log=options.host_log,
Chris Sosa0f1ec842011-02-14 16:33:22 -08001895 )
Chris Sosa7c931362010-10-11 19:49:01 -07001896
Chris Sosa6a3697f2013-01-29 16:44:43 -08001897 if options.pregenerate_update:
1898 updater.PreGenerateUpdate()
Chris Sosa0356d3b2010-09-16 15:46:22 -07001899
J. Richard Barnette3d977b82013-04-23 11:05:19 -07001900 if options.exit:
1901 return
Chris Sosa2f1c41e2012-07-10 14:32:33 -07001902
joychen3cb228e2013-06-12 12:13:13 -07001903 dev_server = DevServerRoot(_xbuddy)
1904
Gilad Arnold11fbef42014-02-10 11:04:13 -08001905 # Patch CherryPy to support binding to any available port (--port=0).
1906 cherrypy_ext.ZeroPortPatcher.DoPatch(cherrypy)
1907
Chris Sosa855b8932013-08-21 13:24:55 -07001908 if options.pidfile:
1909 plugins.PIDFile(cherrypy.engine, options.pidfile).subscribe()
1910
Gilad Arnold11fbef42014-02-10 11:04:13 -08001911 if options.portfile:
1912 cherrypy_ext.PortFile(cherrypy.engine, options.portfile).subscribe()
1913
Dan Shiafd5c6c2016-01-07 10:27:03 -08001914 if (options.android_build_credential and
1915 os.path.exists(options.android_build_credential)):
1916 try:
1917 with open(options.android_build_credential) as f:
1918 android_build.BuildAccessor.credential_info = json.load(f)
1919 except ValueError as e:
1920 _Log('Failed to load the android build credential: %s. Error: %s.' %
1921 (options.android_build_credential, e))
joychen3cb228e2013-06-12 12:13:13 -07001922 cherrypy.quickstart(dev_server, config=_GetConfig(options))
Chris Sosacde6bf42012-05-31 18:36:39 -07001923
1924
1925if __name__ == '__main__':
1926 main()