Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 1 | # -*- coding: utf-8 -*- |
| 2 | # Copyright 2019 The Chromium OS Authors. All rights reserved. |
| 3 | # Use of this source code is governed by a BSD-style license that can be |
| 4 | # found in the LICENSE file. |
| 5 | |
Alex Klein | e0fa642 | 2019-06-21 12:01:39 -0600 | [diff] [blame] | 6 | """Router class for the Build API. |
| 7 | |
| 8 | Handles routing requests to the appropriate controller and handles service |
| 9 | registration. |
| 10 | """ |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 11 | |
| 12 | from __future__ import print_function |
| 13 | |
| 14 | import importlib |
| 15 | import os |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 16 | |
| 17 | from google.protobuf import json_format |
| 18 | from google.protobuf import symbol_database |
| 19 | |
| 20 | from chromite.api import controller |
| 21 | from chromite.api import field_handler |
Alex Klein | 4de25e8 | 2019-08-05 15:58:39 -0600 | [diff] [blame] | 22 | from chromite.api.gen.chromite.api import android_pb2 |
Alex Klein | 54e38e3 | 2019-06-21 14:54:17 -0600 | [diff] [blame] | 23 | from chromite.api.gen.chromite.api import api_pb2 |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 24 | from chromite.api.gen.chromite.api import artifacts_pb2 |
| 25 | from chromite.api.gen.chromite.api import binhost_pb2 |
| 26 | from chromite.api.gen.chromite.api import build_api_pb2 |
| 27 | from chromite.api.gen.chromite.api import depgraph_pb2 |
| 28 | from chromite.api.gen.chromite.api import image_pb2 |
Alex Klein | eb77ffa | 2019-05-28 14:47:44 -0600 | [diff] [blame] | 29 | from chromite.api.gen.chromite.api import packages_pb2 |
George Engelbrecht | fe63c8c | 2019-08-31 22:51:29 -0600 | [diff] [blame] | 30 | from chromite.api.gen.chromite.api import payload_pb2 |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 31 | from chromite.api.gen.chromite.api import sdk_pb2 |
| 32 | from chromite.api.gen.chromite.api import sysroot_pb2 |
| 33 | from chromite.api.gen.chromite.api import test_pb2 |
Tiancong Wang | af05017 | 2019-07-10 11:52:03 -0700 | [diff] [blame] | 34 | from chromite.api.gen.chromite.api import toolchain_pb2 |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 35 | from chromite.lib import cros_build_lib |
| 36 | from chromite.lib import cros_logging as logging |
| 37 | from chromite.lib import osutils |
| 38 | |
| 39 | |
| 40 | class Error(Exception): |
| 41 | """Base error class for the module.""" |
| 42 | |
| 43 | |
| 44 | class InvalidInputFileError(Error): |
| 45 | """Raised when the input file cannot be read.""" |
| 46 | |
| 47 | |
| 48 | class InvalidInputFormatError(Error): |
| 49 | """Raised when the passed input protobuf can't be parsed.""" |
| 50 | |
| 51 | |
| 52 | class InvalidOutputFileError(Error): |
| 53 | """Raised when the output file cannot be written.""" |
| 54 | |
| 55 | |
| 56 | class CrosSdkNotRunError(Error): |
| 57 | """Raised when the cros_sdk command could not be run to enter the chroot.""" |
| 58 | |
| 59 | |
| 60 | # API Service Errors. |
| 61 | class UnknownServiceError(Error): |
| 62 | """Error raised when the requested service has not been registered.""" |
| 63 | |
| 64 | |
| 65 | class ControllerModuleNotDefinedError(Error): |
| 66 | """Error class for when no controller is defined for a service.""" |
| 67 | |
| 68 | |
| 69 | class ServiceControllerNotFoundError(Error): |
| 70 | """Error raised when the service's controller cannot be imported.""" |
| 71 | |
| 72 | |
| 73 | # API Method Errors. |
| 74 | class UnknownMethodError(Error): |
| 75 | """The service is defined in the proto but the method is not.""" |
| 76 | |
| 77 | |
| 78 | class MethodNotFoundError(Error): |
| 79 | """The method's implementation cannot be found in the service's controller.""" |
| 80 | |
| 81 | |
| 82 | class Router(object): |
| 83 | """Encapsulates the request dispatching logic.""" |
| 84 | |
Alex Klein | bd6edf8 | 2019-07-18 10:30:49 -0600 | [diff] [blame] | 85 | REEXEC_INPUT_FILE = 'input.json' |
| 86 | REEXEC_OUTPUT_FILE = 'output.json' |
| 87 | |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 88 | def __init__(self): |
| 89 | self._services = {} |
| 90 | self._aliases = {} |
| 91 | # All imported generated messages get added to this symbol db. |
| 92 | self._sym_db = symbol_database.Default() |
| 93 | |
| 94 | extensions = build_api_pb2.DESCRIPTOR.extensions_by_name |
| 95 | self._service_options = extensions['service_options'] |
| 96 | self._method_options = extensions['method_options'] |
| 97 | |
| 98 | def Register(self, proto_module): |
| 99 | """Register the services from a generated proto module. |
| 100 | |
| 101 | Args: |
| 102 | proto_module (module): The generated proto module whose service is being |
| 103 | registered. |
| 104 | |
| 105 | Raises: |
| 106 | ServiceModuleNotDefinedError when the service cannot be found in the |
| 107 | provided module. |
| 108 | """ |
| 109 | services = proto_module.DESCRIPTOR.services_by_name |
| 110 | for service_name, svc in services.items(): |
| 111 | module_name = svc.GetOptions().Extensions[self._service_options].module |
| 112 | |
| 113 | if not module_name: |
| 114 | raise ControllerModuleNotDefinedError( |
| 115 | 'The module must be defined in the service definition: %s.%s' % |
| 116 | (proto_module, service_name)) |
| 117 | |
| 118 | self._services[svc.full_name] = (svc, module_name) |
| 119 | |
| 120 | def ListMethods(self): |
| 121 | """List all methods registered with the router.""" |
| 122 | services = [] |
| 123 | for service_name, (svc, _module) in self._services.items(): |
| 124 | for method_name in svc.methods_by_name.keys(): |
| 125 | services.append('%s/%s' % (service_name, method_name)) |
| 126 | |
| 127 | return sorted(services) |
| 128 | |
Alex Klein | 69339cc | 2019-07-22 14:08:35 -0600 | [diff] [blame] | 129 | def Route(self, service_name, method_name, input_path, output_path, config): |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 130 | """Dispatch the request. |
| 131 | |
| 132 | Args: |
| 133 | service_name (str): The fully qualified service name. |
| 134 | method_name (str): The name of the method being called. |
| 135 | input_path (str): The path to the input message file. |
| 136 | output_path (str): The path where the output message should be written. |
Alex Klein | 69339cc | 2019-07-22 14:08:35 -0600 | [diff] [blame] | 137 | config (api_config.ApiConfig): The optional call configs. |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 138 | |
| 139 | Returns: |
| 140 | int: The return code. |
| 141 | |
| 142 | Raises: |
| 143 | InvalidInputFileError when the input file cannot be read. |
| 144 | InvalidOutputFileError when the output file cannot be written. |
| 145 | ServiceModuleNotFoundError when the service module cannot be imported. |
| 146 | MethodNotFoundError when the method cannot be retrieved from the module. |
| 147 | """ |
| 148 | try: |
| 149 | input_json = osutils.ReadFile(input_path).strip() |
| 150 | except IOError as e: |
Mike Frysinger | 6b5c3cd | 2019-08-27 16:51:00 -0400 | [diff] [blame] | 151 | raise InvalidInputFileError('Unable to read input file: %s' % e) |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 152 | |
| 153 | try: |
| 154 | svc, module_name = self._services[service_name] |
| 155 | except KeyError: |
| 156 | raise UnknownServiceError('The %s service has not been registered.' |
| 157 | % service_name) |
| 158 | |
| 159 | try: |
| 160 | method_desc = svc.methods_by_name[method_name] |
| 161 | except KeyError: |
| 162 | raise UnknownMethodError('The %s method has not been defined in the %s ' |
| 163 | 'service.' % (method_name, service_name)) |
| 164 | |
| 165 | # Parse the input file to build an instance of the input message. |
| 166 | input_msg = self._sym_db.GetPrototype(method_desc.input_type)() |
| 167 | try: |
| 168 | json_format.Parse(input_json, input_msg, ignore_unknown_fields=True) |
| 169 | except json_format.ParseError as e: |
Mike Frysinger | 6b5c3cd | 2019-08-27 16:51:00 -0400 | [diff] [blame] | 170 | raise InvalidInputFormatError('Unable to parse the input json: %s' % e) |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 171 | |
| 172 | # Get an empty output message instance. |
| 173 | output_msg = self._sym_db.GetPrototype(method_desc.output_type)() |
| 174 | |
Alex Klein | bd6edf8 | 2019-07-18 10:30:49 -0600 | [diff] [blame] | 175 | # Fetch the method options for chroot and method name overrides. |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 176 | method_options = method_desc.GetOptions().Extensions[self._method_options] |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 177 | |
| 178 | # Check the chroot settings before running. |
| 179 | service_options = svc.GetOptions().Extensions[self._service_options] |
| 180 | if self._ChrootCheck(service_options, method_options): |
| 181 | # Run inside the chroot instead. |
| 182 | logging.info('Re-executing the endpoint inside the chroot.') |
Alex Klein | bd6edf8 | 2019-07-18 10:30:49 -0600 | [diff] [blame] | 183 | return self._ReexecuteInside(input_msg, output_msg, output_path, |
Alex Klein | 69339cc | 2019-07-22 14:08:35 -0600 | [diff] [blame] | 184 | service_name, method_name, config) |
Alex Klein | bd6edf8 | 2019-07-18 10:30:49 -0600 | [diff] [blame] | 185 | |
| 186 | # Allow proto-based method name override. |
| 187 | if method_options.HasField('implementation_name'): |
| 188 | method_name = method_options.implementation_name |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 189 | |
| 190 | # Import the module and get the method. |
| 191 | method_impl = self._GetMethod(module_name, method_name) |
| 192 | |
| 193 | # Successfully located; call and return. |
Alex Klein | 69339cc | 2019-07-22 14:08:35 -0600 | [diff] [blame] | 194 | return_code = method_impl(input_msg, output_msg, config) |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 195 | if return_code is None: |
| 196 | return_code = controller.RETURN_CODE_SUCCESS |
| 197 | |
| 198 | try: |
| 199 | osutils.WriteFile(output_path, json_format.MessageToJson(output_msg)) |
| 200 | except IOError as e: |
Mike Frysinger | 6b5c3cd | 2019-08-27 16:51:00 -0400 | [diff] [blame] | 201 | raise InvalidOutputFileError('Cannot write output file: %s' % e) |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 202 | |
| 203 | return return_code |
| 204 | |
| 205 | def _ChrootCheck(self, service_options, method_options): |
| 206 | """Check the chroot options, and execute assertion or note reexec as needed. |
| 207 | |
| 208 | Args: |
| 209 | service_options (google.protobuf.Message): The service options. |
| 210 | method_options (google.protobuf.Message): The method options. |
| 211 | |
| 212 | Returns: |
| 213 | bool - True iff it needs to be reexeced inside the chroot. |
| 214 | |
| 215 | Raises: |
| 216 | cros_build_lib.DieSystemExit when the chroot setting cannot be satisfied. |
| 217 | """ |
| 218 | chroot_assert = build_api_pb2.NO_ASSERTION |
| 219 | if method_options.HasField('method_chroot_assert'): |
| 220 | # Prefer the method option when set. |
| 221 | chroot_assert = method_options.method_chroot_assert |
| 222 | elif service_options.HasField('service_chroot_assert'): |
| 223 | # Fall back to the service option. |
| 224 | chroot_assert = service_options.service_chroot_assert |
| 225 | |
| 226 | if chroot_assert == build_api_pb2.INSIDE: |
| 227 | return not cros_build_lib.IsInsideChroot() |
| 228 | elif chroot_assert == build_api_pb2.OUTSIDE: |
| 229 | # If it must be run outside we have to already be outside. |
| 230 | cros_build_lib.AssertOutsideChroot() |
| 231 | |
| 232 | return False |
| 233 | |
Alex Klein | bd6edf8 | 2019-07-18 10:30:49 -0600 | [diff] [blame] | 234 | def _ReexecuteInside(self, input_msg, output_msg, output_path, service_name, |
Alex Klein | 69339cc | 2019-07-22 14:08:35 -0600 | [diff] [blame] | 235 | method_name, config): |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 236 | """Re-execute the service inside the chroot. |
| 237 | |
| 238 | Args: |
| 239 | input_msg (Message): The parsed input message. |
Alex Klein | bd6edf8 | 2019-07-18 10:30:49 -0600 | [diff] [blame] | 240 | output_msg (Message): The empty output message instance. |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 241 | output_path (str): The path for the serialized output. |
| 242 | service_name (str): The name of the service to run. |
| 243 | method_name (str): The name of the method to run. |
Alex Klein | 69339cc | 2019-07-22 14:08:35 -0600 | [diff] [blame] | 244 | config (api_config.ApiConfig): The optional call configs. |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 245 | """ |
| 246 | # Parse the chroot and clear the chroot field in the input message. |
| 247 | chroot = field_handler.handle_chroot(input_msg) |
| 248 | |
Alex Klein | aae4977 | 2019-07-26 10:20:50 -0600 | [diff] [blame] | 249 | with field_handler.copy_paths_in(input_msg, chroot.tmp, prefix=chroot.path): |
Alex Klein | f0717a6 | 2019-12-06 09:45:00 -0700 | [diff] [blame^] | 250 | with chroot.tempdir() as tempdir, chroot.tempdir() as sync_tmp: |
| 251 | with field_handler.sync_dirs(input_msg, sync_tmp, prefix=chroot.path): |
| 252 | new_input = os.path.join(tempdir, self.REEXEC_INPUT_FILE) |
| 253 | chroot_input = '/%s' % os.path.relpath(new_input, chroot.path) |
| 254 | new_output = os.path.join(tempdir, self.REEXEC_OUTPUT_FILE) |
| 255 | chroot_output = '/%s' % os.path.relpath(new_output, chroot.path) |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 256 | |
Alex Klein | f0717a6 | 2019-12-06 09:45:00 -0700 | [diff] [blame^] | 257 | logging.info('Writing input message to: %s', new_input) |
| 258 | osutils.WriteFile(new_input, json_format.MessageToJson(input_msg)) |
| 259 | osutils.Touch(new_output) |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 260 | |
Alex Klein | f0717a6 | 2019-12-06 09:45:00 -0700 | [diff] [blame^] | 261 | cmd = ['build_api', '%s/%s' % (service_name, method_name), |
| 262 | '--input-json', chroot_input, '--output-json', chroot_output] |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 263 | |
Alex Klein | f0717a6 | 2019-12-06 09:45:00 -0700 | [diff] [blame^] | 264 | if config.validate_only: |
| 265 | cmd.append('--validate-only') |
Alex Klein | 69339cc | 2019-07-22 14:08:35 -0600 | [diff] [blame] | 266 | |
Alex Klein | f0717a6 | 2019-12-06 09:45:00 -0700 | [diff] [blame^] | 267 | try: |
| 268 | result = cros_build_lib.run( |
| 269 | cmd, |
| 270 | enter_chroot=True, |
| 271 | chroot_args=chroot.get_enter_args(), |
| 272 | error_code_ok=True, |
| 273 | extra_env=chroot.env) |
| 274 | except cros_build_lib.RunCommandError: |
| 275 | # A non-zero return code will not result in an error, but one |
| 276 | # is still thrown when the command cannot be run in the first |
| 277 | # place. This is known to happen at least when the PATH does |
| 278 | # not include the chromite bin dir. |
| 279 | raise CrosSdkNotRunError('Unable to enter the chroot.') |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 280 | |
Alex Klein | f0717a6 | 2019-12-06 09:45:00 -0700 | [diff] [blame^] | 281 | logging.info('Endpoint execution completed, return code: %d', |
| 282 | result.returncode) |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 283 | |
Alex Klein | f0717a6 | 2019-12-06 09:45:00 -0700 | [diff] [blame^] | 284 | # Transfer result files out of the chroot. |
| 285 | output_content = osutils.ReadFile(new_output) |
| 286 | if output_content: |
| 287 | json_format.Parse(output_content, output_msg) |
| 288 | field_handler.extract_results(input_msg, output_msg, chroot) |
Alex Klein | bd6edf8 | 2019-07-18 10:30:49 -0600 | [diff] [blame] | 289 | |
Alex Klein | f0717a6 | 2019-12-06 09:45:00 -0700 | [diff] [blame^] | 290 | osutils.WriteFile(output_path, json_format.MessageToJson(output_msg)) |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 291 | |
Alex Klein | f0717a6 | 2019-12-06 09:45:00 -0700 | [diff] [blame^] | 292 | return result.returncode |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 293 | |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 294 | def _GetMethod(self, module_name, method_name): |
| 295 | """Get the implementation of the method for the service module. |
| 296 | |
| 297 | Args: |
| 298 | module_name (str): The name of the service module. |
| 299 | method_name (str): The name of the method. |
| 300 | |
| 301 | Returns: |
| 302 | callable - The method. |
| 303 | |
| 304 | Raises: |
| 305 | MethodNotFoundError when the method cannot be found in the module. |
| 306 | ServiceModuleNotFoundError when the service module cannot be imported. |
| 307 | """ |
| 308 | try: |
| 309 | module = importlib.import_module(controller.IMPORT_PATTERN % module_name) |
| 310 | except ImportError as e: |
Mike Frysinger | 6b5c3cd | 2019-08-27 16:51:00 -0400 | [diff] [blame] | 311 | raise ServiceControllerNotFoundError(str(e)) |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 312 | try: |
| 313 | return getattr(module, method_name) |
| 314 | except AttributeError as e: |
Mike Frysinger | 6b5c3cd | 2019-08-27 16:51:00 -0400 | [diff] [blame] | 315 | raise MethodNotFoundError(str(e)) |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 316 | |
| 317 | |
| 318 | def RegisterServices(router): |
| 319 | """Register all the services. |
| 320 | |
| 321 | Args: |
| 322 | router (Router): The router. |
| 323 | """ |
Alex Klein | 4de25e8 | 2019-08-05 15:58:39 -0600 | [diff] [blame] | 324 | router.Register(android_pb2) |
Alex Klein | 54e38e3 | 2019-06-21 14:54:17 -0600 | [diff] [blame] | 325 | router.Register(api_pb2) |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 326 | router.Register(artifacts_pb2) |
| 327 | router.Register(binhost_pb2) |
| 328 | router.Register(depgraph_pb2) |
| 329 | router.Register(image_pb2) |
Alex Klein | eb77ffa | 2019-05-28 14:47:44 -0600 | [diff] [blame] | 330 | router.Register(packages_pb2) |
George Engelbrecht | fe63c8c | 2019-08-31 22:51:29 -0600 | [diff] [blame] | 331 | router.Register(payload_pb2) |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 332 | router.Register(sdk_pb2) |
| 333 | router.Register(sysroot_pb2) |
| 334 | router.Register(test_pb2) |
Tiancong Wang | af05017 | 2019-07-10 11:52:03 -0700 | [diff] [blame] | 335 | router.Register(toolchain_pb2) |
Alex Klein | 146d477 | 2019-06-20 13:48:25 -0600 | [diff] [blame] | 336 | logging.debug('Services registered successfully.') |
| 337 | |
| 338 | |
| 339 | def GetRouter(): |
| 340 | """Get a router that has had all of the services registered.""" |
| 341 | router = Router() |
| 342 | RegisterServices(router) |
| 343 | |
| 344 | return router |