blob: 6c51d4f56e3ffa0c3c449ffd1a90990dd8c00b78 [file] [log] [blame]
Evan Hernandezf388cbf2019-04-01 11:15:23 -06001# Copyright 2019 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Implements ArtifactService."""
6
Evan Hernandezf388cbf2019-04-01 11:15:23 -06007import os
George Engelbrechtc9a8e812021-06-16 18:14:17 -06008from typing import Any, NamedTuple
Evan Hernandezf388cbf2019-04-01 11:15:23 -06009
Alex Klein231d2da2019-07-22 16:44:45 -060010from chromite.api import controller
Alex Klein076841b2019-08-29 15:19:39 -060011from chromite.api import faux
Alex Klein2b236722019-06-19 15:44:26 -060012from chromite.api import validate
Alex Klein238d8862019-05-07 11:32:46 -060013from chromite.api.controller import controller_util
George Engelbrechtc9a8e812021-06-16 18:14:17 -060014from chromite.api.controller import image as image_controller
15from chromite.api.controller import sysroot as sysroot_controller
LaMont Jones58362a42021-02-04 17:40:08 -070016from chromite.api.gen.chromite.api import artifacts_pb2
Tiancong Wang24a3df72019-08-20 15:48:51 -070017from chromite.api.gen.chromite.api import toolchain_pb2
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000018from chromite.api.gen.chromiumos import common_pb2
Alex Klein2275d692019-04-23 16:04:12 -060019from chromite.lib import chroot_lib
Evan Hernandezf388cbf2019-04-01 11:15:23 -060020from chromite.lib import constants
21from chromite.lib import cros_build_lib
Evan Hernandezde445982019-04-22 13:42:34 -060022from chromite.lib import cros_logging as logging
Alex Klein2275d692019-04-23 16:04:12 -060023from chromite.lib import sysroot_lib
24from chromite.service import artifacts
Evan Hernandezf388cbf2019-04-01 11:15:23 -060025
26
George Engelbrechtc9a8e812021-06-16 18:14:17 -060027class RegisteredGet(NamedTuple):
28 """An registered function for calling Get on an artifact type."""
29 output_proto: artifacts_pb2.GetResponse
30 artifact_dict: Any
LaMont Jones0f5171b2021-01-29 12:28:56 -070031
32
George Engelbrechtc9a8e812021-06-16 18:14:17 -060033def ExampleGetResponse(_input_proto, _output_proto, _config):
34 """Give an example GetResponse with a minimal coverage set."""
35 _output_proto = artifacts_pb2.GetResponse(
36 artifacts=common_pb2.UploadedArtifactsByService(
37 image=image_controller.ExampleGetResponse(),
38 sysroot=sysroot_controller.ExampleGetResponse(),
39 ))
40 return controller.RETURN_CODE_SUCCESS
41
42
LaMont Jones0f5171b2021-01-29 12:28:56 -070043@faux.empty_error
George Engelbrechtc9a8e812021-06-16 18:14:17 -060044@faux.success(ExampleGetResponse)
LaMont Jones0f5171b2021-01-29 12:28:56 -070045@validate.exists('result_path.path.path')
46@validate.validation_complete
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000047def Get(input_proto, output_proto, _config):
LaMont Jones0f5171b2021-01-29 12:28:56 -070048 """Get all artifacts.
49
50 Get all artifacts for the build.
51
George Engelbrechtc9a8e812021-06-16 18:14:17 -060052 Note: As the individual artifact_type bundlers are added here, they *must*
53 stop uploading it via the individual bundler function.
LaMont Jones0f5171b2021-01-29 12:28:56 -070054
55 Args:
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000056 input_proto (GetRequest): The input proto.
57 output_proto (GetResponse): The output proto.
LaMont Jones0f5171b2021-01-29 12:28:56 -070058 _config (api_config.ApiConfig): The API call config.
59 """
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000060 output_dir = input_proto.result_path.path.path
61
George Engelbrechtc9a8e812021-06-16 18:14:17 -060062 sysroot = controller_util.ParseSysroot(input_proto.sysroot)
LaMont Jones8b88e9d2021-06-28 16:37:34 -060063 # This endpoint does not currently support any artifacts that are built
64 # without a sysroot being present.
65 if not sysroot.path:
66 return controller.RETURN_CODE_SUCCESS
67
George Engelbrechtc9a8e812021-06-16 18:14:17 -060068 chroot = controller_util.ParseChroot(input_proto.chroot)
69 build_target = controller_util.ParseBuildTarget(
70 input_proto.sysroot.build_target)
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000071
George Engelbrechtc9a8e812021-06-16 18:14:17 -060072 # A list of RegisteredGet tuples (input proto, output proto, get results).
73 get_res_list = [
74 RegisteredGet(
75 output_proto.artifacts.image,
76 image_controller.GetArtifacts(
77 input_proto.artifact_info.image, chroot, sysroot, build_target,
78 output_dir)),
79 RegisteredGet(
80 output_proto.artifacts.sysroot,
81 sysroot_controller.GetArtifacts(
82 input_proto.artifact_info.sysroot, chroot, sysroot, build_target,
83 output_dir))
84 ]
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000085
George Engelbrechtc9a8e812021-06-16 18:14:17 -060086 for get_res in get_res_list:
87 for artifact_dict in get_res.artifact_dict:
88 get_res.output_proto.artifacts.add(
89 artifact_type=artifact_dict['type'],
90 paths=[
91 common_pb2.Path(
92 path=x, location=common_pb2.Path.Location.OUTSIDE)
93 for x in artifact_dict['paths']
94 ])
LaMont Jones0f5171b2021-01-29 12:28:56 -070095 return controller.RETURN_CODE_SUCCESS
96
97
LaMont Jones58362a42021-02-04 17:40:08 -070098def _BuildSetupResponse(_input_proto, output_proto, _config):
99 """Just return POINTLESS for now."""
100 # All of the artifact types we support claim that the build is POINTLESS.
101 output_proto.build_relevance = artifacts_pb2.BuildSetupResponse.POINTLESS
102
103
104@faux.success(_BuildSetupResponse)
105@faux.empty_error
106@validate.validation_complete
107def BuildSetup(_input_proto, output_proto, _config):
108 """Setup anything needed for building artifacts
109
110 If any artifact types require steps prior to building the package, they go
111 here. For example, see ToolchainService/PrepareForBuild.
112
113 Note: crbug/1034529 introduces this method as a noop. As the individual
114 artifact_type bundlers are added here, they *must* stop uploading it via the
115 individual bundler function.
116
117 Args:
118 _input_proto (GetRequest): The input proto.
119 output_proto (GetResponse): The output proto.
120 _config (api_config.ApiConfig): The API call config.
121 """
122 # If any artifact_type says "NEEDED", the return is NEEDED.
123 # Otherwise, if any artifact_type says "UNKNOWN", the return is UNKNOWN.
124 # Otherwise, the return is POINTLESS.
125 output_proto.build_relevance = artifacts_pb2.BuildSetupResponse.POINTLESS
126 return controller.RETURN_CODE_SUCCESS
127
128
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600129def _GetImageDir(build_root, target):
130 """Return path containing images for the given build target.
131
Alex Kleine2612a02019-04-18 13:51:06 -0600132 TODO(saklein) Expand image_lib.GetLatestImageLink to support this use case.
133
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600134 Args:
135 build_root (str): Path to checkout where build occurs.
136 target (str): Name of the build target.
137
138 Returns:
Alex Kleind2bf1462019-10-24 16:37:04 -0600139 Path to the latest directory containing target images or None.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600140 """
141 image_dir = os.path.join(build_root, 'src/build/images', target, 'latest')
142 if not os.path.exists(image_dir):
Alex Kleind2bf1462019-10-24 16:37:04 -0600143 logging.warning('Expected to find image output for target %s at %s, but '
144 'path does not exist', target, image_dir)
145 return None
146
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600147 return image_dir
148
149
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700150def _BundleImageArchivesResponse(input_proto, output_proto, _config):
151 """Add artifact paths to a successful response."""
152 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
153 'path0.tar.xz')
154 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
155 'path1.tar.xz')
156
157
158@faux.success(_BundleImageArchivesResponse)
159@faux.empty_error
Alex Kleind91e95a2019-09-17 10:39:02 -0600160@validate.require('build_target.name')
161@validate.exists('output_dir')
162@validate.validation_complete
163def BundleImageArchives(input_proto, output_proto, _config):
164 """Create a .tar.xz archive for each image that has been created."""
165 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
166 output_dir = input_proto.output_dir
167 image_dir = _GetImageDir(constants.SOURCE_ROOT, build_target.name)
Alex Kleind2bf1462019-10-24 16:37:04 -0600168 if image_dir is None:
169 return
Alex Kleind91e95a2019-09-17 10:39:02 -0600170
171 archives = artifacts.ArchiveImages(image_dir, output_dir)
172
173 for archive in archives:
174 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
175
176
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700177def _BundleImageZipResponse(input_proto, output_proto, _config):
178 """Add artifact zip files to a successful response."""
179 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
180 'image.zip')
181
182
183@faux.success(_BundleImageZipResponse)
184@faux.empty_error
Michael Mortensen01910922019-07-24 14:48:10 -0600185@validate.require('build_target.name', 'output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600186@validate.exists('output_dir')
187@validate.validation_complete
188def BundleImageZip(input_proto, output_proto, _config):
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600189 """Bundle image.zip.
190
191 Args:
192 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600193 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600194 _config (api_config.ApiConfig): The API call config.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600195 """
196 target = input_proto.build_target.name
197 output_dir = input_proto.output_dir
198 image_dir = _GetImageDir(constants.SOURCE_ROOT, target)
Alex Kleind2bf1462019-10-24 16:37:04 -0600199 if image_dir is None:
200 return None
Alex Klein231d2da2019-07-22 16:44:45 -0600201
Michael Mortensen01910922019-07-24 14:48:10 -0600202 archive = artifacts.BundleImageZip(output_dir, image_dir)
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600203 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
204
205
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700206def _BundleTestUpdatePayloadsResponse(input_proto, output_proto, _config):
207 """Add test payload files to a successful response."""
208 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
209 'payload1.bin')
210
211
212@faux.success(_BundleTestUpdatePayloadsResponse)
213@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600214@validate.require('build_target.name', 'output_dir')
215@validate.exists('output_dir')
216@validate.validation_complete
217def BundleTestUpdatePayloads(input_proto, output_proto, _config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600218 """Generate minimal update payloads for the build target for testing.
219
220 Args:
221 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600222 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600223 _config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600224 """
225 target = input_proto.build_target.name
226 output_dir = input_proto.output_dir
227 build_root = constants.SOURCE_ROOT
228
229 # Use the first available image to create the update payload.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600230 img_dir = _GetImageDir(build_root, target)
Alex Kleind2bf1462019-10-24 16:37:04 -0600231 if img_dir is None:
232 return None
233
Alex Kleincb541e82019-06-26 15:06:11 -0600234 img_types = [constants.IMAGE_TYPE_TEST, constants.IMAGE_TYPE_DEV,
235 constants.IMAGE_TYPE_BASE]
236 img_names = [constants.IMAGE_TYPE_TO_NAME[t] for t in img_types]
Mike Frysinger66ce4132019-07-17 22:52:52 -0400237 img_paths = [os.path.join(img_dir, x) for x in img_names]
Mike Frysingera552be42018-08-17 14:39:32 -0400238 valid_images = [x for x in img_paths if os.path.exists(x)]
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600239
Alex Kleincb541e82019-06-26 15:06:11 -0600240 if not valid_images:
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600241 cros_build_lib.Die(
242 'Expected to find an image of type among %r for target "%s" '
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600243 'at path %s.', img_types, target, img_dir)
Alex Kleincb541e82019-06-26 15:06:11 -0600244 image = valid_images[0]
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600245
Alex Kleincb541e82019-06-26 15:06:11 -0600246 payloads = artifacts.BundleTestUpdatePayloads(image, output_dir)
247 for payload in payloads:
248 output_proto.artifacts.add().path = payload
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600249
250
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700251def _BundleAutotestFilesResponse(input_proto, output_proto, _config):
252 """Add test autotest files to a successful response."""
253 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
254 'autotest-a.tar.gz')
255
256
257@faux.success(_BundleAutotestFilesResponse)
258@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600259@validate.require('output_dir')
260@validate.exists('output_dir')
261def BundleAutotestFiles(input_proto, output_proto, config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600262 """Tar the autotest files for a build target.
263
264 Args:
265 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600266 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600267 config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600268 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600269 output_dir = input_proto.output_dir
Alex Klein238d8862019-05-07 11:32:46 -0600270 target = input_proto.build_target.name
Alex Kleine21a0952019-08-23 16:08:16 -0600271 chroot = controller_util.ParseChroot(input_proto.chroot)
272
Alex Klein238d8862019-05-07 11:32:46 -0600273 if target:
Alex Kleine21a0952019-08-23 16:08:16 -0600274 sysroot_path = os.path.join('/build', target)
Alex Klein238d8862019-05-07 11:32:46 -0600275 else:
276 # New style call, use chroot and sysroot.
Alex Klein238d8862019-05-07 11:32:46 -0600277 sysroot_path = input_proto.sysroot.path
278 if not sysroot_path:
279 cros_build_lib.Die('sysroot.path is required.')
280
Alex Kleine21a0952019-08-23 16:08:16 -0600281 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein238d8862019-05-07 11:32:46 -0600282
Alex Klein231d2da2019-07-22 16:44:45 -0600283 # TODO(saklein): Switch to the validate_only decorator when legacy handling
284 # is removed.
285 if config.validate_only:
286 return controller.RETURN_CODE_VALID_INPUT
287
Alex Kleine21a0952019-08-23 16:08:16 -0600288 if not sysroot.Exists(chroot=chroot):
Alex Klein238d8862019-05-07 11:32:46 -0600289 cros_build_lib.Die('Sysroot path must exist: %s', sysroot.path)
290
291 try:
292 # Note that this returns the full path to *multiple* tarballs.
Alex Kleine21a0952019-08-23 16:08:16 -0600293 archives = artifacts.BundleAutotestFiles(chroot, sysroot, output_dir)
Alex Klein238d8862019-05-07 11:32:46 -0600294 except artifacts.Error as e:
Alex Klein03890312020-06-30 09:59:50 -0600295 logging.warning(e)
296 return
Alex Klein238d8862019-05-07 11:32:46 -0600297
298 for archive in archives.values():
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600299 output_proto.artifacts.add().path = archive
300
301
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700302def _BundleTastFilesResponse(input_proto, output_proto, _config):
303 """Add test tast files to a successful response."""
304 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
305 'tast_bundles.tar.gz')
306
307
308@faux.success(_BundleTastFilesResponse)
309@faux.empty_error
Alex Kleinb9d810b2019-07-01 12:38:02 -0600310@validate.require('output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600311@validate.exists('output_dir')
312def BundleTastFiles(input_proto, output_proto, config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600313 """Tar the tast files for a build target.
314
315 Args:
316 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600317 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600318 config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600319 """
320 target = input_proto.build_target.name
321 output_dir = input_proto.output_dir
322 build_root = constants.SOURCE_ROOT
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600323
Alex Kleinb9d810b2019-07-01 12:38:02 -0600324 chroot = controller_util.ParseChroot(input_proto.chroot)
325 sysroot_path = input_proto.sysroot.path
326
327 # TODO(saklein) Cleanup legacy handling after it has been switched over.
328 if target:
329 # Legacy handling.
Alex Klein171da612019-08-06 14:00:42 -0600330 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
Alex Kleinb9d810b2019-07-01 12:38:02 -0600331 sysroot_path = os.path.join('/build', target)
332
333 # New handling - chroot & sysroot based.
334 # TODO(saklein) Switch this to the require decorator when legacy is removed.
335 if not sysroot_path:
336 cros_build_lib.Die('sysroot.path is required.')
337
Alex Klein231d2da2019-07-22 16:44:45 -0600338 # TODO(saklein): Switch to the validation_complete decorator when legacy
339 # handling is removed.
340 if config.validate_only:
341 return controller.RETURN_CODE_VALID_INPUT
342
Alex Kleinb9d810b2019-07-01 12:38:02 -0600343 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein231d2da2019-07-22 16:44:45 -0600344 if not sysroot.Exists(chroot=chroot):
Alex Kleinb9d810b2019-07-01 12:38:02 -0600345 cros_build_lib.Die('Sysroot must exist.')
346
347 archive = artifacts.BundleTastFiles(chroot, sysroot, output_dir)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600348
LaMont Jonesb9793cd2020-06-11 08:14:46 -0600349 if archive:
350 output_proto.artifacts.add().path = archive
351 else:
352 logging.warning('Found no tast files for %s.', target)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600353
354
Fergus Dall34d74e12020-09-29 15:52:32 +1000355def BundlePinnedGuestImages(_input_proto, _output_proto, _config):
356 # TODO(crbug/1034529): Remove this endpoint
357 pass
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700358
Fergus Dall34d74e12020-09-29 15:52:32 +1000359def FetchPinnedGuestImageUris(_input_proto, _output_proto, _config):
360 # TODO(crbug/1034529): Remove this endpoint
361 pass
Alex Klein7bf0ecb2019-06-25 10:04:15 -0600362
363
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700364def _BundleFirmwareResponse(input_proto, output_proto, _config):
365 """Add test firmware image files to a successful response."""
366 output_proto.artifacts.add().path = os.path.join(
367 input_proto.output_dir, 'firmware.tar.gz')
368
369
370@faux.success(_BundleFirmwareResponse)
371@faux.empty_error
Michael Mortensen38675192019-06-28 16:52:55 +0000372@validate.require('output_dir', 'sysroot.path')
Alex Klein231d2da2019-07-22 16:44:45 -0600373@validate.exists('output_dir')
374@validate.validation_complete
375def BundleFirmware(input_proto, output_proto, _config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600376 """Tar the firmware images for a build target.
377
378 Args:
379 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600380 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600381 _config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600382 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600383 output_dir = input_proto.output_dir
Michael Mortensen38675192019-06-28 16:52:55 +0000384 chroot = controller_util.ParseChroot(input_proto.chroot)
385 sysroot_path = input_proto.sysroot.path
386 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein231d2da2019-07-22 16:44:45 -0600387
388 if not chroot.exists():
389 cros_build_lib.Die('Chroot does not exist: %s', chroot.path)
390 elif not sysroot.Exists(chroot=chroot):
391 cros_build_lib.Die('Sysroot does not exist: %s',
392 chroot.full_path(sysroot.path))
393
Michael Mortensen38675192019-06-28 16:52:55 +0000394 archive = artifacts.BuildFirmwareArchive(chroot, sysroot, output_dir)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600395
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600396 if archive is None:
397 cros_build_lib.Die(
Michael Mortensen38675192019-06-28 16:52:55 +0000398 'Could not create firmware archive. No firmware found for %s.',
399 sysroot_path)
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600400
Alex Klein231d2da2019-07-22 16:44:45 -0600401 output_proto.artifacts.add().path = archive
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600402
403
Yicheng Liea1181f2020-09-22 11:51:10 -0700404def _BundleFpmcuUnittestsResponse(input_proto, output_proto, _config):
405 """Add fingerprint MCU unittest binaries to a successful response."""
406 output_proto.artifacts.add().path = os.path.join(
407 input_proto.output_dir, 'fpmcu_unittests.tar.gz')
408
409
410@faux.success(_BundleFpmcuUnittestsResponse)
411@faux.empty_error
412@validate.require('output_dir', 'sysroot.path')
413@validate.exists('output_dir')
414@validate.validation_complete
415def BundleFpmcuUnittests(input_proto, output_proto, _config):
416 """Tar the fingerprint MCU unittest binaries for a build target.
417
418 Args:
419 input_proto (BundleRequest): The input proto.
420 output_proto (BundleResponse): The output proto.
421 _config (api_config.ApiConfig): The API call config.
422 """
423 output_dir = input_proto.output_dir
424 chroot = controller_util.ParseChroot(input_proto.chroot)
425 sysroot_path = input_proto.sysroot.path
426 sysroot = sysroot_lib.Sysroot(sysroot_path)
427
428 if not chroot.exists():
429 cros_build_lib.Die('Chroot does not exist: %s', chroot.path)
430 elif not sysroot.Exists(chroot=chroot):
431 cros_build_lib.Die('Sysroot does not exist: %s',
432 chroot.full_path(sysroot.path))
433
434 archive = artifacts.BundleFpmcuUnittests(chroot, sysroot, output_dir)
435
436 if archive is None:
437 logging.warning(
438 'No fpmcu unittests found for %s.', sysroot_path)
439 return
440
441 output_proto.artifacts.add().path = archive
442
443
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700444def _BundleEbuildLogsResponse(input_proto, output_proto, _config):
445 """Add test log files to a successful response."""
446 output_proto.artifacts.add().path = os.path.join(
447 input_proto.output_dir, 'ebuild-logs.tar.gz')
448
449
450@faux.success(_BundleEbuildLogsResponse)
451@faux.empty_error
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600452@validate.exists('output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600453def BundleEbuildLogs(input_proto, output_proto, config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600454 """Tar the ebuild logs for a build target.
455
456 Args:
457 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600458 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600459 config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600460 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600461 output_dir = input_proto.output_dir
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600462 sysroot_path = input_proto.sysroot.path
463 chroot = controller_util.ParseChroot(input_proto.chroot)
Evan Hernandeza478d802019-04-08 15:08:24 -0600464
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600465 # TODO(mmortensen) Cleanup legacy handling after it has been switched over.
466 target = input_proto.build_target.name
467 if target:
468 # Legacy handling.
469 build_root = constants.SOURCE_ROOT
Alex Klein171da612019-08-06 14:00:42 -0600470 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600471 sysroot_path = os.path.join('/build', target)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600472
Alex Klein231d2da2019-07-22 16:44:45 -0600473 # TODO(saklein): Switch to validation_complete decorator after legacy
474 # handling has been cleaned up.
475 if config.validate_only:
476 return controller.RETURN_CODE_VALID_INPUT
477
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600478 sysroot = sysroot_lib.Sysroot(sysroot_path)
479 archive = artifacts.BundleEBuildLogsTarball(chroot, sysroot, output_dir)
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600480 if archive is None:
481 cros_build_lib.Die(
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600482 'Could not create ebuild logs archive. No logs found for %s.',
483 sysroot.path)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600484 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
Alex Klein6504eca2019-04-18 15:37:56 -0600485
486
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700487def _BundleChromeOSConfigResponse(input_proto, output_proto, _config):
488 """Add test config files to a successful response."""
489 output_proto.artifacts.add().path = os.path.join(
490 input_proto.output_dir, 'config.yaml')
491
492
493@faux.success(_BundleChromeOSConfigResponse)
494@faux.empty_error
Andrew Lamb811aead2019-08-12 10:25:05 -0600495@validate.exists('output_dir')
496@validate.validation_complete
497def BundleChromeOSConfig(input_proto, output_proto, _config):
498 """Output the ChromeOS Config payload for a build target.
499
500 Args:
501 input_proto (BundleRequest): The input proto.
502 output_proto (BundleResponse): The output proto.
503 _config (api_config.ApiConfig): The API call config.
504 """
505 output_dir = input_proto.output_dir
Andrew Lamb67bd68f2019-08-15 09:09:15 -0600506 sysroot_path = input_proto.sysroot.path
Andrew Lamb811aead2019-08-12 10:25:05 -0600507 chroot = controller_util.ParseChroot(input_proto.chroot)
508
Andrew Lamb67bd68f2019-08-15 09:09:15 -0600509 # TODO(mmortensen) Cleanup legacy handling after it has been switched over.
510 target = input_proto.build_target.name
511 if target:
512 # Legacy handling.
513 build_root = constants.SOURCE_ROOT
514 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
515 sysroot_path = os.path.join('/build', target)
516
517 sysroot = sysroot_lib.Sysroot(sysroot_path)
Andrew Lamb811aead2019-08-12 10:25:05 -0600518 chromeos_config = artifacts.BundleChromeOSConfig(chroot, sysroot, output_dir)
519 if chromeos_config is None:
520 cros_build_lib.Die(
521 'Could not create ChromeOS Config payload. No config found for %s.',
522 sysroot.path)
523 output_proto.artifacts.add().path = os.path.join(output_dir, chromeos_config)
524
525
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700526def _BundleSimpleChromeArtifactsResponse(input_proto, output_proto, _config):
527 """Add test simple chrome files to a successful response."""
528 output_proto.artifacts.add().path = os.path.join(
529 input_proto.output_dir, 'simple_chrome.txt')
530
531
532@faux.success(_BundleSimpleChromeArtifactsResponse)
533@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600534@validate.require('output_dir', 'sysroot.build_target.name', 'sysroot.path')
535@validate.exists('output_dir')
536@validate.validation_complete
537def BundleSimpleChromeArtifacts(input_proto, output_proto, _config):
Alex Klein2275d692019-04-23 16:04:12 -0600538 """Create the simple chrome artifacts."""
Alex Klein2275d692019-04-23 16:04:12 -0600539 sysroot_path = input_proto.sysroot.path
Alex Klein2275d692019-04-23 16:04:12 -0600540 output_dir = input_proto.output_dir
541
Alex Klein2275d692019-04-23 16:04:12 -0600542 # Build out the argument instances.
Alex Klein26e472b2020-03-10 14:35:01 -0600543 build_target = controller_util.ParseBuildTarget(
544 input_proto.sysroot.build_target)
545 chroot = controller_util.ParseChroot(input_proto.chroot)
Alex Klein2275d692019-04-23 16:04:12 -0600546 # Sysroot.path needs to be the fully qualified path, including the chroot.
547 full_sysroot_path = os.path.join(chroot.path, sysroot_path.lstrip(os.sep))
548 sysroot = sysroot_lib.Sysroot(full_sysroot_path)
549
550 # Quick sanity check that the sysroot exists before we go on.
551 if not sysroot.Exists():
552 cros_build_lib.Die('The sysroot does not exist.')
553
554 try:
555 results = artifacts.BundleSimpleChromeArtifacts(chroot, sysroot,
556 build_target, output_dir)
557 except artifacts.Error as e:
558 cros_build_lib.Die('Error %s raised in BundleSimpleChromeArtifacts: %s',
559 type(e), e)
560
561 for file_name in results:
562 output_proto.artifacts.add().path = file_name
563
564
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700565def _BundleVmFilesResponse(input_proto, output_proto, _config):
566 """Add test vm files to a successful response."""
567 output_proto.artifacts.add().path = os.path.join(
568 input_proto.output_dir, 'f1.tar')
569
570
571@faux.success(_BundleVmFilesResponse)
572@faux.empty_error
Michael Mortensen51f06722019-07-18 09:55:50 -0600573@validate.require('chroot.path', 'test_results_dir', 'output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600574@validate.exists('output_dir')
575@validate.validation_complete
576def BundleVmFiles(input_proto, output_proto, _config):
Alex Klein6504eca2019-04-18 15:37:56 -0600577 """Tar VM disk and memory files.
578
579 Args:
Trent Begin008cade2019-10-31 13:40:59 -0600580 input_proto (BundleVmFilesRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600581 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600582 _config (api_config.ApiConfig): The API call config.
Alex Klein6504eca2019-04-18 15:37:56 -0600583 """
Michael Mortensen51f06722019-07-18 09:55:50 -0600584 chroot = controller_util.ParseChroot(input_proto.chroot)
585 test_results_dir = input_proto.test_results_dir
Alex Klein6504eca2019-04-18 15:37:56 -0600586 output_dir = input_proto.output_dir
587
Michael Mortensen51f06722019-07-18 09:55:50 -0600588 archives = artifacts.BundleVmFiles(
589 chroot, test_results_dir, output_dir)
Alex Klein6504eca2019-04-18 15:37:56 -0600590 for archive in archives:
591 output_proto.artifacts.add().path = archive
Tiancong Wangc4805b72019-06-11 12:12:03 -0700592
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700593def _BundleAFDOGenerationArtifactsResponse(input_proto, output_proto, _config):
594 """Add test tarball AFDO file to a successful response."""
595 output_proto.artifacts.add().path = os.path.join(
596 input_proto.output_dir, 'artifact1')
597
Alex Klein231d2da2019-07-22 16:44:45 -0600598
Tiancong Wang24a3df72019-08-20 15:48:51 -0700599_VALID_ARTIFACT_TYPES = [toolchain_pb2.BENCHMARK_AFDO,
600 toolchain_pb2.ORDERFILE]
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700601@faux.success(_BundleAFDOGenerationArtifactsResponse)
602@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600603@validate.require('build_target.name', 'output_dir')
Tiancong Wang50b80a92019-08-01 14:46:15 -0700604@validate.is_in('artifact_type', _VALID_ARTIFACT_TYPES)
Alex Klein231d2da2019-07-22 16:44:45 -0600605@validate.exists('output_dir')
Tiancong Wang2ade7932019-09-27 14:15:40 -0700606@validate.exists('chroot.chrome_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600607@validate.validation_complete
Tiancong Wang50b80a92019-08-01 14:46:15 -0700608def BundleAFDOGenerationArtifacts(input_proto, output_proto, _config):
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700609 """Generic function for creating tarballs of both AFDO and orderfile.
Tiancong Wangc4805b72019-06-11 12:12:03 -0700610
611 Args:
Tiancong Wang50b80a92019-08-01 14:46:15 -0700612 input_proto (BundleChromeAFDORequest): The input proto.
Tiancong Wangc4805b72019-06-11 12:12:03 -0700613 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600614 _config (api_config.ApiConfig): The API call config.
Tiancong Wangc4805b72019-06-11 12:12:03 -0700615 """
Tiancong Wang2ade7932019-09-27 14:15:40 -0700616 chrome_root = input_proto.chroot.chrome_dir
Tiancong Wangc4805b72019-06-11 12:12:03 -0700617 output_dir = input_proto.output_dir
Tiancong Wang50b80a92019-08-01 14:46:15 -0700618 artifact_type = input_proto.artifact_type
Tiancong Wangc4805b72019-06-11 12:12:03 -0700619
Alex Klein26e472b2020-03-10 14:35:01 -0600620 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
Tiancong Wangc4805b72019-06-11 12:12:03 -0700621 chroot = controller_util.ParseChroot(input_proto.chroot)
622
623 try:
Tiancong Wang24a3df72019-08-20 15:48:51 -0700624 is_orderfile = bool(artifact_type is toolchain_pb2.ORDERFILE)
Tiancong Wang50b80a92019-08-01 14:46:15 -0700625 results = artifacts.BundleAFDOGenerationArtifacts(
Tiancong Wang2ade7932019-09-27 14:15:40 -0700626 is_orderfile, chroot, chrome_root,
Tiancong Wang50b80a92019-08-01 14:46:15 -0700627 build_target, output_dir)
Tiancong Wangc4805b72019-06-11 12:12:03 -0700628 except artifacts.Error as e:
629 cros_build_lib.Die('Error %s raised in BundleSimpleChromeArtifacts: %s',
630 type(e), e)
631
632 for file_name in results:
633 output_proto.artifacts.add().path = file_name
Alex Klein0b1cbfc2019-08-14 10:09:58 -0600634
635
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700636def _ExportCpeReportResponse(input_proto, output_proto, _config):
637 """Add test cpe results to a successful response."""
638 output_proto.artifacts.add().path = os.path.join(
639 input_proto.output_dir, 'cpe_report.txt')
640 output_proto.artifacts.add().path = os.path.join(
641 input_proto.output_dir, 'cpe_warnings.txt')
642
643
644@faux.success(_ExportCpeReportResponse)
645@faux.empty_error
Alex Klein0b1cbfc2019-08-14 10:09:58 -0600646@validate.exists('output_dir')
647def ExportCpeReport(input_proto, output_proto, config):
648 """Export a CPE report.
649
650 Args:
651 input_proto (BundleRequest): The input proto.
652 output_proto (BundleResponse): The output proto.
653 config (api_config.ApiConfig): The API call config.
654 """
655 chroot = controller_util.ParseChroot(input_proto.chroot)
656 output_dir = input_proto.output_dir
657
658 if input_proto.build_target.name:
659 # Legacy handling - use the default sysroot path for the build target.
660 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
661 sysroot = sysroot_lib.Sysroot(build_target.root)
662 elif input_proto.sysroot.path:
663 sysroot = sysroot_lib.Sysroot(input_proto.sysroot.path)
664 else:
665 # TODO(saklein): Switch to validate decorators once legacy handling can be
666 # cleaned up.
667 cros_build_lib.Die('sysroot.path is required.')
668
669 if config.validate_only:
670 return controller.RETURN_CODE_VALID_INPUT
671
672 cpe_result = artifacts.GenerateCpeReport(chroot, sysroot, output_dir)
673
674 output_proto.artifacts.add().path = cpe_result.report
675 output_proto.artifacts.add().path = cpe_result.warnings
Shao-Chuan Leea44dddc2020-10-30 17:16:55 +0900676
677
678def _BundleGceTarballResponse(input_proto, output_proto, _config):
679 """Add artifact tarball to a successful response."""
680 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
681 constants.TEST_IMAGE_GCE_TAR)
682
683
684@faux.success(_BundleGceTarballResponse)
685@faux.empty_error
686@validate.require('build_target.name', 'output_dir')
687@validate.exists('output_dir')
688@validate.validation_complete
689def BundleGceTarball(input_proto, output_proto, _config):
690 """Bundle the test image into a tarball suitable for importing into GCE.
691
692 Args:
693 input_proto (BundleRequest): The input proto.
694 output_proto (BundleResponse): The output proto.
695 _config (api_config.ApiConfig): The API call config.
696 """
697 target = input_proto.build_target.name
698 output_dir = input_proto.output_dir
699 image_dir = _GetImageDir(constants.SOURCE_ROOT, target)
700 if image_dir is None:
701 return None
702
703 tarball = artifacts.BundleGceTarball(output_dir, image_dir)
704 output_proto.artifacts.add().path = tarball