blob: 5586feaf5654853cfb50480626ca13e24aa018c3 [file] [log] [blame]
Evan Hernandezf388cbf2019-04-01 11:15:23 -06001# Copyright 2019 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Implements ArtifactService."""
6
Chris McDonald1672ddb2021-07-21 11:48:23 -06007import logging
Evan Hernandezf388cbf2019-04-01 11:15:23 -06008import os
Varun Somani04dccd72021-10-09 01:06:11 +00009from typing import Any, NamedTuple, Optional, TYPE_CHECKING
Evan Hernandezf388cbf2019-04-01 11:15:23 -060010
Alex Klein231d2da2019-07-22 16:44:45 -060011from chromite.api import controller
Alex Klein076841b2019-08-29 15:19:39 -060012from chromite.api import faux
Alex Klein2b236722019-06-19 15:44:26 -060013from chromite.api import validate
Alex Klein238d8862019-05-07 11:32:46 -060014from chromite.api.controller import controller_util
George Engelbrechtc9a8e812021-06-16 18:14:17 -060015from chromite.api.controller import image as image_controller
16from chromite.api.controller import sysroot as sysroot_controller
David Wellingc1433c22021-06-25 16:29:48 +000017from chromite.api.controller import test as test_controller
LaMont Jones58362a42021-02-04 17:40:08 -070018from chromite.api.gen.chromite.api import artifacts_pb2
Tiancong Wang24a3df72019-08-20 15:48:51 -070019from chromite.api.gen.chromite.api import toolchain_pb2
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000020from chromite.api.gen.chromiumos import common_pb2
Alex Klein2275d692019-04-23 16:04:12 -060021from chromite.lib import chroot_lib
Evan Hernandezf388cbf2019-04-01 11:15:23 -060022from chromite.lib import constants
23from chromite.lib import cros_build_lib
Alex Klein2275d692019-04-23 16:04:12 -060024from chromite.lib import sysroot_lib
25from chromite.service import artifacts
Greg Edelstondc941072021-08-11 12:32:30 -060026from chromite.service import test
Evan Hernandezf388cbf2019-04-01 11:15:23 -060027
Varun Somani04dccd72021-10-09 01:06:11 +000028if TYPE_CHECKING:
29 from chromite.api import api_config
Evan Hernandezf388cbf2019-04-01 11:15:23 -060030
George Engelbrechtc9a8e812021-06-16 18:14:17 -060031class RegisteredGet(NamedTuple):
32 """An registered function for calling Get on an artifact type."""
33 output_proto: artifacts_pb2.GetResponse
34 artifact_dict: Any
LaMont Jones0f5171b2021-01-29 12:28:56 -070035
36
George Engelbrechtc9a8e812021-06-16 18:14:17 -060037def ExampleGetResponse(_input_proto, _output_proto, _config):
38 """Give an example GetResponse with a minimal coverage set."""
39 _output_proto = artifacts_pb2.GetResponse(
40 artifacts=common_pb2.UploadedArtifactsByService(
41 image=image_controller.ExampleGetResponse(),
42 sysroot=sysroot_controller.ExampleGetResponse(),
43 ))
44 return controller.RETURN_CODE_SUCCESS
45
46
LaMont Jones0f5171b2021-01-29 12:28:56 -070047@faux.empty_error
George Engelbrechtc9a8e812021-06-16 18:14:17 -060048@faux.success(ExampleGetResponse)
LaMont Jones0f5171b2021-01-29 12:28:56 -070049@validate.exists('result_path.path.path')
50@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +000051def Get(
52 input_proto: artifacts_pb2.GetRequest,
53 output_proto: artifacts_pb2.GetResponse,
54 _config: 'api_config.ApiConfig'):
LaMont Jones0f5171b2021-01-29 12:28:56 -070055 """Get all artifacts.
56
57 Get all artifacts for the build.
58
George Engelbrechtc9a8e812021-06-16 18:14:17 -060059 Note: As the individual artifact_type bundlers are added here, they *must*
60 stop uploading it via the individual bundler function.
LaMont Jones0f5171b2021-01-29 12:28:56 -070061
62 Args:
Varun Somani04dccd72021-10-09 01:06:11 +000063 input_proto: The input proto.
64 output_proto: The output proto.
65 _config: The API call config.
LaMont Jones0f5171b2021-01-29 12:28:56 -070066 """
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000067 output_dir = input_proto.result_path.path.path
68
George Engelbrechtc9a8e812021-06-16 18:14:17 -060069 sysroot = controller_util.ParseSysroot(input_proto.sysroot)
LaMont Jones8b88e9d2021-06-28 16:37:34 -060070 # This endpoint does not currently support any artifacts that are built
71 # without a sysroot being present.
72 if not sysroot.path:
73 return controller.RETURN_CODE_SUCCESS
74
George Engelbrechtc9a8e812021-06-16 18:14:17 -060075 chroot = controller_util.ParseChroot(input_proto.chroot)
76 build_target = controller_util.ParseBuildTarget(
77 input_proto.sysroot.build_target)
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000078
George Engelbrechtc9a8e812021-06-16 18:14:17 -060079 # A list of RegisteredGet tuples (input proto, output proto, get results).
80 get_res_list = [
81 RegisteredGet(
82 output_proto.artifacts.image,
83 image_controller.GetArtifacts(
84 input_proto.artifact_info.image, chroot, sysroot, build_target,
85 output_dir)),
86 RegisteredGet(
87 output_proto.artifacts.sysroot,
88 sysroot_controller.GetArtifacts(
89 input_proto.artifact_info.sysroot, chroot, sysroot, build_target,
David Wellingc1433c22021-06-25 16:29:48 +000090 output_dir)),
91 RegisteredGet(
92 output_proto.artifacts.test,
93 test_controller.GetArtifacts(
Jack Neusc9707c32021-07-23 21:48:54 +000094 input_proto.artifact_info.test, chroot, sysroot, build_target,
95 output_dir)),
George Engelbrechtc9a8e812021-06-16 18:14:17 -060096 ]
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000097
George Engelbrechtc9a8e812021-06-16 18:14:17 -060098 for get_res in get_res_list:
99 for artifact_dict in get_res.artifact_dict:
100 get_res.output_proto.artifacts.add(
101 artifact_type=artifact_dict['type'],
102 paths=[
103 common_pb2.Path(
104 path=x, location=common_pb2.Path.Location.OUTSIDE)
105 for x in artifact_dict['paths']
106 ])
LaMont Jones0f5171b2021-01-29 12:28:56 -0700107 return controller.RETURN_CODE_SUCCESS
108
109
LaMont Jones58362a42021-02-04 17:40:08 -0700110def _BuildSetupResponse(_input_proto, output_proto, _config):
111 """Just return POINTLESS for now."""
112 # All of the artifact types we support claim that the build is POINTLESS.
113 output_proto.build_relevance = artifacts_pb2.BuildSetupResponse.POINTLESS
114
115
116@faux.success(_BuildSetupResponse)
117@faux.empty_error
118@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +0000119def BuildSetup(
120 _input_proto: artifacts_pb2.GetRequest,
121 output_proto: artifacts_pb2.GetResponse,
122 _config: 'api_config.ApiConfig'):
123
LaMont Jones58362a42021-02-04 17:40:08 -0700124 """Setup anything needed for building artifacts
125
126 If any artifact types require steps prior to building the package, they go
127 here. For example, see ToolchainService/PrepareForBuild.
128
129 Note: crbug/1034529 introduces this method as a noop. As the individual
130 artifact_type bundlers are added here, they *must* stop uploading it via the
131 individual bundler function.
132
133 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000134 _input_proto: The input proto.
135 output_proto: The output proto.
136 _config: The API call config.
LaMont Jones58362a42021-02-04 17:40:08 -0700137 """
138 # If any artifact_type says "NEEDED", the return is NEEDED.
139 # Otherwise, if any artifact_type says "UNKNOWN", the return is UNKNOWN.
140 # Otherwise, the return is POINTLESS.
141 output_proto.build_relevance = artifacts_pb2.BuildSetupResponse.POINTLESS
142 return controller.RETURN_CODE_SUCCESS
143
144
Varun Somani04dccd72021-10-09 01:06:11 +0000145def _GetImageDir(build_root: str, target: str) -> Optional[str]:
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600146 """Return path containing images for the given build target.
147
Alex Kleine2612a02019-04-18 13:51:06 -0600148 TODO(saklein) Expand image_lib.GetLatestImageLink to support this use case.
149
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600150 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000151 build_root: Path to checkout where build occurs.
152 target: Name of the build target.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600153
154 Returns:
Alex Kleind2bf1462019-10-24 16:37:04 -0600155 Path to the latest directory containing target images or None.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600156 """
157 image_dir = os.path.join(build_root, 'src/build/images', target, 'latest')
158 if not os.path.exists(image_dir):
Alex Kleind2bf1462019-10-24 16:37:04 -0600159 logging.warning('Expected to find image output for target %s at %s, but '
160 'path does not exist', target, image_dir)
161 return None
162
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600163 return image_dir
164
165
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700166def _BundleImageArchivesResponse(input_proto, output_proto, _config):
167 """Add artifact paths to a successful response."""
168 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
169 'path0.tar.xz')
170 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
171 'path1.tar.xz')
172
173
174@faux.success(_BundleImageArchivesResponse)
175@faux.empty_error
Alex Kleind91e95a2019-09-17 10:39:02 -0600176@validate.require('build_target.name')
177@validate.exists('output_dir')
178@validate.validation_complete
179def BundleImageArchives(input_proto, output_proto, _config):
180 """Create a .tar.xz archive for each image that has been created."""
181 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
182 output_dir = input_proto.output_dir
183 image_dir = _GetImageDir(constants.SOURCE_ROOT, build_target.name)
Alex Kleind2bf1462019-10-24 16:37:04 -0600184 if image_dir is None:
185 return
Alex Kleind91e95a2019-09-17 10:39:02 -0600186
187 archives = artifacts.ArchiveImages(image_dir, output_dir)
188
189 for archive in archives:
190 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
191
192
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700193def _BundleImageZipResponse(input_proto, output_proto, _config):
194 """Add artifact zip files to a successful response."""
195 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
196 'image.zip')
197
198
199@faux.success(_BundleImageZipResponse)
200@faux.empty_error
Michael Mortensen01910922019-07-24 14:48:10 -0600201@validate.require('build_target.name', 'output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600202@validate.exists('output_dir')
203@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +0000204def BundleImageZip(
205 input_proto: artifacts_pb2.BundleRequest,
206 output_proto: artifacts_pb2.BundleResponse,
207 _config: 'api_config.ApiConfig'):
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600208 """Bundle image.zip.
209
210 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000211 input_proto: The input proto.
212 output_proto: The output proto.
213 _config: The API call config.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600214 """
215 target = input_proto.build_target.name
216 output_dir = input_proto.output_dir
217 image_dir = _GetImageDir(constants.SOURCE_ROOT, target)
Alex Kleind2bf1462019-10-24 16:37:04 -0600218 if image_dir is None:
219 return None
Alex Klein231d2da2019-07-22 16:44:45 -0600220
Michael Mortensen01910922019-07-24 14:48:10 -0600221 archive = artifacts.BundleImageZip(output_dir, image_dir)
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600222 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
223
224
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700225def _BundleTestUpdatePayloadsResponse(input_proto, output_proto, _config):
226 """Add test payload files to a successful response."""
227 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
228 'payload1.bin')
229
230
231@faux.success(_BundleTestUpdatePayloadsResponse)
232@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600233@validate.require('build_target.name', 'output_dir')
234@validate.exists('output_dir')
235@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +0000236def BundleTestUpdatePayloads(
237 input_proto: artifacts_pb2.BundleRequest,
238 output_proto: artifacts_pb2.BundleResponse,
239 _config: 'api_config.ApiConfig'):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600240 """Generate minimal update payloads for the build target for testing.
241
242 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000243 input_proto: The input proto.
244 output_proto: The output proto.
245 _config: The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600246 """
247 target = input_proto.build_target.name
248 output_dir = input_proto.output_dir
249 build_root = constants.SOURCE_ROOT
250
251 # Use the first available image to create the update payload.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600252 img_dir = _GetImageDir(build_root, target)
Alex Kleind2bf1462019-10-24 16:37:04 -0600253 if img_dir is None:
254 return None
255
Alex Kleincb541e82019-06-26 15:06:11 -0600256 img_types = [constants.IMAGE_TYPE_TEST, constants.IMAGE_TYPE_DEV,
257 constants.IMAGE_TYPE_BASE]
258 img_names = [constants.IMAGE_TYPE_TO_NAME[t] for t in img_types]
Mike Frysinger66ce4132019-07-17 22:52:52 -0400259 img_paths = [os.path.join(img_dir, x) for x in img_names]
Mike Frysingera552be42018-08-17 14:39:32 -0400260 valid_images = [x for x in img_paths if os.path.exists(x)]
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600261
Alex Kleincb541e82019-06-26 15:06:11 -0600262 if not valid_images:
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600263 cros_build_lib.Die(
264 'Expected to find an image of type among %r for target "%s" '
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600265 'at path %s.', img_types, target, img_dir)
Alex Kleincb541e82019-06-26 15:06:11 -0600266 image = valid_images[0]
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600267
Alex Kleincb541e82019-06-26 15:06:11 -0600268 payloads = artifacts.BundleTestUpdatePayloads(image, output_dir)
269 for payload in payloads:
270 output_proto.artifacts.add().path = payload
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600271
272
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700273def _BundleAutotestFilesResponse(input_proto, output_proto, _config):
274 """Add test autotest files to a successful response."""
275 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
276 'autotest-a.tar.gz')
277
278
279@faux.success(_BundleAutotestFilesResponse)
280@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600281@validate.require('output_dir')
282@validate.exists('output_dir')
Varun Somani04dccd72021-10-09 01:06:11 +0000283def BundleAutotestFiles(
284 input_proto: artifacts_pb2.BundleRequest,
285 output_proto: artifacts_pb2.BundleResponse,
286 config: 'api_config.ApiConfig'):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600287 """Tar the autotest files for a build target.
288
289 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000290 input_proto: The input proto.
291 output_proto: The output proto.
292 config: The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600293 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600294 output_dir = input_proto.output_dir
Alex Klein238d8862019-05-07 11:32:46 -0600295 target = input_proto.build_target.name
Alex Kleine21a0952019-08-23 16:08:16 -0600296 chroot = controller_util.ParseChroot(input_proto.chroot)
297
Alex Klein238d8862019-05-07 11:32:46 -0600298 if target:
Alex Kleine21a0952019-08-23 16:08:16 -0600299 sysroot_path = os.path.join('/build', target)
Alex Klein238d8862019-05-07 11:32:46 -0600300 else:
301 # New style call, use chroot and sysroot.
Alex Klein238d8862019-05-07 11:32:46 -0600302 sysroot_path = input_proto.sysroot.path
303 if not sysroot_path:
304 cros_build_lib.Die('sysroot.path is required.')
305
Alex Kleine21a0952019-08-23 16:08:16 -0600306 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein238d8862019-05-07 11:32:46 -0600307
Alex Klein231d2da2019-07-22 16:44:45 -0600308 # TODO(saklein): Switch to the validate_only decorator when legacy handling
309 # is removed.
310 if config.validate_only:
311 return controller.RETURN_CODE_VALID_INPUT
312
Alex Kleine21a0952019-08-23 16:08:16 -0600313 if not sysroot.Exists(chroot=chroot):
Alex Klein238d8862019-05-07 11:32:46 -0600314 cros_build_lib.Die('Sysroot path must exist: %s', sysroot.path)
315
316 try:
317 # Note that this returns the full path to *multiple* tarballs.
Alex Kleine21a0952019-08-23 16:08:16 -0600318 archives = artifacts.BundleAutotestFiles(chroot, sysroot, output_dir)
Alex Klein238d8862019-05-07 11:32:46 -0600319 except artifacts.Error as e:
Alex Klein03890312020-06-30 09:59:50 -0600320 logging.warning(e)
321 return
Alex Klein238d8862019-05-07 11:32:46 -0600322
323 for archive in archives.values():
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600324 output_proto.artifacts.add().path = archive
325
326
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700327def _BundleTastFilesResponse(input_proto, output_proto, _config):
328 """Add test tast files to a successful response."""
329 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
330 'tast_bundles.tar.gz')
331
332
333@faux.success(_BundleTastFilesResponse)
334@faux.empty_error
Alex Kleinb9d810b2019-07-01 12:38:02 -0600335@validate.require('output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600336@validate.exists('output_dir')
Varun Somani04dccd72021-10-09 01:06:11 +0000337def BundleTastFiles(
338 input_proto: artifacts_pb2.BundleRequest,
339 output_proto: artifacts_pb2.BundleResponse,
340 config: 'api_config.ApiConfig'):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600341 """Tar the tast files for a build target.
342
343 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000344 input_proto: The input proto.
345 output_proto: The output proto.
346 config: The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600347 """
348 target = input_proto.build_target.name
349 output_dir = input_proto.output_dir
350 build_root = constants.SOURCE_ROOT
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600351
Alex Kleinb9d810b2019-07-01 12:38:02 -0600352 chroot = controller_util.ParseChroot(input_proto.chroot)
353 sysroot_path = input_proto.sysroot.path
354
355 # TODO(saklein) Cleanup legacy handling after it has been switched over.
356 if target:
357 # Legacy handling.
Alex Klein171da612019-08-06 14:00:42 -0600358 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
Alex Kleinb9d810b2019-07-01 12:38:02 -0600359 sysroot_path = os.path.join('/build', target)
360
361 # New handling - chroot & sysroot based.
362 # TODO(saklein) Switch this to the require decorator when legacy is removed.
363 if not sysroot_path:
364 cros_build_lib.Die('sysroot.path is required.')
365
Alex Klein231d2da2019-07-22 16:44:45 -0600366 # TODO(saklein): Switch to the validation_complete decorator when legacy
367 # handling is removed.
368 if config.validate_only:
369 return controller.RETURN_CODE_VALID_INPUT
370
Alex Kleinb9d810b2019-07-01 12:38:02 -0600371 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein231d2da2019-07-22 16:44:45 -0600372 if not sysroot.Exists(chroot=chroot):
Alex Kleinb9d810b2019-07-01 12:38:02 -0600373 cros_build_lib.Die('Sysroot must exist.')
374
375 archive = artifacts.BundleTastFiles(chroot, sysroot, output_dir)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600376
LaMont Jonesb9793cd2020-06-11 08:14:46 -0600377 if archive:
378 output_proto.artifacts.add().path = archive
379 else:
380 logging.warning('Found no tast files for %s.', target)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600381
382
Fergus Dall34d74e12020-09-29 15:52:32 +1000383def BundlePinnedGuestImages(_input_proto, _output_proto, _config):
384 # TODO(crbug/1034529): Remove this endpoint
385 pass
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700386
Fergus Dall34d74e12020-09-29 15:52:32 +1000387def FetchPinnedGuestImageUris(_input_proto, _output_proto, _config):
388 # TODO(crbug/1034529): Remove this endpoint
389 pass
Alex Klein7bf0ecb2019-06-25 10:04:15 -0600390
391
Greg Edelstondc941072021-08-11 12:32:30 -0600392def _FetchMetadataResponse(_input_proto, output_proto, _config):
393 """Populate the output_proto with sample data."""
394 for fp in ('/metadata/foo.txt', '/metadata/bar.jsonproto'):
395 output_proto.filepaths.add(path=common_pb2.Path(
396 path=fp, location=common_pb2.Path.OUTSIDE))
397 return controller.RETURN_CODE_SUCCESS
398
399
400@faux.success(_FetchMetadataResponse)
401@faux.empty_error
402@validate.exists('chroot.path')
403@validate.require('sysroot.path')
404@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +0000405def FetchMetadata(
406 input_proto: artifacts_pb2.FetchMetadataRequest,
407 output_proto: artifacts_pb2.FetchMetadataResponse,
408 _config: 'api_config.ApiConfig'):
Greg Edelstondc941072021-08-11 12:32:30 -0600409 """FetchMetadata returns the paths to all build/test metadata files.
410
411 This implements ArtifactsService.FetchMetadata.
412
413 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000414 input_proto: The input proto.
415 output_proto: The output proto.
416 config: The API call config.
Greg Edelstondc941072021-08-11 12:32:30 -0600417 """
418 chroot = controller_util.ParseChroot(input_proto.chroot)
419 sysroot = controller_util.ParseSysroot(input_proto.sysroot)
420 for path in test.FindAllMetadataFiles(chroot, sysroot):
421 output_proto.filepaths.add(
422 path=common_pb2.Path(path=path, location=common_pb2.Path.OUTSIDE))
423 return controller.RETURN_CODE_SUCCESS
424
425
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700426def _BundleFirmwareResponse(input_proto, output_proto, _config):
427 """Add test firmware image files to a successful response."""
428 output_proto.artifacts.add().path = os.path.join(
429 input_proto.output_dir, 'firmware.tar.gz')
430
431
432@faux.success(_BundleFirmwareResponse)
433@faux.empty_error
Michael Mortensen38675192019-06-28 16:52:55 +0000434@validate.require('output_dir', 'sysroot.path')
Alex Klein231d2da2019-07-22 16:44:45 -0600435@validate.exists('output_dir')
436@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +0000437def BundleFirmware(
438 input_proto: artifacts_pb2.BundleRequest,
439 output_proto: artifacts_pb2.BundleResponse,
440 _config: 'api_config.ApiConfig'):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600441 """Tar the firmware images for a build target.
442
443 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000444 input_proto: The input proto.
445 output_proto: The output proto.
446 _config: The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600447 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600448 output_dir = input_proto.output_dir
Michael Mortensen38675192019-06-28 16:52:55 +0000449 chroot = controller_util.ParseChroot(input_proto.chroot)
450 sysroot_path = input_proto.sysroot.path
451 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein231d2da2019-07-22 16:44:45 -0600452
453 if not chroot.exists():
454 cros_build_lib.Die('Chroot does not exist: %s', chroot.path)
455 elif not sysroot.Exists(chroot=chroot):
456 cros_build_lib.Die('Sysroot does not exist: %s',
457 chroot.full_path(sysroot.path))
458
Michael Mortensen38675192019-06-28 16:52:55 +0000459 archive = artifacts.BuildFirmwareArchive(chroot, sysroot, output_dir)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600460
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600461 if archive is None:
George Engelbrecht9e41e172021-11-18 17:04:22 -0700462 logging.warning(
Michael Mortensen38675192019-06-28 16:52:55 +0000463 'Could not create firmware archive. No firmware found for %s.',
464 sysroot_path)
George Engelbrecht9e41e172021-11-18 17:04:22 -0700465 return
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600466
Alex Klein231d2da2019-07-22 16:44:45 -0600467 output_proto.artifacts.add().path = archive
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600468
469
Yicheng Liea1181f2020-09-22 11:51:10 -0700470def _BundleFpmcuUnittestsResponse(input_proto, output_proto, _config):
471 """Add fingerprint MCU unittest binaries to a successful response."""
472 output_proto.artifacts.add().path = os.path.join(
473 input_proto.output_dir, 'fpmcu_unittests.tar.gz')
474
475
476@faux.success(_BundleFpmcuUnittestsResponse)
477@faux.empty_error
478@validate.require('output_dir', 'sysroot.path')
479@validate.exists('output_dir')
480@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +0000481def BundleFpmcuUnittests(
482 input_proto: artifacts_pb2.BundleRequest,
483 output_proto: artifacts_pb2.BundleResponse,
484 _config: 'api_config.ApiConfig'):
Yicheng Liea1181f2020-09-22 11:51:10 -0700485 """Tar the fingerprint MCU unittest binaries for a build target.
486
487 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000488 input_proto: The input proto.
489 output_proto: The output proto.
490 _config: The API call config.
Yicheng Liea1181f2020-09-22 11:51:10 -0700491 """
492 output_dir = input_proto.output_dir
493 chroot = controller_util.ParseChroot(input_proto.chroot)
494 sysroot_path = input_proto.sysroot.path
495 sysroot = sysroot_lib.Sysroot(sysroot_path)
496
497 if not chroot.exists():
498 cros_build_lib.Die('Chroot does not exist: %s', chroot.path)
499 elif not sysroot.Exists(chroot=chroot):
500 cros_build_lib.Die('Sysroot does not exist: %s',
501 chroot.full_path(sysroot.path))
502
503 archive = artifacts.BundleFpmcuUnittests(chroot, sysroot, output_dir)
504
505 if archive is None:
506 logging.warning(
507 'No fpmcu unittests found for %s.', sysroot_path)
508 return
509
510 output_proto.artifacts.add().path = archive
511
512
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700513def _BundleEbuildLogsResponse(input_proto, output_proto, _config):
514 """Add test log files to a successful response."""
515 output_proto.artifacts.add().path = os.path.join(
516 input_proto.output_dir, 'ebuild-logs.tar.gz')
517
518
519@faux.success(_BundleEbuildLogsResponse)
520@faux.empty_error
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600521@validate.exists('output_dir')
Varun Somani04dccd72021-10-09 01:06:11 +0000522def BundleEbuildLogs(
523 input_proto: artifacts_pb2.BundleRequest,
524 output_proto: artifacts_pb2.BundleResponse,
525 config: 'api_config.ApiConfig'):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600526 """Tar the ebuild logs for a build target.
527
528 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000529 input_proto: The input proto.
530 output_proto: The output proto.
531 config: The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600532 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600533 output_dir = input_proto.output_dir
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600534 sysroot_path = input_proto.sysroot.path
535 chroot = controller_util.ParseChroot(input_proto.chroot)
Evan Hernandeza478d802019-04-08 15:08:24 -0600536
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600537 # TODO(mmortensen) Cleanup legacy handling after it has been switched over.
538 target = input_proto.build_target.name
539 if target:
540 # Legacy handling.
541 build_root = constants.SOURCE_ROOT
Alex Klein171da612019-08-06 14:00:42 -0600542 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600543 sysroot_path = os.path.join('/build', target)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600544
Alex Klein231d2da2019-07-22 16:44:45 -0600545 # TODO(saklein): Switch to validation_complete decorator after legacy
546 # handling has been cleaned up.
547 if config.validate_only:
548 return controller.RETURN_CODE_VALID_INPUT
549
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600550 sysroot = sysroot_lib.Sysroot(sysroot_path)
551 archive = artifacts.BundleEBuildLogsTarball(chroot, sysroot, output_dir)
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600552 if archive is None:
553 cros_build_lib.Die(
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600554 'Could not create ebuild logs archive. No logs found for %s.',
555 sysroot.path)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600556 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
Alex Klein6504eca2019-04-18 15:37:56 -0600557
558
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700559def _BundleChromeOSConfigResponse(input_proto, output_proto, _config):
560 """Add test config files to a successful response."""
561 output_proto.artifacts.add().path = os.path.join(
562 input_proto.output_dir, 'config.yaml')
563
564
565@faux.success(_BundleChromeOSConfigResponse)
566@faux.empty_error
Andrew Lamb811aead2019-08-12 10:25:05 -0600567@validate.exists('output_dir')
568@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +0000569def BundleChromeOSConfig(
570 input_proto: artifacts_pb2.BundleRequest,
571 output_proto: artifacts_pb2.BundleResponse,
572 _config: 'api_config.ApiConfig'):
Andrew Lamb811aead2019-08-12 10:25:05 -0600573 """Output the ChromeOS Config payload for a build target.
574
575 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000576 input_proto: The input proto.
577 output_proto: The output proto.
578 _config: The API call config.
Andrew Lamb811aead2019-08-12 10:25:05 -0600579 """
580 output_dir = input_proto.output_dir
Andrew Lamb67bd68f2019-08-15 09:09:15 -0600581 sysroot_path = input_proto.sysroot.path
Andrew Lamb811aead2019-08-12 10:25:05 -0600582 chroot = controller_util.ParseChroot(input_proto.chroot)
583
Andrew Lamb67bd68f2019-08-15 09:09:15 -0600584 # TODO(mmortensen) Cleanup legacy handling after it has been switched over.
585 target = input_proto.build_target.name
586 if target:
587 # Legacy handling.
588 build_root = constants.SOURCE_ROOT
589 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
590 sysroot_path = os.path.join('/build', target)
591
592 sysroot = sysroot_lib.Sysroot(sysroot_path)
Andrew Lamb811aead2019-08-12 10:25:05 -0600593 chromeos_config = artifacts.BundleChromeOSConfig(chroot, sysroot, output_dir)
594 if chromeos_config is None:
595 cros_build_lib.Die(
596 'Could not create ChromeOS Config payload. No config found for %s.',
597 sysroot.path)
598 output_proto.artifacts.add().path = os.path.join(output_dir, chromeos_config)
599
600
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700601def _BundleSimpleChromeArtifactsResponse(input_proto, output_proto, _config):
602 """Add test simple chrome files to a successful response."""
603 output_proto.artifacts.add().path = os.path.join(
604 input_proto.output_dir, 'simple_chrome.txt')
605
606
607@faux.success(_BundleSimpleChromeArtifactsResponse)
608@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600609@validate.require('output_dir', 'sysroot.build_target.name', 'sysroot.path')
610@validate.exists('output_dir')
611@validate.validation_complete
612def BundleSimpleChromeArtifacts(input_proto, output_proto, _config):
Alex Klein2275d692019-04-23 16:04:12 -0600613 """Create the simple chrome artifacts."""
Alex Klein2275d692019-04-23 16:04:12 -0600614 sysroot_path = input_proto.sysroot.path
Alex Klein2275d692019-04-23 16:04:12 -0600615 output_dir = input_proto.output_dir
616
Alex Klein2275d692019-04-23 16:04:12 -0600617 # Build out the argument instances.
Alex Klein26e472b2020-03-10 14:35:01 -0600618 build_target = controller_util.ParseBuildTarget(
619 input_proto.sysroot.build_target)
620 chroot = controller_util.ParseChroot(input_proto.chroot)
Alex Klein2275d692019-04-23 16:04:12 -0600621 # Sysroot.path needs to be the fully qualified path, including the chroot.
622 full_sysroot_path = os.path.join(chroot.path, sysroot_path.lstrip(os.sep))
623 sysroot = sysroot_lib.Sysroot(full_sysroot_path)
624
625 # Quick sanity check that the sysroot exists before we go on.
626 if not sysroot.Exists():
627 cros_build_lib.Die('The sysroot does not exist.')
628
629 try:
630 results = artifacts.BundleSimpleChromeArtifacts(chroot, sysroot,
631 build_target, output_dir)
632 except artifacts.Error as e:
633 cros_build_lib.Die('Error %s raised in BundleSimpleChromeArtifacts: %s',
634 type(e), e)
635
636 for file_name in results:
637 output_proto.artifacts.add().path = file_name
638
639
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700640def _BundleVmFilesResponse(input_proto, output_proto, _config):
641 """Add test vm files to a successful response."""
642 output_proto.artifacts.add().path = os.path.join(
643 input_proto.output_dir, 'f1.tar')
644
645
646@faux.success(_BundleVmFilesResponse)
647@faux.empty_error
Michael Mortensen51f06722019-07-18 09:55:50 -0600648@validate.require('chroot.path', 'test_results_dir', 'output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600649@validate.exists('output_dir')
650@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +0000651def BundleVmFiles(
652 input_proto: artifacts_pb2.BundleVmFilesRequest,
653 output_proto: artifacts_pb2.BundleResponse,
654 _config: 'api_config.ApiConfig'):
Alex Klein6504eca2019-04-18 15:37:56 -0600655 """Tar VM disk and memory files.
656
657 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000658 input_proto: The input proto.
659 output_proto: The output proto.
660 _config: The API call config.
Alex Klein6504eca2019-04-18 15:37:56 -0600661 """
Michael Mortensen51f06722019-07-18 09:55:50 -0600662 chroot = controller_util.ParseChroot(input_proto.chroot)
663 test_results_dir = input_proto.test_results_dir
Alex Klein6504eca2019-04-18 15:37:56 -0600664 output_dir = input_proto.output_dir
665
Michael Mortensen51f06722019-07-18 09:55:50 -0600666 archives = artifacts.BundleVmFiles(
667 chroot, test_results_dir, output_dir)
Alex Klein6504eca2019-04-18 15:37:56 -0600668 for archive in archives:
669 output_proto.artifacts.add().path = archive
Tiancong Wangc4805b72019-06-11 12:12:03 -0700670
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700671def _BundleAFDOGenerationArtifactsResponse(input_proto, output_proto, _config):
672 """Add test tarball AFDO file to a successful response."""
673 output_proto.artifacts.add().path = os.path.join(
674 input_proto.output_dir, 'artifact1')
675
Alex Klein231d2da2019-07-22 16:44:45 -0600676
Tiancong Wang24a3df72019-08-20 15:48:51 -0700677_VALID_ARTIFACT_TYPES = [toolchain_pb2.BENCHMARK_AFDO,
678 toolchain_pb2.ORDERFILE]
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700679@faux.success(_BundleAFDOGenerationArtifactsResponse)
680@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600681@validate.require('build_target.name', 'output_dir')
Tiancong Wang50b80a92019-08-01 14:46:15 -0700682@validate.is_in('artifact_type', _VALID_ARTIFACT_TYPES)
Alex Klein231d2da2019-07-22 16:44:45 -0600683@validate.exists('output_dir')
Tiancong Wang2ade7932019-09-27 14:15:40 -0700684@validate.exists('chroot.chrome_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600685@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +0000686def BundleAFDOGenerationArtifacts(
687 input_proto: artifacts_pb2.BundleChromeAFDORequest,
688 output_proto: artifacts_pb2.BundleResponse,
689 _config: 'api_config.ApiConfig'):
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700690 """Generic function for creating tarballs of both AFDO and orderfile.
Tiancong Wangc4805b72019-06-11 12:12:03 -0700691
692 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000693 input_proto: The input proto.
694 output_proto: The output proto.
695 _config: The API call config.
Tiancong Wangc4805b72019-06-11 12:12:03 -0700696 """
Tiancong Wang2ade7932019-09-27 14:15:40 -0700697 chrome_root = input_proto.chroot.chrome_dir
Tiancong Wangc4805b72019-06-11 12:12:03 -0700698 output_dir = input_proto.output_dir
Tiancong Wang50b80a92019-08-01 14:46:15 -0700699 artifact_type = input_proto.artifact_type
Tiancong Wangc4805b72019-06-11 12:12:03 -0700700
Alex Klein26e472b2020-03-10 14:35:01 -0600701 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
Tiancong Wangc4805b72019-06-11 12:12:03 -0700702 chroot = controller_util.ParseChroot(input_proto.chroot)
703
704 try:
Tiancong Wang24a3df72019-08-20 15:48:51 -0700705 is_orderfile = bool(artifact_type is toolchain_pb2.ORDERFILE)
Tiancong Wang50b80a92019-08-01 14:46:15 -0700706 results = artifacts.BundleAFDOGenerationArtifacts(
Tiancong Wang2ade7932019-09-27 14:15:40 -0700707 is_orderfile, chroot, chrome_root,
Tiancong Wang50b80a92019-08-01 14:46:15 -0700708 build_target, output_dir)
Tiancong Wangc4805b72019-06-11 12:12:03 -0700709 except artifacts.Error as e:
710 cros_build_lib.Die('Error %s raised in BundleSimpleChromeArtifacts: %s',
711 type(e), e)
712
713 for file_name in results:
714 output_proto.artifacts.add().path = file_name
Alex Klein0b1cbfc2019-08-14 10:09:58 -0600715
716
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700717def _ExportCpeReportResponse(input_proto, output_proto, _config):
718 """Add test cpe results to a successful response."""
719 output_proto.artifacts.add().path = os.path.join(
720 input_proto.output_dir, 'cpe_report.txt')
721 output_proto.artifacts.add().path = os.path.join(
722 input_proto.output_dir, 'cpe_warnings.txt')
723
724
725@faux.success(_ExportCpeReportResponse)
726@faux.empty_error
Alex Klein0b1cbfc2019-08-14 10:09:58 -0600727@validate.exists('output_dir')
Varun Somani04dccd72021-10-09 01:06:11 +0000728def ExportCpeReport(
729 input_proto: artifacts_pb2.BundleRequest,
730 output_proto: artifacts_pb2.BundleResponse,
731 config: 'api_config.ApiConfig'):
Alex Klein0b1cbfc2019-08-14 10:09:58 -0600732 """Export a CPE report.
733
734 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000735 input_proto: The input proto.
736 output_proto: The output proto.
737 config: The API call config.
Alex Klein0b1cbfc2019-08-14 10:09:58 -0600738 """
739 chroot = controller_util.ParseChroot(input_proto.chroot)
740 output_dir = input_proto.output_dir
741
742 if input_proto.build_target.name:
743 # Legacy handling - use the default sysroot path for the build target.
744 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
745 sysroot = sysroot_lib.Sysroot(build_target.root)
746 elif input_proto.sysroot.path:
747 sysroot = sysroot_lib.Sysroot(input_proto.sysroot.path)
748 else:
749 # TODO(saklein): Switch to validate decorators once legacy handling can be
750 # cleaned up.
751 cros_build_lib.Die('sysroot.path is required.')
752
753 if config.validate_only:
754 return controller.RETURN_CODE_VALID_INPUT
755
756 cpe_result = artifacts.GenerateCpeReport(chroot, sysroot, output_dir)
757
758 output_proto.artifacts.add().path = cpe_result.report
759 output_proto.artifacts.add().path = cpe_result.warnings
Shao-Chuan Leea44dddc2020-10-30 17:16:55 +0900760
761
762def _BundleGceTarballResponse(input_proto, output_proto, _config):
763 """Add artifact tarball to a successful response."""
764 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
765 constants.TEST_IMAGE_GCE_TAR)
766
767
768@faux.success(_BundleGceTarballResponse)
769@faux.empty_error
770@validate.require('build_target.name', 'output_dir')
771@validate.exists('output_dir')
772@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +0000773def BundleGceTarball(
774 input_proto: artifacts_pb2.BundleRequest,
775 output_proto: artifacts_pb2.BundleResponse,
776 _config: 'api_config.ApiConfig'):
Shao-Chuan Leea44dddc2020-10-30 17:16:55 +0900777 """Bundle the test image into a tarball suitable for importing into GCE.
778
779 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000780 input_proto: The input proto.
781 output_proto: The output proto.
782 _config: The API call config.
Shao-Chuan Leea44dddc2020-10-30 17:16:55 +0900783 """
784 target = input_proto.build_target.name
785 output_dir = input_proto.output_dir
786 image_dir = _GetImageDir(constants.SOURCE_ROOT, target)
787 if image_dir is None:
788 return None
789
790 tarball = artifacts.BundleGceTarball(output_dir, image_dir)
791 output_proto.artifacts.add().path = tarball