blob: 28db93892cb88ad69d463884263cb7610355a335 [file] [log] [blame]
Evan Hernandezf388cbf2019-04-01 11:15:23 -06001# Copyright 2019 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Implements ArtifactService."""
6
Chris McDonald1672ddb2021-07-21 11:48:23 -06007import logging
Evan Hernandezf388cbf2019-04-01 11:15:23 -06008import os
Varun Somani04dccd72021-10-09 01:06:11 +00009from typing import Any, NamedTuple, Optional, TYPE_CHECKING
Evan Hernandezf388cbf2019-04-01 11:15:23 -060010
Alex Klein231d2da2019-07-22 16:44:45 -060011from chromite.api import controller
Alex Klein076841b2019-08-29 15:19:39 -060012from chromite.api import faux
Alex Klein2b236722019-06-19 15:44:26 -060013from chromite.api import validate
Alex Klein238d8862019-05-07 11:32:46 -060014from chromite.api.controller import controller_util
George Engelbrechtc9a8e812021-06-16 18:14:17 -060015from chromite.api.controller import image as image_controller
16from chromite.api.controller import sysroot as sysroot_controller
David Wellingc1433c22021-06-25 16:29:48 +000017from chromite.api.controller import test as test_controller
LaMont Jones58362a42021-02-04 17:40:08 -070018from chromite.api.gen.chromite.api import artifacts_pb2
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000019from chromite.api.gen.chromiumos import common_pb2
Alex Klein2275d692019-04-23 16:04:12 -060020from chromite.lib import chroot_lib
Evan Hernandezf388cbf2019-04-01 11:15:23 -060021from chromite.lib import constants
22from chromite.lib import cros_build_lib
Alex Klein2275d692019-04-23 16:04:12 -060023from chromite.lib import sysroot_lib
24from chromite.service import artifacts
Greg Edelstondc941072021-08-11 12:32:30 -060025from chromite.service import test
Evan Hernandezf388cbf2019-04-01 11:15:23 -060026
Mike Frysinger1cc8f1f2022-04-28 22:40:40 -040027
Varun Somani04dccd72021-10-09 01:06:11 +000028if TYPE_CHECKING:
29 from chromite.api import api_config
Evan Hernandezf388cbf2019-04-01 11:15:23 -060030
George Engelbrechtc9a8e812021-06-16 18:14:17 -060031class RegisteredGet(NamedTuple):
32 """An registered function for calling Get on an artifact type."""
33 output_proto: artifacts_pb2.GetResponse
34 artifact_dict: Any
LaMont Jones0f5171b2021-01-29 12:28:56 -070035
36
George Engelbrechtc9a8e812021-06-16 18:14:17 -060037def ExampleGetResponse(_input_proto, _output_proto, _config):
38 """Give an example GetResponse with a minimal coverage set."""
39 _output_proto = artifacts_pb2.GetResponse(
40 artifacts=common_pb2.UploadedArtifactsByService(
41 image=image_controller.ExampleGetResponse(),
42 sysroot=sysroot_controller.ExampleGetResponse(),
43 ))
44 return controller.RETURN_CODE_SUCCESS
45
46
LaMont Jones0f5171b2021-01-29 12:28:56 -070047@faux.empty_error
George Engelbrechtc9a8e812021-06-16 18:14:17 -060048@faux.success(ExampleGetResponse)
LaMont Jones0f5171b2021-01-29 12:28:56 -070049@validate.exists('result_path.path.path')
50@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +000051def Get(
52 input_proto: artifacts_pb2.GetRequest,
53 output_proto: artifacts_pb2.GetResponse,
54 _config: 'api_config.ApiConfig'):
LaMont Jones0f5171b2021-01-29 12:28:56 -070055 """Get all artifacts.
56
57 Get all artifacts for the build.
58
George Engelbrechtc9a8e812021-06-16 18:14:17 -060059 Note: As the individual artifact_type bundlers are added here, they *must*
60 stop uploading it via the individual bundler function.
LaMont Jones0f5171b2021-01-29 12:28:56 -070061
62 Args:
Varun Somani04dccd72021-10-09 01:06:11 +000063 input_proto: The input proto.
64 output_proto: The output proto.
65 _config: The API call config.
LaMont Jones0f5171b2021-01-29 12:28:56 -070066 """
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000067 output_dir = input_proto.result_path.path.path
68
George Engelbrechtc9a8e812021-06-16 18:14:17 -060069 sysroot = controller_util.ParseSysroot(input_proto.sysroot)
LaMont Jones8b88e9d2021-06-28 16:37:34 -060070 # This endpoint does not currently support any artifacts that are built
71 # without a sysroot being present.
72 if not sysroot.path:
73 return controller.RETURN_CODE_SUCCESS
74
George Engelbrechtc9a8e812021-06-16 18:14:17 -060075 chroot = controller_util.ParseChroot(input_proto.chroot)
76 build_target = controller_util.ParseBuildTarget(
77 input_proto.sysroot.build_target)
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000078
George Engelbrechtc9a8e812021-06-16 18:14:17 -060079 # A list of RegisteredGet tuples (input proto, output proto, get results).
80 get_res_list = [
81 RegisteredGet(
82 output_proto.artifacts.image,
83 image_controller.GetArtifacts(
84 input_proto.artifact_info.image, chroot, sysroot, build_target,
85 output_dir)),
86 RegisteredGet(
87 output_proto.artifacts.sysroot,
88 sysroot_controller.GetArtifacts(
89 input_proto.artifact_info.sysroot, chroot, sysroot, build_target,
David Wellingc1433c22021-06-25 16:29:48 +000090 output_dir)),
91 RegisteredGet(
92 output_proto.artifacts.test,
93 test_controller.GetArtifacts(
Jack Neusc9707c32021-07-23 21:48:54 +000094 input_proto.artifact_info.test, chroot, sysroot, build_target,
95 output_dir)),
George Engelbrechtc9a8e812021-06-16 18:14:17 -060096 ]
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000097
George Engelbrechtc9a8e812021-06-16 18:14:17 -060098 for get_res in get_res_list:
99 for artifact_dict in get_res.artifact_dict:
100 get_res.output_proto.artifacts.add(
101 artifact_type=artifact_dict['type'],
102 paths=[
103 common_pb2.Path(
104 path=x, location=common_pb2.Path.Location.OUTSIDE)
105 for x in artifact_dict['paths']
106 ])
LaMont Jones0f5171b2021-01-29 12:28:56 -0700107 return controller.RETURN_CODE_SUCCESS
108
109
LaMont Jones58362a42021-02-04 17:40:08 -0700110def _BuildSetupResponse(_input_proto, output_proto, _config):
111 """Just return POINTLESS for now."""
112 # All of the artifact types we support claim that the build is POINTLESS.
113 output_proto.build_relevance = artifacts_pb2.BuildSetupResponse.POINTLESS
114
115
116@faux.success(_BuildSetupResponse)
117@faux.empty_error
118@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +0000119def BuildSetup(
120 _input_proto: artifacts_pb2.GetRequest,
121 output_proto: artifacts_pb2.GetResponse,
122 _config: 'api_config.ApiConfig'):
123
LaMont Jones58362a42021-02-04 17:40:08 -0700124 """Setup anything needed for building artifacts
125
126 If any artifact types require steps prior to building the package, they go
127 here. For example, see ToolchainService/PrepareForBuild.
128
129 Note: crbug/1034529 introduces this method as a noop. As the individual
130 artifact_type bundlers are added here, they *must* stop uploading it via the
131 individual bundler function.
132
133 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000134 _input_proto: The input proto.
135 output_proto: The output proto.
136 _config: The API call config.
LaMont Jones58362a42021-02-04 17:40:08 -0700137 """
138 # If any artifact_type says "NEEDED", the return is NEEDED.
139 # Otherwise, if any artifact_type says "UNKNOWN", the return is UNKNOWN.
140 # Otherwise, the return is POINTLESS.
141 output_proto.build_relevance = artifacts_pb2.BuildSetupResponse.POINTLESS
142 return controller.RETURN_CODE_SUCCESS
143
144
Varun Somani04dccd72021-10-09 01:06:11 +0000145def _GetImageDir(build_root: str, target: str) -> Optional[str]:
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600146 """Return path containing images for the given build target.
147
Alex Kleine2612a02019-04-18 13:51:06 -0600148 TODO(saklein) Expand image_lib.GetLatestImageLink to support this use case.
149
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600150 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000151 build_root: Path to checkout where build occurs.
152 target: Name of the build target.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600153
154 Returns:
Alex Kleind2bf1462019-10-24 16:37:04 -0600155 Path to the latest directory containing target images or None.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600156 """
157 image_dir = os.path.join(build_root, 'src/build/images', target, 'latest')
158 if not os.path.exists(image_dir):
Alex Kleind2bf1462019-10-24 16:37:04 -0600159 logging.warning('Expected to find image output for target %s at %s, but '
160 'path does not exist', target, image_dir)
161 return None
162
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600163 return image_dir
164
165
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700166def _BundleImageArchivesResponse(input_proto, output_proto, _config):
167 """Add artifact paths to a successful response."""
168 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
169 'path0.tar.xz')
170 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
171 'path1.tar.xz')
172
173
174@faux.success(_BundleImageArchivesResponse)
175@faux.empty_error
Alex Kleind91e95a2019-09-17 10:39:02 -0600176@validate.require('build_target.name')
177@validate.exists('output_dir')
178@validate.validation_complete
179def BundleImageArchives(input_proto, output_proto, _config):
180 """Create a .tar.xz archive for each image that has been created."""
181 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
182 output_dir = input_proto.output_dir
183 image_dir = _GetImageDir(constants.SOURCE_ROOT, build_target.name)
Alex Kleind2bf1462019-10-24 16:37:04 -0600184 if image_dir is None:
185 return
Alex Kleind91e95a2019-09-17 10:39:02 -0600186
187 archives = artifacts.ArchiveImages(image_dir, output_dir)
188
189 for archive in archives:
190 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
191
192
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700193def _BundleImageZipResponse(input_proto, output_proto, _config):
194 """Add artifact zip files to a successful response."""
195 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
196 'image.zip')
197
198
199@faux.success(_BundleImageZipResponse)
200@faux.empty_error
Michael Mortensen01910922019-07-24 14:48:10 -0600201@validate.require('build_target.name', 'output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600202@validate.exists('output_dir')
203@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +0000204def BundleImageZip(
205 input_proto: artifacts_pb2.BundleRequest,
206 output_proto: artifacts_pb2.BundleResponse,
207 _config: 'api_config.ApiConfig'):
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600208 """Bundle image.zip.
209
210 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000211 input_proto: The input proto.
212 output_proto: The output proto.
213 _config: The API call config.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600214 """
215 target = input_proto.build_target.name
216 output_dir = input_proto.output_dir
217 image_dir = _GetImageDir(constants.SOURCE_ROOT, target)
Alex Kleind2bf1462019-10-24 16:37:04 -0600218 if image_dir is None:
219 return None
Alex Klein231d2da2019-07-22 16:44:45 -0600220
Michael Mortensen01910922019-07-24 14:48:10 -0600221 archive = artifacts.BundleImageZip(output_dir, image_dir)
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600222 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
223
224
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700225def _BundleTestUpdatePayloadsResponse(input_proto, output_proto, _config):
226 """Add test payload files to a successful response."""
227 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
228 'payload1.bin')
George Engelbrechtf0239d52022-04-06 13:09:33 -0600229 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
230 'payload1.json')
231 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
232 'payload1.log')
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700233
234
235@faux.success(_BundleTestUpdatePayloadsResponse)
236@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600237@validate.require('build_target.name', 'output_dir')
238@validate.exists('output_dir')
239@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +0000240def BundleTestUpdatePayloads(
241 input_proto: artifacts_pb2.BundleRequest,
242 output_proto: artifacts_pb2.BundleResponse,
243 _config: 'api_config.ApiConfig'):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600244 """Generate minimal update payloads for the build target for testing.
245
246 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000247 input_proto: The input proto.
248 output_proto: The output proto.
249 _config: The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600250 """
251 target = input_proto.build_target.name
252 output_dir = input_proto.output_dir
253 build_root = constants.SOURCE_ROOT
254
255 # Use the first available image to create the update payload.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600256 img_dir = _GetImageDir(build_root, target)
Alex Kleind2bf1462019-10-24 16:37:04 -0600257 if img_dir is None:
258 return None
259
Alex Kleincb541e82019-06-26 15:06:11 -0600260 img_types = [constants.IMAGE_TYPE_TEST, constants.IMAGE_TYPE_DEV,
261 constants.IMAGE_TYPE_BASE]
262 img_names = [constants.IMAGE_TYPE_TO_NAME[t] for t in img_types]
Mike Frysinger66ce4132019-07-17 22:52:52 -0400263 img_paths = [os.path.join(img_dir, x) for x in img_names]
Mike Frysingera552be42018-08-17 14:39:32 -0400264 valid_images = [x for x in img_paths if os.path.exists(x)]
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600265
Alex Kleincb541e82019-06-26 15:06:11 -0600266 if not valid_images:
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600267 cros_build_lib.Die(
268 'Expected to find an image of type among %r for target "%s" '
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600269 'at path %s.', img_types, target, img_dir)
Alex Kleincb541e82019-06-26 15:06:11 -0600270 image = valid_images[0]
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600271
Alex Kleincb541e82019-06-26 15:06:11 -0600272 payloads = artifacts.BundleTestUpdatePayloads(image, output_dir)
273 for payload in payloads:
274 output_proto.artifacts.add().path = payload
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600275
276
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700277def _BundleAutotestFilesResponse(input_proto, output_proto, _config):
278 """Add test autotest files to a successful response."""
279 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
280 'autotest-a.tar.gz')
281
282
283@faux.success(_BundleAutotestFilesResponse)
284@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600285@validate.require('output_dir')
286@validate.exists('output_dir')
Varun Somani04dccd72021-10-09 01:06:11 +0000287def BundleAutotestFiles(
288 input_proto: artifacts_pb2.BundleRequest,
289 output_proto: artifacts_pb2.BundleResponse,
290 config: 'api_config.ApiConfig'):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600291 """Tar the autotest files for a build target.
292
293 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000294 input_proto: The input proto.
295 output_proto: The output proto.
296 config: The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600297 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600298 output_dir = input_proto.output_dir
Alex Klein238d8862019-05-07 11:32:46 -0600299 target = input_proto.build_target.name
Alex Kleine21a0952019-08-23 16:08:16 -0600300 chroot = controller_util.ParseChroot(input_proto.chroot)
301
Alex Klein238d8862019-05-07 11:32:46 -0600302 if target:
Alex Kleine21a0952019-08-23 16:08:16 -0600303 sysroot_path = os.path.join('/build', target)
Alex Klein238d8862019-05-07 11:32:46 -0600304 else:
305 # New style call, use chroot and sysroot.
Alex Klein238d8862019-05-07 11:32:46 -0600306 sysroot_path = input_proto.sysroot.path
307 if not sysroot_path:
308 cros_build_lib.Die('sysroot.path is required.')
309
Alex Kleine21a0952019-08-23 16:08:16 -0600310 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein238d8862019-05-07 11:32:46 -0600311
Alex Klein231d2da2019-07-22 16:44:45 -0600312 # TODO(saklein): Switch to the validate_only decorator when legacy handling
313 # is removed.
314 if config.validate_only:
315 return controller.RETURN_CODE_VALID_INPUT
316
Alex Kleine21a0952019-08-23 16:08:16 -0600317 if not sysroot.Exists(chroot=chroot):
Alex Klein238d8862019-05-07 11:32:46 -0600318 cros_build_lib.Die('Sysroot path must exist: %s', sysroot.path)
319
320 try:
321 # Note that this returns the full path to *multiple* tarballs.
Alex Kleine21a0952019-08-23 16:08:16 -0600322 archives = artifacts.BundleAutotestFiles(chroot, sysroot, output_dir)
Alex Klein238d8862019-05-07 11:32:46 -0600323 except artifacts.Error as e:
Alex Klein03890312020-06-30 09:59:50 -0600324 logging.warning(e)
325 return
Alex Klein238d8862019-05-07 11:32:46 -0600326
327 for archive in archives.values():
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600328 output_proto.artifacts.add().path = archive
329
330
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700331def _BundleTastFilesResponse(input_proto, output_proto, _config):
332 """Add test tast files to a successful response."""
333 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
334 'tast_bundles.tar.gz')
335
336
337@faux.success(_BundleTastFilesResponse)
338@faux.empty_error
Alex Kleinb9d810b2019-07-01 12:38:02 -0600339@validate.require('output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600340@validate.exists('output_dir')
Varun Somani04dccd72021-10-09 01:06:11 +0000341def BundleTastFiles(
342 input_proto: artifacts_pb2.BundleRequest,
343 output_proto: artifacts_pb2.BundleResponse,
344 config: 'api_config.ApiConfig'):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600345 """Tar the tast files for a build target.
346
347 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000348 input_proto: The input proto.
349 output_proto: The output proto.
350 config: The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600351 """
352 target = input_proto.build_target.name
353 output_dir = input_proto.output_dir
354 build_root = constants.SOURCE_ROOT
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600355
Alex Kleinb9d810b2019-07-01 12:38:02 -0600356 chroot = controller_util.ParseChroot(input_proto.chroot)
357 sysroot_path = input_proto.sysroot.path
358
359 # TODO(saklein) Cleanup legacy handling after it has been switched over.
360 if target:
361 # Legacy handling.
Alex Klein171da612019-08-06 14:00:42 -0600362 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
Alex Kleinb9d810b2019-07-01 12:38:02 -0600363 sysroot_path = os.path.join('/build', target)
364
365 # New handling - chroot & sysroot based.
366 # TODO(saklein) Switch this to the require decorator when legacy is removed.
367 if not sysroot_path:
368 cros_build_lib.Die('sysroot.path is required.')
369
Alex Klein231d2da2019-07-22 16:44:45 -0600370 # TODO(saklein): Switch to the validation_complete decorator when legacy
371 # handling is removed.
372 if config.validate_only:
373 return controller.RETURN_CODE_VALID_INPUT
374
Alex Kleinb9d810b2019-07-01 12:38:02 -0600375 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein231d2da2019-07-22 16:44:45 -0600376 if not sysroot.Exists(chroot=chroot):
Alex Kleinb9d810b2019-07-01 12:38:02 -0600377 cros_build_lib.Die('Sysroot must exist.')
378
379 archive = artifacts.BundleTastFiles(chroot, sysroot, output_dir)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600380
LaMont Jonesb9793cd2020-06-11 08:14:46 -0600381 if archive:
382 output_proto.artifacts.add().path = archive
383 else:
384 logging.warning('Found no tast files for %s.', target)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600385
386
Fergus Dall34d74e12020-09-29 15:52:32 +1000387def BundlePinnedGuestImages(_input_proto, _output_proto, _config):
388 # TODO(crbug/1034529): Remove this endpoint
389 pass
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700390
Fergus Dall34d74e12020-09-29 15:52:32 +1000391def FetchPinnedGuestImageUris(_input_proto, _output_proto, _config):
392 # TODO(crbug/1034529): Remove this endpoint
393 pass
Alex Klein7bf0ecb2019-06-25 10:04:15 -0600394
395
Greg Edelstondc941072021-08-11 12:32:30 -0600396def _FetchMetadataResponse(_input_proto, output_proto, _config):
397 """Populate the output_proto with sample data."""
398 for fp in ('/metadata/foo.txt', '/metadata/bar.jsonproto'):
399 output_proto.filepaths.add(path=common_pb2.Path(
400 path=fp, location=common_pb2.Path.OUTSIDE))
401 return controller.RETURN_CODE_SUCCESS
402
403
404@faux.success(_FetchMetadataResponse)
405@faux.empty_error
406@validate.exists('chroot.path')
407@validate.require('sysroot.path')
408@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +0000409def FetchMetadata(
410 input_proto: artifacts_pb2.FetchMetadataRequest,
411 output_proto: artifacts_pb2.FetchMetadataResponse,
412 _config: 'api_config.ApiConfig'):
Greg Edelstondc941072021-08-11 12:32:30 -0600413 """FetchMetadata returns the paths to all build/test metadata files.
414
415 This implements ArtifactsService.FetchMetadata.
416
417 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000418 input_proto: The input proto.
419 output_proto: The output proto.
420 config: The API call config.
Greg Edelstondc941072021-08-11 12:32:30 -0600421 """
422 chroot = controller_util.ParseChroot(input_proto.chroot)
423 sysroot = controller_util.ParseSysroot(input_proto.sysroot)
424 for path in test.FindAllMetadataFiles(chroot, sysroot):
425 output_proto.filepaths.add(
426 path=common_pb2.Path(path=path, location=common_pb2.Path.OUTSIDE))
427 return controller.RETURN_CODE_SUCCESS
428
429
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700430def _BundleFirmwareResponse(input_proto, output_proto, _config):
431 """Add test firmware image files to a successful response."""
432 output_proto.artifacts.add().path = os.path.join(
433 input_proto.output_dir, 'firmware.tar.gz')
434
435
436@faux.success(_BundleFirmwareResponse)
437@faux.empty_error
Michael Mortensen38675192019-06-28 16:52:55 +0000438@validate.require('output_dir', 'sysroot.path')
Alex Klein231d2da2019-07-22 16:44:45 -0600439@validate.exists('output_dir')
440@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +0000441def BundleFirmware(
442 input_proto: artifacts_pb2.BundleRequest,
443 output_proto: artifacts_pb2.BundleResponse,
444 _config: 'api_config.ApiConfig'):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600445 """Tar the firmware images for a build target.
446
447 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000448 input_proto: The input proto.
449 output_proto: The output proto.
450 _config: The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600451 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600452 output_dir = input_proto.output_dir
Michael Mortensen38675192019-06-28 16:52:55 +0000453 chroot = controller_util.ParseChroot(input_proto.chroot)
454 sysroot_path = input_proto.sysroot.path
455 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein231d2da2019-07-22 16:44:45 -0600456
457 if not chroot.exists():
458 cros_build_lib.Die('Chroot does not exist: %s', chroot.path)
459 elif not sysroot.Exists(chroot=chroot):
460 cros_build_lib.Die('Sysroot does not exist: %s',
461 chroot.full_path(sysroot.path))
462
Michael Mortensen38675192019-06-28 16:52:55 +0000463 archive = artifacts.BuildFirmwareArchive(chroot, sysroot, output_dir)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600464
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600465 if archive is None:
George Engelbrecht9e41e172021-11-18 17:04:22 -0700466 logging.warning(
Michael Mortensen38675192019-06-28 16:52:55 +0000467 'Could not create firmware archive. No firmware found for %s.',
468 sysroot_path)
George Engelbrecht9e41e172021-11-18 17:04:22 -0700469 return
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600470
Alex Klein231d2da2019-07-22 16:44:45 -0600471 output_proto.artifacts.add().path = archive
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600472
473
Yicheng Liea1181f2020-09-22 11:51:10 -0700474def _BundleFpmcuUnittestsResponse(input_proto, output_proto, _config):
475 """Add fingerprint MCU unittest binaries to a successful response."""
476 output_proto.artifacts.add().path = os.path.join(
477 input_proto.output_dir, 'fpmcu_unittests.tar.gz')
478
479
480@faux.success(_BundleFpmcuUnittestsResponse)
481@faux.empty_error
482@validate.require('output_dir', 'sysroot.path')
483@validate.exists('output_dir')
484@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +0000485def BundleFpmcuUnittests(
486 input_proto: artifacts_pb2.BundleRequest,
487 output_proto: artifacts_pb2.BundleResponse,
488 _config: 'api_config.ApiConfig'):
Yicheng Liea1181f2020-09-22 11:51:10 -0700489 """Tar the fingerprint MCU unittest binaries for a build target.
490
491 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000492 input_proto: The input proto.
493 output_proto: The output proto.
494 _config: The API call config.
Yicheng Liea1181f2020-09-22 11:51:10 -0700495 """
496 output_dir = input_proto.output_dir
497 chroot = controller_util.ParseChroot(input_proto.chroot)
498 sysroot_path = input_proto.sysroot.path
499 sysroot = sysroot_lib.Sysroot(sysroot_path)
500
501 if not chroot.exists():
502 cros_build_lib.Die('Chroot does not exist: %s', chroot.path)
503 elif not sysroot.Exists(chroot=chroot):
504 cros_build_lib.Die('Sysroot does not exist: %s',
505 chroot.full_path(sysroot.path))
506
507 archive = artifacts.BundleFpmcuUnittests(chroot, sysroot, output_dir)
508
509 if archive is None:
510 logging.warning(
511 'No fpmcu unittests found for %s.', sysroot_path)
512 return
513
514 output_proto.artifacts.add().path = archive
515
516
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700517def _BundleEbuildLogsResponse(input_proto, output_proto, _config):
518 """Add test log files to a successful response."""
519 output_proto.artifacts.add().path = os.path.join(
520 input_proto.output_dir, 'ebuild-logs.tar.gz')
521
522
523@faux.success(_BundleEbuildLogsResponse)
524@faux.empty_error
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600525@validate.exists('output_dir')
Varun Somani04dccd72021-10-09 01:06:11 +0000526def BundleEbuildLogs(
527 input_proto: artifacts_pb2.BundleRequest,
528 output_proto: artifacts_pb2.BundleResponse,
529 config: 'api_config.ApiConfig'):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600530 """Tar the ebuild logs for a build target.
531
532 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000533 input_proto: The input proto.
534 output_proto: The output proto.
535 config: The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600536 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600537 output_dir = input_proto.output_dir
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600538 sysroot_path = input_proto.sysroot.path
539 chroot = controller_util.ParseChroot(input_proto.chroot)
Evan Hernandeza478d802019-04-08 15:08:24 -0600540
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600541 # TODO(mmortensen) Cleanup legacy handling after it has been switched over.
542 target = input_proto.build_target.name
543 if target:
544 # Legacy handling.
545 build_root = constants.SOURCE_ROOT
Alex Klein171da612019-08-06 14:00:42 -0600546 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600547 sysroot_path = os.path.join('/build', target)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600548
Alex Klein231d2da2019-07-22 16:44:45 -0600549 # TODO(saklein): Switch to validation_complete decorator after legacy
550 # handling has been cleaned up.
551 if config.validate_only:
552 return controller.RETURN_CODE_VALID_INPUT
553
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600554 sysroot = sysroot_lib.Sysroot(sysroot_path)
555 archive = artifacts.BundleEBuildLogsTarball(chroot, sysroot, output_dir)
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600556 if archive is None:
557 cros_build_lib.Die(
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600558 'Could not create ebuild logs archive. No logs found for %s.',
559 sysroot.path)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600560 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
Alex Klein6504eca2019-04-18 15:37:56 -0600561
562
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700563def _BundleChromeOSConfigResponse(input_proto, output_proto, _config):
564 """Add test config files to a successful response."""
565 output_proto.artifacts.add().path = os.path.join(
566 input_proto.output_dir, 'config.yaml')
567
568
569@faux.success(_BundleChromeOSConfigResponse)
570@faux.empty_error
Andrew Lamb811aead2019-08-12 10:25:05 -0600571@validate.exists('output_dir')
572@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +0000573def BundleChromeOSConfig(
574 input_proto: artifacts_pb2.BundleRequest,
575 output_proto: artifacts_pb2.BundleResponse,
576 _config: 'api_config.ApiConfig'):
Andrew Lamb811aead2019-08-12 10:25:05 -0600577 """Output the ChromeOS Config payload for a build target.
578
579 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000580 input_proto: The input proto.
581 output_proto: The output proto.
582 _config: The API call config.
Andrew Lamb811aead2019-08-12 10:25:05 -0600583 """
584 output_dir = input_proto.output_dir
Andrew Lamb67bd68f2019-08-15 09:09:15 -0600585 sysroot_path = input_proto.sysroot.path
Andrew Lamb811aead2019-08-12 10:25:05 -0600586 chroot = controller_util.ParseChroot(input_proto.chroot)
587
Andrew Lamb67bd68f2019-08-15 09:09:15 -0600588 # TODO(mmortensen) Cleanup legacy handling after it has been switched over.
589 target = input_proto.build_target.name
590 if target:
591 # Legacy handling.
592 build_root = constants.SOURCE_ROOT
593 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
594 sysroot_path = os.path.join('/build', target)
595
596 sysroot = sysroot_lib.Sysroot(sysroot_path)
Andrew Lamb811aead2019-08-12 10:25:05 -0600597 chromeos_config = artifacts.BundleChromeOSConfig(chroot, sysroot, output_dir)
Alex Klein383a7a32021-12-07 16:01:19 -0700598 if not chromeos_config:
599 return
600
Andrew Lamb811aead2019-08-12 10:25:05 -0600601 output_proto.artifacts.add().path = os.path.join(output_dir, chromeos_config)
602
603
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700604def _BundleSimpleChromeArtifactsResponse(input_proto, output_proto, _config):
605 """Add test simple chrome files to a successful response."""
606 output_proto.artifacts.add().path = os.path.join(
607 input_proto.output_dir, 'simple_chrome.txt')
608
609
610@faux.success(_BundleSimpleChromeArtifactsResponse)
611@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600612@validate.require('output_dir', 'sysroot.build_target.name', 'sysroot.path')
613@validate.exists('output_dir')
614@validate.validation_complete
615def BundleSimpleChromeArtifacts(input_proto, output_proto, _config):
Alex Klein2275d692019-04-23 16:04:12 -0600616 """Create the simple chrome artifacts."""
Alex Klein2275d692019-04-23 16:04:12 -0600617 sysroot_path = input_proto.sysroot.path
Alex Klein2275d692019-04-23 16:04:12 -0600618 output_dir = input_proto.output_dir
619
Alex Klein2275d692019-04-23 16:04:12 -0600620 # Build out the argument instances.
Alex Klein26e472b2020-03-10 14:35:01 -0600621 build_target = controller_util.ParseBuildTarget(
622 input_proto.sysroot.build_target)
623 chroot = controller_util.ParseChroot(input_proto.chroot)
Alex Klein2275d692019-04-23 16:04:12 -0600624 # Sysroot.path needs to be the fully qualified path, including the chroot.
625 full_sysroot_path = os.path.join(chroot.path, sysroot_path.lstrip(os.sep))
626 sysroot = sysroot_lib.Sysroot(full_sysroot_path)
627
628 # Quick sanity check that the sysroot exists before we go on.
629 if not sysroot.Exists():
630 cros_build_lib.Die('The sysroot does not exist.')
631
632 try:
633 results = artifacts.BundleSimpleChromeArtifacts(chroot, sysroot,
634 build_target, output_dir)
635 except artifacts.Error as e:
636 cros_build_lib.Die('Error %s raised in BundleSimpleChromeArtifacts: %s',
637 type(e), e)
638
639 for file_name in results:
640 output_proto.artifacts.add().path = file_name
641
642
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700643def _BundleVmFilesResponse(input_proto, output_proto, _config):
644 """Add test vm files to a successful response."""
645 output_proto.artifacts.add().path = os.path.join(
646 input_proto.output_dir, 'f1.tar')
647
648
649@faux.success(_BundleVmFilesResponse)
650@faux.empty_error
Michael Mortensen51f06722019-07-18 09:55:50 -0600651@validate.require('chroot.path', 'test_results_dir', 'output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600652@validate.exists('output_dir')
653@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +0000654def BundleVmFiles(
655 input_proto: artifacts_pb2.BundleVmFilesRequest,
656 output_proto: artifacts_pb2.BundleResponse,
657 _config: 'api_config.ApiConfig'):
Alex Klein6504eca2019-04-18 15:37:56 -0600658 """Tar VM disk and memory files.
659
660 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000661 input_proto: The input proto.
662 output_proto: The output proto.
663 _config: The API call config.
Alex Klein6504eca2019-04-18 15:37:56 -0600664 """
Michael Mortensen51f06722019-07-18 09:55:50 -0600665 chroot = controller_util.ParseChroot(input_proto.chroot)
666 test_results_dir = input_proto.test_results_dir
Alex Klein6504eca2019-04-18 15:37:56 -0600667 output_dir = input_proto.output_dir
668
Michael Mortensen51f06722019-07-18 09:55:50 -0600669 archives = artifacts.BundleVmFiles(
670 chroot, test_results_dir, output_dir)
Alex Klein6504eca2019-04-18 15:37:56 -0600671 for archive in archives:
672 output_proto.artifacts.add().path = archive
Tiancong Wangc4805b72019-06-11 12:12:03 -0700673
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700674def _ExportCpeReportResponse(input_proto, output_proto, _config):
675 """Add test cpe results to a successful response."""
676 output_proto.artifacts.add().path = os.path.join(
677 input_proto.output_dir, 'cpe_report.txt')
678 output_proto.artifacts.add().path = os.path.join(
679 input_proto.output_dir, 'cpe_warnings.txt')
680
681
682@faux.success(_ExportCpeReportResponse)
683@faux.empty_error
Alex Klein0b1cbfc2019-08-14 10:09:58 -0600684@validate.exists('output_dir')
Varun Somani04dccd72021-10-09 01:06:11 +0000685def ExportCpeReport(
686 input_proto: artifacts_pb2.BundleRequest,
687 output_proto: artifacts_pb2.BundleResponse,
688 config: 'api_config.ApiConfig'):
Alex Klein0b1cbfc2019-08-14 10:09:58 -0600689 """Export a CPE report.
690
691 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000692 input_proto: The input proto.
693 output_proto: The output proto.
694 config: The API call config.
Alex Klein0b1cbfc2019-08-14 10:09:58 -0600695 """
696 chroot = controller_util.ParseChroot(input_proto.chroot)
697 output_dir = input_proto.output_dir
698
699 if input_proto.build_target.name:
700 # Legacy handling - use the default sysroot path for the build target.
701 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
702 sysroot = sysroot_lib.Sysroot(build_target.root)
703 elif input_proto.sysroot.path:
704 sysroot = sysroot_lib.Sysroot(input_proto.sysroot.path)
705 else:
706 # TODO(saklein): Switch to validate decorators once legacy handling can be
707 # cleaned up.
708 cros_build_lib.Die('sysroot.path is required.')
709
710 if config.validate_only:
711 return controller.RETURN_CODE_VALID_INPUT
712
713 cpe_result = artifacts.GenerateCpeReport(chroot, sysroot, output_dir)
714
715 output_proto.artifacts.add().path = cpe_result.report
716 output_proto.artifacts.add().path = cpe_result.warnings
Shao-Chuan Leea44dddc2020-10-30 17:16:55 +0900717
718
719def _BundleGceTarballResponse(input_proto, output_proto, _config):
720 """Add artifact tarball to a successful response."""
721 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
722 constants.TEST_IMAGE_GCE_TAR)
723
724
725@faux.success(_BundleGceTarballResponse)
726@faux.empty_error
727@validate.require('build_target.name', 'output_dir')
728@validate.exists('output_dir')
729@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +0000730def BundleGceTarball(
731 input_proto: artifacts_pb2.BundleRequest,
732 output_proto: artifacts_pb2.BundleResponse,
733 _config: 'api_config.ApiConfig'):
Shao-Chuan Leea44dddc2020-10-30 17:16:55 +0900734 """Bundle the test image into a tarball suitable for importing into GCE.
735
736 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000737 input_proto: The input proto.
738 output_proto: The output proto.
739 _config: The API call config.
Shao-Chuan Leea44dddc2020-10-30 17:16:55 +0900740 """
741 target = input_proto.build_target.name
742 output_dir = input_proto.output_dir
743 image_dir = _GetImageDir(constants.SOURCE_ROOT, target)
744 if image_dir is None:
745 return None
746
747 tarball = artifacts.BundleGceTarball(output_dir, image_dir)
748 output_proto.artifacts.add().path = tarball