blob: c6ae2a48fa406da34d959ac163f1170ec1fdf39c [file] [log] [blame]
Evan Hernandezf388cbf2019-04-01 11:15:23 -06001# Copyright 2019 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Implements ArtifactService."""
6
Chris McDonald1672ddb2021-07-21 11:48:23 -06007import logging
Evan Hernandezf388cbf2019-04-01 11:15:23 -06008import os
George Engelbrechtc9a8e812021-06-16 18:14:17 -06009from typing import Any, NamedTuple
Evan Hernandezf388cbf2019-04-01 11:15:23 -060010
Alex Klein231d2da2019-07-22 16:44:45 -060011from chromite.api import controller
Alex Klein076841b2019-08-29 15:19:39 -060012from chromite.api import faux
Alex Klein2b236722019-06-19 15:44:26 -060013from chromite.api import validate
Alex Klein238d8862019-05-07 11:32:46 -060014from chromite.api.controller import controller_util
George Engelbrechtc9a8e812021-06-16 18:14:17 -060015from chromite.api.controller import image as image_controller
16from chromite.api.controller import sysroot as sysroot_controller
David Wellingc1433c22021-06-25 16:29:48 +000017from chromite.api.controller import test as test_controller
LaMont Jones58362a42021-02-04 17:40:08 -070018from chromite.api.gen.chromite.api import artifacts_pb2
Tiancong Wang24a3df72019-08-20 15:48:51 -070019from chromite.api.gen.chromite.api import toolchain_pb2
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000020from chromite.api.gen.chromiumos import common_pb2
Alex Klein2275d692019-04-23 16:04:12 -060021from chromite.lib import chroot_lib
Evan Hernandezf388cbf2019-04-01 11:15:23 -060022from chromite.lib import constants
23from chromite.lib import cros_build_lib
Alex Klein2275d692019-04-23 16:04:12 -060024from chromite.lib import sysroot_lib
25from chromite.service import artifacts
Greg Edelstondc941072021-08-11 12:32:30 -060026from chromite.service import test
Evan Hernandezf388cbf2019-04-01 11:15:23 -060027
28
George Engelbrechtc9a8e812021-06-16 18:14:17 -060029class RegisteredGet(NamedTuple):
30 """An registered function for calling Get on an artifact type."""
31 output_proto: artifacts_pb2.GetResponse
32 artifact_dict: Any
LaMont Jones0f5171b2021-01-29 12:28:56 -070033
34
George Engelbrechtc9a8e812021-06-16 18:14:17 -060035def ExampleGetResponse(_input_proto, _output_proto, _config):
36 """Give an example GetResponse with a minimal coverage set."""
37 _output_proto = artifacts_pb2.GetResponse(
38 artifacts=common_pb2.UploadedArtifactsByService(
39 image=image_controller.ExampleGetResponse(),
40 sysroot=sysroot_controller.ExampleGetResponse(),
41 ))
42 return controller.RETURN_CODE_SUCCESS
43
44
LaMont Jones0f5171b2021-01-29 12:28:56 -070045@faux.empty_error
George Engelbrechtc9a8e812021-06-16 18:14:17 -060046@faux.success(ExampleGetResponse)
LaMont Jones0f5171b2021-01-29 12:28:56 -070047@validate.exists('result_path.path.path')
48@validate.validation_complete
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000049def Get(input_proto, output_proto, _config):
LaMont Jones0f5171b2021-01-29 12:28:56 -070050 """Get all artifacts.
51
52 Get all artifacts for the build.
53
George Engelbrechtc9a8e812021-06-16 18:14:17 -060054 Note: As the individual artifact_type bundlers are added here, they *must*
55 stop uploading it via the individual bundler function.
LaMont Jones0f5171b2021-01-29 12:28:56 -070056
57 Args:
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000058 input_proto (GetRequest): The input proto.
59 output_proto (GetResponse): The output proto.
LaMont Jones0f5171b2021-01-29 12:28:56 -070060 _config (api_config.ApiConfig): The API call config.
61 """
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000062 output_dir = input_proto.result_path.path.path
63
George Engelbrechtc9a8e812021-06-16 18:14:17 -060064 sysroot = controller_util.ParseSysroot(input_proto.sysroot)
LaMont Jones8b88e9d2021-06-28 16:37:34 -060065 # This endpoint does not currently support any artifacts that are built
66 # without a sysroot being present.
67 if not sysroot.path:
68 return controller.RETURN_CODE_SUCCESS
69
George Engelbrechtc9a8e812021-06-16 18:14:17 -060070 chroot = controller_util.ParseChroot(input_proto.chroot)
71 build_target = controller_util.ParseBuildTarget(
72 input_proto.sysroot.build_target)
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000073
George Engelbrechtc9a8e812021-06-16 18:14:17 -060074 # A list of RegisteredGet tuples (input proto, output proto, get results).
75 get_res_list = [
76 RegisteredGet(
77 output_proto.artifacts.image,
78 image_controller.GetArtifacts(
79 input_proto.artifact_info.image, chroot, sysroot, build_target,
80 output_dir)),
81 RegisteredGet(
82 output_proto.artifacts.sysroot,
83 sysroot_controller.GetArtifacts(
84 input_proto.artifact_info.sysroot, chroot, sysroot, build_target,
David Wellingc1433c22021-06-25 16:29:48 +000085 output_dir)),
86 RegisteredGet(
87 output_proto.artifacts.test,
88 test_controller.GetArtifacts(
89 input_proto.artifact_info.test, chroot, sysroot, output_dir)),
George Engelbrechtc9a8e812021-06-16 18:14:17 -060090 ]
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000091
George Engelbrechtc9a8e812021-06-16 18:14:17 -060092 for get_res in get_res_list:
93 for artifact_dict in get_res.artifact_dict:
94 get_res.output_proto.artifacts.add(
95 artifact_type=artifact_dict['type'],
96 paths=[
97 common_pb2.Path(
98 path=x, location=common_pb2.Path.Location.OUTSIDE)
99 for x in artifact_dict['paths']
100 ])
LaMont Jones0f5171b2021-01-29 12:28:56 -0700101 return controller.RETURN_CODE_SUCCESS
102
103
LaMont Jones58362a42021-02-04 17:40:08 -0700104def _BuildSetupResponse(_input_proto, output_proto, _config):
105 """Just return POINTLESS for now."""
106 # All of the artifact types we support claim that the build is POINTLESS.
107 output_proto.build_relevance = artifacts_pb2.BuildSetupResponse.POINTLESS
108
109
110@faux.success(_BuildSetupResponse)
111@faux.empty_error
112@validate.validation_complete
113def BuildSetup(_input_proto, output_proto, _config):
114 """Setup anything needed for building artifacts
115
116 If any artifact types require steps prior to building the package, they go
117 here. For example, see ToolchainService/PrepareForBuild.
118
119 Note: crbug/1034529 introduces this method as a noop. As the individual
120 artifact_type bundlers are added here, they *must* stop uploading it via the
121 individual bundler function.
122
123 Args:
124 _input_proto (GetRequest): The input proto.
125 output_proto (GetResponse): The output proto.
126 _config (api_config.ApiConfig): The API call config.
127 """
128 # If any artifact_type says "NEEDED", the return is NEEDED.
129 # Otherwise, if any artifact_type says "UNKNOWN", the return is UNKNOWN.
130 # Otherwise, the return is POINTLESS.
131 output_proto.build_relevance = artifacts_pb2.BuildSetupResponse.POINTLESS
132 return controller.RETURN_CODE_SUCCESS
133
134
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600135def _GetImageDir(build_root, target):
136 """Return path containing images for the given build target.
137
Alex Kleine2612a02019-04-18 13:51:06 -0600138 TODO(saklein) Expand image_lib.GetLatestImageLink to support this use case.
139
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600140 Args:
141 build_root (str): Path to checkout where build occurs.
142 target (str): Name of the build target.
143
144 Returns:
Alex Kleind2bf1462019-10-24 16:37:04 -0600145 Path to the latest directory containing target images or None.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600146 """
147 image_dir = os.path.join(build_root, 'src/build/images', target, 'latest')
148 if not os.path.exists(image_dir):
Alex Kleind2bf1462019-10-24 16:37:04 -0600149 logging.warning('Expected to find image output for target %s at %s, but '
150 'path does not exist', target, image_dir)
151 return None
152
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600153 return image_dir
154
155
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700156def _BundleImageArchivesResponse(input_proto, output_proto, _config):
157 """Add artifact paths to a successful response."""
158 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
159 'path0.tar.xz')
160 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
161 'path1.tar.xz')
162
163
164@faux.success(_BundleImageArchivesResponse)
165@faux.empty_error
Alex Kleind91e95a2019-09-17 10:39:02 -0600166@validate.require('build_target.name')
167@validate.exists('output_dir')
168@validate.validation_complete
169def BundleImageArchives(input_proto, output_proto, _config):
170 """Create a .tar.xz archive for each image that has been created."""
171 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
172 output_dir = input_proto.output_dir
173 image_dir = _GetImageDir(constants.SOURCE_ROOT, build_target.name)
Alex Kleind2bf1462019-10-24 16:37:04 -0600174 if image_dir is None:
175 return
Alex Kleind91e95a2019-09-17 10:39:02 -0600176
177 archives = artifacts.ArchiveImages(image_dir, output_dir)
178
179 for archive in archives:
180 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
181
182
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700183def _BundleImageZipResponse(input_proto, output_proto, _config):
184 """Add artifact zip files to a successful response."""
185 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
186 'image.zip')
187
188
189@faux.success(_BundleImageZipResponse)
190@faux.empty_error
Michael Mortensen01910922019-07-24 14:48:10 -0600191@validate.require('build_target.name', 'output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600192@validate.exists('output_dir')
193@validate.validation_complete
194def BundleImageZip(input_proto, output_proto, _config):
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600195 """Bundle image.zip.
196
197 Args:
198 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600199 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600200 _config (api_config.ApiConfig): The API call config.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600201 """
202 target = input_proto.build_target.name
203 output_dir = input_proto.output_dir
204 image_dir = _GetImageDir(constants.SOURCE_ROOT, target)
Alex Kleind2bf1462019-10-24 16:37:04 -0600205 if image_dir is None:
206 return None
Alex Klein231d2da2019-07-22 16:44:45 -0600207
Michael Mortensen01910922019-07-24 14:48:10 -0600208 archive = artifacts.BundleImageZip(output_dir, image_dir)
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600209 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
210
211
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700212def _BundleTestUpdatePayloadsResponse(input_proto, output_proto, _config):
213 """Add test payload files to a successful response."""
214 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
215 'payload1.bin')
216
217
218@faux.success(_BundleTestUpdatePayloadsResponse)
219@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600220@validate.require('build_target.name', 'output_dir')
221@validate.exists('output_dir')
222@validate.validation_complete
223def BundleTestUpdatePayloads(input_proto, output_proto, _config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600224 """Generate minimal update payloads for the build target for testing.
225
226 Args:
227 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600228 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600229 _config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600230 """
231 target = input_proto.build_target.name
232 output_dir = input_proto.output_dir
233 build_root = constants.SOURCE_ROOT
234
235 # Use the first available image to create the update payload.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600236 img_dir = _GetImageDir(build_root, target)
Alex Kleind2bf1462019-10-24 16:37:04 -0600237 if img_dir is None:
238 return None
239
Alex Kleincb541e82019-06-26 15:06:11 -0600240 img_types = [constants.IMAGE_TYPE_TEST, constants.IMAGE_TYPE_DEV,
241 constants.IMAGE_TYPE_BASE]
242 img_names = [constants.IMAGE_TYPE_TO_NAME[t] for t in img_types]
Mike Frysinger66ce4132019-07-17 22:52:52 -0400243 img_paths = [os.path.join(img_dir, x) for x in img_names]
Mike Frysingera552be42018-08-17 14:39:32 -0400244 valid_images = [x for x in img_paths if os.path.exists(x)]
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600245
Alex Kleincb541e82019-06-26 15:06:11 -0600246 if not valid_images:
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600247 cros_build_lib.Die(
248 'Expected to find an image of type among %r for target "%s" '
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600249 'at path %s.', img_types, target, img_dir)
Alex Kleincb541e82019-06-26 15:06:11 -0600250 image = valid_images[0]
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600251
Alex Kleincb541e82019-06-26 15:06:11 -0600252 payloads = artifacts.BundleTestUpdatePayloads(image, output_dir)
253 for payload in payloads:
254 output_proto.artifacts.add().path = payload
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600255
256
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700257def _BundleAutotestFilesResponse(input_proto, output_proto, _config):
258 """Add test autotest files to a successful response."""
259 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
260 'autotest-a.tar.gz')
261
262
263@faux.success(_BundleAutotestFilesResponse)
264@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600265@validate.require('output_dir')
266@validate.exists('output_dir')
267def BundleAutotestFiles(input_proto, output_proto, config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600268 """Tar the autotest files for a build target.
269
270 Args:
271 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600272 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600273 config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600274 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600275 output_dir = input_proto.output_dir
Alex Klein238d8862019-05-07 11:32:46 -0600276 target = input_proto.build_target.name
Alex Kleine21a0952019-08-23 16:08:16 -0600277 chroot = controller_util.ParseChroot(input_proto.chroot)
278
Alex Klein238d8862019-05-07 11:32:46 -0600279 if target:
Alex Kleine21a0952019-08-23 16:08:16 -0600280 sysroot_path = os.path.join('/build', target)
Alex Klein238d8862019-05-07 11:32:46 -0600281 else:
282 # New style call, use chroot and sysroot.
Alex Klein238d8862019-05-07 11:32:46 -0600283 sysroot_path = input_proto.sysroot.path
284 if not sysroot_path:
285 cros_build_lib.Die('sysroot.path is required.')
286
Alex Kleine21a0952019-08-23 16:08:16 -0600287 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein238d8862019-05-07 11:32:46 -0600288
Alex Klein231d2da2019-07-22 16:44:45 -0600289 # TODO(saklein): Switch to the validate_only decorator when legacy handling
290 # is removed.
291 if config.validate_only:
292 return controller.RETURN_CODE_VALID_INPUT
293
Alex Kleine21a0952019-08-23 16:08:16 -0600294 if not sysroot.Exists(chroot=chroot):
Alex Klein238d8862019-05-07 11:32:46 -0600295 cros_build_lib.Die('Sysroot path must exist: %s', sysroot.path)
296
297 try:
298 # Note that this returns the full path to *multiple* tarballs.
Alex Kleine21a0952019-08-23 16:08:16 -0600299 archives = artifacts.BundleAutotestFiles(chroot, sysroot, output_dir)
Alex Klein238d8862019-05-07 11:32:46 -0600300 except artifacts.Error as e:
Alex Klein03890312020-06-30 09:59:50 -0600301 logging.warning(e)
302 return
Alex Klein238d8862019-05-07 11:32:46 -0600303
304 for archive in archives.values():
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600305 output_proto.artifacts.add().path = archive
306
307
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700308def _BundleTastFilesResponse(input_proto, output_proto, _config):
309 """Add test tast files to a successful response."""
310 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
311 'tast_bundles.tar.gz')
312
313
314@faux.success(_BundleTastFilesResponse)
315@faux.empty_error
Alex Kleinb9d810b2019-07-01 12:38:02 -0600316@validate.require('output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600317@validate.exists('output_dir')
318def BundleTastFiles(input_proto, output_proto, config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600319 """Tar the tast files for a build target.
320
321 Args:
322 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600323 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600324 config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600325 """
326 target = input_proto.build_target.name
327 output_dir = input_proto.output_dir
328 build_root = constants.SOURCE_ROOT
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600329
Alex Kleinb9d810b2019-07-01 12:38:02 -0600330 chroot = controller_util.ParseChroot(input_proto.chroot)
331 sysroot_path = input_proto.sysroot.path
332
333 # TODO(saklein) Cleanup legacy handling after it has been switched over.
334 if target:
335 # Legacy handling.
Alex Klein171da612019-08-06 14:00:42 -0600336 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
Alex Kleinb9d810b2019-07-01 12:38:02 -0600337 sysroot_path = os.path.join('/build', target)
338
339 # New handling - chroot & sysroot based.
340 # TODO(saklein) Switch this to the require decorator when legacy is removed.
341 if not sysroot_path:
342 cros_build_lib.Die('sysroot.path is required.')
343
Alex Klein231d2da2019-07-22 16:44:45 -0600344 # TODO(saklein): Switch to the validation_complete decorator when legacy
345 # handling is removed.
346 if config.validate_only:
347 return controller.RETURN_CODE_VALID_INPUT
348
Alex Kleinb9d810b2019-07-01 12:38:02 -0600349 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein231d2da2019-07-22 16:44:45 -0600350 if not sysroot.Exists(chroot=chroot):
Alex Kleinb9d810b2019-07-01 12:38:02 -0600351 cros_build_lib.Die('Sysroot must exist.')
352
353 archive = artifacts.BundleTastFiles(chroot, sysroot, output_dir)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600354
LaMont Jonesb9793cd2020-06-11 08:14:46 -0600355 if archive:
356 output_proto.artifacts.add().path = archive
357 else:
358 logging.warning('Found no tast files for %s.', target)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600359
360
Fergus Dall34d74e12020-09-29 15:52:32 +1000361def BundlePinnedGuestImages(_input_proto, _output_proto, _config):
362 # TODO(crbug/1034529): Remove this endpoint
363 pass
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700364
Fergus Dall34d74e12020-09-29 15:52:32 +1000365def FetchPinnedGuestImageUris(_input_proto, _output_proto, _config):
366 # TODO(crbug/1034529): Remove this endpoint
367 pass
Alex Klein7bf0ecb2019-06-25 10:04:15 -0600368
369
Greg Edelstondc941072021-08-11 12:32:30 -0600370def _FetchMetadataResponse(_input_proto, output_proto, _config):
371 """Populate the output_proto with sample data."""
372 for fp in ('/metadata/foo.txt', '/metadata/bar.jsonproto'):
373 output_proto.filepaths.add(path=common_pb2.Path(
374 path=fp, location=common_pb2.Path.OUTSIDE))
375 return controller.RETURN_CODE_SUCCESS
376
377
378@faux.success(_FetchMetadataResponse)
379@faux.empty_error
380@validate.exists('chroot.path')
381@validate.require('sysroot.path')
382@validate.validation_complete
383def FetchMetadata(input_proto, output_proto, _config):
384 """FetchMetadata returns the paths to all build/test metadata files.
385
386 This implements ArtifactsService.FetchMetadata.
387
388 Args:
389 input_proto (FetchMetadataRequest): The input proto.
390 output_proto (FetchMetadataResponse): The output proto.
391 config (api_config.ApiConfig): The API call config.
392 """
393 chroot = controller_util.ParseChroot(input_proto.chroot)
394 sysroot = controller_util.ParseSysroot(input_proto.sysroot)
395 for path in test.FindAllMetadataFiles(chroot, sysroot):
396 output_proto.filepaths.add(
397 path=common_pb2.Path(path=path, location=common_pb2.Path.OUTSIDE))
398 return controller.RETURN_CODE_SUCCESS
399
400
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700401def _BundleFirmwareResponse(input_proto, output_proto, _config):
402 """Add test firmware image files to a successful response."""
403 output_proto.artifacts.add().path = os.path.join(
404 input_proto.output_dir, 'firmware.tar.gz')
405
406
407@faux.success(_BundleFirmwareResponse)
408@faux.empty_error
Michael Mortensen38675192019-06-28 16:52:55 +0000409@validate.require('output_dir', 'sysroot.path')
Alex Klein231d2da2019-07-22 16:44:45 -0600410@validate.exists('output_dir')
411@validate.validation_complete
412def BundleFirmware(input_proto, output_proto, _config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600413 """Tar the firmware images for a build target.
414
415 Args:
416 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600417 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600418 _config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600419 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600420 output_dir = input_proto.output_dir
Michael Mortensen38675192019-06-28 16:52:55 +0000421 chroot = controller_util.ParseChroot(input_proto.chroot)
422 sysroot_path = input_proto.sysroot.path
423 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein231d2da2019-07-22 16:44:45 -0600424
425 if not chroot.exists():
426 cros_build_lib.Die('Chroot does not exist: %s', chroot.path)
427 elif not sysroot.Exists(chroot=chroot):
428 cros_build_lib.Die('Sysroot does not exist: %s',
429 chroot.full_path(sysroot.path))
430
Michael Mortensen38675192019-06-28 16:52:55 +0000431 archive = artifacts.BuildFirmwareArchive(chroot, sysroot, output_dir)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600432
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600433 if archive is None:
434 cros_build_lib.Die(
Michael Mortensen38675192019-06-28 16:52:55 +0000435 'Could not create firmware archive. No firmware found for %s.',
436 sysroot_path)
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600437
Alex Klein231d2da2019-07-22 16:44:45 -0600438 output_proto.artifacts.add().path = archive
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600439
440
Yicheng Liea1181f2020-09-22 11:51:10 -0700441def _BundleFpmcuUnittestsResponse(input_proto, output_proto, _config):
442 """Add fingerprint MCU unittest binaries to a successful response."""
443 output_proto.artifacts.add().path = os.path.join(
444 input_proto.output_dir, 'fpmcu_unittests.tar.gz')
445
446
447@faux.success(_BundleFpmcuUnittestsResponse)
448@faux.empty_error
449@validate.require('output_dir', 'sysroot.path')
450@validate.exists('output_dir')
451@validate.validation_complete
452def BundleFpmcuUnittests(input_proto, output_proto, _config):
453 """Tar the fingerprint MCU unittest binaries for a build target.
454
455 Args:
456 input_proto (BundleRequest): The input proto.
457 output_proto (BundleResponse): The output proto.
458 _config (api_config.ApiConfig): The API call config.
459 """
460 output_dir = input_proto.output_dir
461 chroot = controller_util.ParseChroot(input_proto.chroot)
462 sysroot_path = input_proto.sysroot.path
463 sysroot = sysroot_lib.Sysroot(sysroot_path)
464
465 if not chroot.exists():
466 cros_build_lib.Die('Chroot does not exist: %s', chroot.path)
467 elif not sysroot.Exists(chroot=chroot):
468 cros_build_lib.Die('Sysroot does not exist: %s',
469 chroot.full_path(sysroot.path))
470
471 archive = artifacts.BundleFpmcuUnittests(chroot, sysroot, output_dir)
472
473 if archive is None:
474 logging.warning(
475 'No fpmcu unittests found for %s.', sysroot_path)
476 return
477
478 output_proto.artifacts.add().path = archive
479
480
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700481def _BundleEbuildLogsResponse(input_proto, output_proto, _config):
482 """Add test log files to a successful response."""
483 output_proto.artifacts.add().path = os.path.join(
484 input_proto.output_dir, 'ebuild-logs.tar.gz')
485
486
487@faux.success(_BundleEbuildLogsResponse)
488@faux.empty_error
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600489@validate.exists('output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600490def BundleEbuildLogs(input_proto, output_proto, config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600491 """Tar the ebuild logs for a build target.
492
493 Args:
494 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600495 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600496 config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600497 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600498 output_dir = input_proto.output_dir
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600499 sysroot_path = input_proto.sysroot.path
500 chroot = controller_util.ParseChroot(input_proto.chroot)
Evan Hernandeza478d802019-04-08 15:08:24 -0600501
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600502 # TODO(mmortensen) Cleanup legacy handling after it has been switched over.
503 target = input_proto.build_target.name
504 if target:
505 # Legacy handling.
506 build_root = constants.SOURCE_ROOT
Alex Klein171da612019-08-06 14:00:42 -0600507 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600508 sysroot_path = os.path.join('/build', target)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600509
Alex Klein231d2da2019-07-22 16:44:45 -0600510 # TODO(saklein): Switch to validation_complete decorator after legacy
511 # handling has been cleaned up.
512 if config.validate_only:
513 return controller.RETURN_CODE_VALID_INPUT
514
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600515 sysroot = sysroot_lib.Sysroot(sysroot_path)
516 archive = artifacts.BundleEBuildLogsTarball(chroot, sysroot, output_dir)
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600517 if archive is None:
518 cros_build_lib.Die(
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600519 'Could not create ebuild logs archive. No logs found for %s.',
520 sysroot.path)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600521 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
Alex Klein6504eca2019-04-18 15:37:56 -0600522
523
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700524def _BundleChromeOSConfigResponse(input_proto, output_proto, _config):
525 """Add test config files to a successful response."""
526 output_proto.artifacts.add().path = os.path.join(
527 input_proto.output_dir, 'config.yaml')
528
529
530@faux.success(_BundleChromeOSConfigResponse)
531@faux.empty_error
Andrew Lamb811aead2019-08-12 10:25:05 -0600532@validate.exists('output_dir')
533@validate.validation_complete
534def BundleChromeOSConfig(input_proto, output_proto, _config):
535 """Output the ChromeOS Config payload for a build target.
536
537 Args:
538 input_proto (BundleRequest): The input proto.
539 output_proto (BundleResponse): The output proto.
540 _config (api_config.ApiConfig): The API call config.
541 """
542 output_dir = input_proto.output_dir
Andrew Lamb67bd68f2019-08-15 09:09:15 -0600543 sysroot_path = input_proto.sysroot.path
Andrew Lamb811aead2019-08-12 10:25:05 -0600544 chroot = controller_util.ParseChroot(input_proto.chroot)
545
Andrew Lamb67bd68f2019-08-15 09:09:15 -0600546 # TODO(mmortensen) Cleanup legacy handling after it has been switched over.
547 target = input_proto.build_target.name
548 if target:
549 # Legacy handling.
550 build_root = constants.SOURCE_ROOT
551 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
552 sysroot_path = os.path.join('/build', target)
553
554 sysroot = sysroot_lib.Sysroot(sysroot_path)
Andrew Lamb811aead2019-08-12 10:25:05 -0600555 chromeos_config = artifacts.BundleChromeOSConfig(chroot, sysroot, output_dir)
556 if chromeos_config is None:
557 cros_build_lib.Die(
558 'Could not create ChromeOS Config payload. No config found for %s.',
559 sysroot.path)
560 output_proto.artifacts.add().path = os.path.join(output_dir, chromeos_config)
561
562
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700563def _BundleSimpleChromeArtifactsResponse(input_proto, output_proto, _config):
564 """Add test simple chrome files to a successful response."""
565 output_proto.artifacts.add().path = os.path.join(
566 input_proto.output_dir, 'simple_chrome.txt')
567
568
569@faux.success(_BundleSimpleChromeArtifactsResponse)
570@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600571@validate.require('output_dir', 'sysroot.build_target.name', 'sysroot.path')
572@validate.exists('output_dir')
573@validate.validation_complete
574def BundleSimpleChromeArtifacts(input_proto, output_proto, _config):
Alex Klein2275d692019-04-23 16:04:12 -0600575 """Create the simple chrome artifacts."""
Alex Klein2275d692019-04-23 16:04:12 -0600576 sysroot_path = input_proto.sysroot.path
Alex Klein2275d692019-04-23 16:04:12 -0600577 output_dir = input_proto.output_dir
578
Alex Klein2275d692019-04-23 16:04:12 -0600579 # Build out the argument instances.
Alex Klein26e472b2020-03-10 14:35:01 -0600580 build_target = controller_util.ParseBuildTarget(
581 input_proto.sysroot.build_target)
582 chroot = controller_util.ParseChroot(input_proto.chroot)
Alex Klein2275d692019-04-23 16:04:12 -0600583 # Sysroot.path needs to be the fully qualified path, including the chroot.
584 full_sysroot_path = os.path.join(chroot.path, sysroot_path.lstrip(os.sep))
585 sysroot = sysroot_lib.Sysroot(full_sysroot_path)
586
587 # Quick sanity check that the sysroot exists before we go on.
588 if not sysroot.Exists():
589 cros_build_lib.Die('The sysroot does not exist.')
590
591 try:
592 results = artifacts.BundleSimpleChromeArtifacts(chroot, sysroot,
593 build_target, output_dir)
594 except artifacts.Error as e:
595 cros_build_lib.Die('Error %s raised in BundleSimpleChromeArtifacts: %s',
596 type(e), e)
597
598 for file_name in results:
599 output_proto.artifacts.add().path = file_name
600
601
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700602def _BundleVmFilesResponse(input_proto, output_proto, _config):
603 """Add test vm files to a successful response."""
604 output_proto.artifacts.add().path = os.path.join(
605 input_proto.output_dir, 'f1.tar')
606
607
608@faux.success(_BundleVmFilesResponse)
609@faux.empty_error
Michael Mortensen51f06722019-07-18 09:55:50 -0600610@validate.require('chroot.path', 'test_results_dir', 'output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600611@validate.exists('output_dir')
612@validate.validation_complete
613def BundleVmFiles(input_proto, output_proto, _config):
Alex Klein6504eca2019-04-18 15:37:56 -0600614 """Tar VM disk and memory files.
615
616 Args:
Trent Begin008cade2019-10-31 13:40:59 -0600617 input_proto (BundleVmFilesRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600618 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600619 _config (api_config.ApiConfig): The API call config.
Alex Klein6504eca2019-04-18 15:37:56 -0600620 """
Michael Mortensen51f06722019-07-18 09:55:50 -0600621 chroot = controller_util.ParseChroot(input_proto.chroot)
622 test_results_dir = input_proto.test_results_dir
Alex Klein6504eca2019-04-18 15:37:56 -0600623 output_dir = input_proto.output_dir
624
Michael Mortensen51f06722019-07-18 09:55:50 -0600625 archives = artifacts.BundleVmFiles(
626 chroot, test_results_dir, output_dir)
Alex Klein6504eca2019-04-18 15:37:56 -0600627 for archive in archives:
628 output_proto.artifacts.add().path = archive
Tiancong Wangc4805b72019-06-11 12:12:03 -0700629
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700630def _BundleAFDOGenerationArtifactsResponse(input_proto, output_proto, _config):
631 """Add test tarball AFDO file to a successful response."""
632 output_proto.artifacts.add().path = os.path.join(
633 input_proto.output_dir, 'artifact1')
634
Alex Klein231d2da2019-07-22 16:44:45 -0600635
Tiancong Wang24a3df72019-08-20 15:48:51 -0700636_VALID_ARTIFACT_TYPES = [toolchain_pb2.BENCHMARK_AFDO,
637 toolchain_pb2.ORDERFILE]
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700638@faux.success(_BundleAFDOGenerationArtifactsResponse)
639@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600640@validate.require('build_target.name', 'output_dir')
Tiancong Wang50b80a92019-08-01 14:46:15 -0700641@validate.is_in('artifact_type', _VALID_ARTIFACT_TYPES)
Alex Klein231d2da2019-07-22 16:44:45 -0600642@validate.exists('output_dir')
Tiancong Wang2ade7932019-09-27 14:15:40 -0700643@validate.exists('chroot.chrome_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600644@validate.validation_complete
Tiancong Wang50b80a92019-08-01 14:46:15 -0700645def BundleAFDOGenerationArtifacts(input_proto, output_proto, _config):
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700646 """Generic function for creating tarballs of both AFDO and orderfile.
Tiancong Wangc4805b72019-06-11 12:12:03 -0700647
648 Args:
Tiancong Wang50b80a92019-08-01 14:46:15 -0700649 input_proto (BundleChromeAFDORequest): The input proto.
Tiancong Wangc4805b72019-06-11 12:12:03 -0700650 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600651 _config (api_config.ApiConfig): The API call config.
Tiancong Wangc4805b72019-06-11 12:12:03 -0700652 """
Tiancong Wang2ade7932019-09-27 14:15:40 -0700653 chrome_root = input_proto.chroot.chrome_dir
Tiancong Wangc4805b72019-06-11 12:12:03 -0700654 output_dir = input_proto.output_dir
Tiancong Wang50b80a92019-08-01 14:46:15 -0700655 artifact_type = input_proto.artifact_type
Tiancong Wangc4805b72019-06-11 12:12:03 -0700656
Alex Klein26e472b2020-03-10 14:35:01 -0600657 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
Tiancong Wangc4805b72019-06-11 12:12:03 -0700658 chroot = controller_util.ParseChroot(input_proto.chroot)
659
660 try:
Tiancong Wang24a3df72019-08-20 15:48:51 -0700661 is_orderfile = bool(artifact_type is toolchain_pb2.ORDERFILE)
Tiancong Wang50b80a92019-08-01 14:46:15 -0700662 results = artifacts.BundleAFDOGenerationArtifacts(
Tiancong Wang2ade7932019-09-27 14:15:40 -0700663 is_orderfile, chroot, chrome_root,
Tiancong Wang50b80a92019-08-01 14:46:15 -0700664 build_target, output_dir)
Tiancong Wangc4805b72019-06-11 12:12:03 -0700665 except artifacts.Error as e:
666 cros_build_lib.Die('Error %s raised in BundleSimpleChromeArtifacts: %s',
667 type(e), e)
668
669 for file_name in results:
670 output_proto.artifacts.add().path = file_name
Alex Klein0b1cbfc2019-08-14 10:09:58 -0600671
672
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700673def _ExportCpeReportResponse(input_proto, output_proto, _config):
674 """Add test cpe results to a successful response."""
675 output_proto.artifacts.add().path = os.path.join(
676 input_proto.output_dir, 'cpe_report.txt')
677 output_proto.artifacts.add().path = os.path.join(
678 input_proto.output_dir, 'cpe_warnings.txt')
679
680
681@faux.success(_ExportCpeReportResponse)
682@faux.empty_error
Alex Klein0b1cbfc2019-08-14 10:09:58 -0600683@validate.exists('output_dir')
684def ExportCpeReport(input_proto, output_proto, config):
685 """Export a CPE report.
686
687 Args:
688 input_proto (BundleRequest): The input proto.
689 output_proto (BundleResponse): The output proto.
690 config (api_config.ApiConfig): The API call config.
691 """
692 chroot = controller_util.ParseChroot(input_proto.chroot)
693 output_dir = input_proto.output_dir
694
695 if input_proto.build_target.name:
696 # Legacy handling - use the default sysroot path for the build target.
697 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
698 sysroot = sysroot_lib.Sysroot(build_target.root)
699 elif input_proto.sysroot.path:
700 sysroot = sysroot_lib.Sysroot(input_proto.sysroot.path)
701 else:
702 # TODO(saklein): Switch to validate decorators once legacy handling can be
703 # cleaned up.
704 cros_build_lib.Die('sysroot.path is required.')
705
706 if config.validate_only:
707 return controller.RETURN_CODE_VALID_INPUT
708
709 cpe_result = artifacts.GenerateCpeReport(chroot, sysroot, output_dir)
710
711 output_proto.artifacts.add().path = cpe_result.report
712 output_proto.artifacts.add().path = cpe_result.warnings
Shao-Chuan Leea44dddc2020-10-30 17:16:55 +0900713
714
715def _BundleGceTarballResponse(input_proto, output_proto, _config):
716 """Add artifact tarball to a successful response."""
717 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
718 constants.TEST_IMAGE_GCE_TAR)
719
720
721@faux.success(_BundleGceTarballResponse)
722@faux.empty_error
723@validate.require('build_target.name', 'output_dir')
724@validate.exists('output_dir')
725@validate.validation_complete
726def BundleGceTarball(input_proto, output_proto, _config):
727 """Bundle the test image into a tarball suitable for importing into GCE.
728
729 Args:
730 input_proto (BundleRequest): The input proto.
731 output_proto (BundleResponse): The output proto.
732 _config (api_config.ApiConfig): The API call config.
733 """
734 target = input_proto.build_target.name
735 output_dir = input_proto.output_dir
736 image_dir = _GetImageDir(constants.SOURCE_ROOT, target)
737 if image_dir is None:
738 return None
739
740 tarball = artifacts.BundleGceTarball(output_dir, image_dir)
741 output_proto.artifacts.add().path = tarball