blob: 1a626055a9c0262fd836ed026d90eadb9ad62d64 [file] [log] [blame]
Evan Hernandezf388cbf2019-04-01 11:15:23 -06001# Copyright 2019 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Implements ArtifactService."""
6
Evan Hernandezf388cbf2019-04-01 11:15:23 -06007import os
George Engelbrechtc9a8e812021-06-16 18:14:17 -06008from typing import Any, NamedTuple
Evan Hernandezf388cbf2019-04-01 11:15:23 -06009
Alex Klein231d2da2019-07-22 16:44:45 -060010from chromite.api import controller
Alex Klein076841b2019-08-29 15:19:39 -060011from chromite.api import faux
Alex Klein2b236722019-06-19 15:44:26 -060012from chromite.api import validate
Alex Klein238d8862019-05-07 11:32:46 -060013from chromite.api.controller import controller_util
George Engelbrechtc9a8e812021-06-16 18:14:17 -060014from chromite.api.controller import image as image_controller
15from chromite.api.controller import sysroot as sysroot_controller
LaMont Jones58362a42021-02-04 17:40:08 -070016from chromite.api.gen.chromite.api import artifacts_pb2
Tiancong Wang24a3df72019-08-20 15:48:51 -070017from chromite.api.gen.chromite.api import toolchain_pb2
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000018from chromite.api.gen.chromiumos import common_pb2
Alex Klein2275d692019-04-23 16:04:12 -060019from chromite.lib import chroot_lib
Evan Hernandezf388cbf2019-04-01 11:15:23 -060020from chromite.lib import constants
21from chromite.lib import cros_build_lib
Evan Hernandezde445982019-04-22 13:42:34 -060022from chromite.lib import cros_logging as logging
Alex Klein2275d692019-04-23 16:04:12 -060023from chromite.lib import sysroot_lib
24from chromite.service import artifacts
Evan Hernandezf388cbf2019-04-01 11:15:23 -060025
26
George Engelbrechtc9a8e812021-06-16 18:14:17 -060027class RegisteredGet(NamedTuple):
28 """An registered function for calling Get on an artifact type."""
29 output_proto: artifacts_pb2.GetResponse
30 artifact_dict: Any
LaMont Jones0f5171b2021-01-29 12:28:56 -070031
32
George Engelbrechtc9a8e812021-06-16 18:14:17 -060033def ExampleGetResponse(_input_proto, _output_proto, _config):
34 """Give an example GetResponse with a minimal coverage set."""
35 _output_proto = artifacts_pb2.GetResponse(
36 artifacts=common_pb2.UploadedArtifactsByService(
37 image=image_controller.ExampleGetResponse(),
38 sysroot=sysroot_controller.ExampleGetResponse(),
39 ))
40 return controller.RETURN_CODE_SUCCESS
41
42
LaMont Jones0f5171b2021-01-29 12:28:56 -070043@faux.empty_error
George Engelbrechtc9a8e812021-06-16 18:14:17 -060044@faux.success(ExampleGetResponse)
LaMont Jones0f5171b2021-01-29 12:28:56 -070045@validate.exists('result_path.path.path')
46@validate.validation_complete
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000047def Get(input_proto, output_proto, _config):
LaMont Jones0f5171b2021-01-29 12:28:56 -070048 """Get all artifacts.
49
50 Get all artifacts for the build.
51
George Engelbrechtc9a8e812021-06-16 18:14:17 -060052 Note: As the individual artifact_type bundlers are added here, they *must*
53 stop uploading it via the individual bundler function.
LaMont Jones0f5171b2021-01-29 12:28:56 -070054
55 Args:
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000056 input_proto (GetRequest): The input proto.
57 output_proto (GetResponse): The output proto.
LaMont Jones0f5171b2021-01-29 12:28:56 -070058 _config (api_config.ApiConfig): The API call config.
59 """
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000060 output_dir = input_proto.result_path.path.path
61
George Engelbrechtc9a8e812021-06-16 18:14:17 -060062 sysroot = controller_util.ParseSysroot(input_proto.sysroot)
63 chroot = controller_util.ParseChroot(input_proto.chroot)
64 build_target = controller_util.ParseBuildTarget(
65 input_proto.sysroot.build_target)
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000066
George Engelbrechtc9a8e812021-06-16 18:14:17 -060067 # A list of RegisteredGet tuples (input proto, output proto, get results).
68 get_res_list = [
69 RegisteredGet(
70 output_proto.artifacts.image,
71 image_controller.GetArtifacts(
72 input_proto.artifact_info.image, chroot, sysroot, build_target,
73 output_dir)),
74 RegisteredGet(
75 output_proto.artifacts.sysroot,
76 sysroot_controller.GetArtifacts(
77 input_proto.artifact_info.sysroot, chroot, sysroot, build_target,
78 output_dir))
79 ]
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000080
George Engelbrechtc9a8e812021-06-16 18:14:17 -060081 for get_res in get_res_list:
82 for artifact_dict in get_res.artifact_dict:
83 get_res.output_proto.artifacts.add(
84 artifact_type=artifact_dict['type'],
85 paths=[
86 common_pb2.Path(
87 path=x, location=common_pb2.Path.Location.OUTSIDE)
88 for x in artifact_dict['paths']
89 ])
LaMont Jones0f5171b2021-01-29 12:28:56 -070090 return controller.RETURN_CODE_SUCCESS
91
92
LaMont Jones58362a42021-02-04 17:40:08 -070093def _BuildSetupResponse(_input_proto, output_proto, _config):
94 """Just return POINTLESS for now."""
95 # All of the artifact types we support claim that the build is POINTLESS.
96 output_proto.build_relevance = artifacts_pb2.BuildSetupResponse.POINTLESS
97
98
99@faux.success(_BuildSetupResponse)
100@faux.empty_error
101@validate.validation_complete
102def BuildSetup(_input_proto, output_proto, _config):
103 """Setup anything needed for building artifacts
104
105 If any artifact types require steps prior to building the package, they go
106 here. For example, see ToolchainService/PrepareForBuild.
107
108 Note: crbug/1034529 introduces this method as a noop. As the individual
109 artifact_type bundlers are added here, they *must* stop uploading it via the
110 individual bundler function.
111
112 Args:
113 _input_proto (GetRequest): The input proto.
114 output_proto (GetResponse): The output proto.
115 _config (api_config.ApiConfig): The API call config.
116 """
117 # If any artifact_type says "NEEDED", the return is NEEDED.
118 # Otherwise, if any artifact_type says "UNKNOWN", the return is UNKNOWN.
119 # Otherwise, the return is POINTLESS.
120 output_proto.build_relevance = artifacts_pb2.BuildSetupResponse.POINTLESS
121 return controller.RETURN_CODE_SUCCESS
122
123
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600124def _GetImageDir(build_root, target):
125 """Return path containing images for the given build target.
126
Alex Kleine2612a02019-04-18 13:51:06 -0600127 TODO(saklein) Expand image_lib.GetLatestImageLink to support this use case.
128
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600129 Args:
130 build_root (str): Path to checkout where build occurs.
131 target (str): Name of the build target.
132
133 Returns:
Alex Kleind2bf1462019-10-24 16:37:04 -0600134 Path to the latest directory containing target images or None.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600135 """
136 image_dir = os.path.join(build_root, 'src/build/images', target, 'latest')
137 if not os.path.exists(image_dir):
Alex Kleind2bf1462019-10-24 16:37:04 -0600138 logging.warning('Expected to find image output for target %s at %s, but '
139 'path does not exist', target, image_dir)
140 return None
141
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600142 return image_dir
143
144
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700145def _BundleImageArchivesResponse(input_proto, output_proto, _config):
146 """Add artifact paths to a successful response."""
147 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
148 'path0.tar.xz')
149 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
150 'path1.tar.xz')
151
152
153@faux.success(_BundleImageArchivesResponse)
154@faux.empty_error
Alex Kleind91e95a2019-09-17 10:39:02 -0600155@validate.require('build_target.name')
156@validate.exists('output_dir')
157@validate.validation_complete
158def BundleImageArchives(input_proto, output_proto, _config):
159 """Create a .tar.xz archive for each image that has been created."""
160 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
161 output_dir = input_proto.output_dir
162 image_dir = _GetImageDir(constants.SOURCE_ROOT, build_target.name)
Alex Kleind2bf1462019-10-24 16:37:04 -0600163 if image_dir is None:
164 return
Alex Kleind91e95a2019-09-17 10:39:02 -0600165
166 archives = artifacts.ArchiveImages(image_dir, output_dir)
167
168 for archive in archives:
169 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
170
171
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700172def _BundleImageZipResponse(input_proto, output_proto, _config):
173 """Add artifact zip files to a successful response."""
174 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
175 'image.zip')
176
177
178@faux.success(_BundleImageZipResponse)
179@faux.empty_error
Michael Mortensen01910922019-07-24 14:48:10 -0600180@validate.require('build_target.name', 'output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600181@validate.exists('output_dir')
182@validate.validation_complete
183def BundleImageZip(input_proto, output_proto, _config):
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600184 """Bundle image.zip.
185
186 Args:
187 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600188 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600189 _config (api_config.ApiConfig): The API call config.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600190 """
191 target = input_proto.build_target.name
192 output_dir = input_proto.output_dir
193 image_dir = _GetImageDir(constants.SOURCE_ROOT, target)
Alex Kleind2bf1462019-10-24 16:37:04 -0600194 if image_dir is None:
195 return None
Alex Klein231d2da2019-07-22 16:44:45 -0600196
Michael Mortensen01910922019-07-24 14:48:10 -0600197 archive = artifacts.BundleImageZip(output_dir, image_dir)
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600198 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
199
200
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700201def _BundleTestUpdatePayloadsResponse(input_proto, output_proto, _config):
202 """Add test payload files to a successful response."""
203 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
204 'payload1.bin')
205
206
207@faux.success(_BundleTestUpdatePayloadsResponse)
208@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600209@validate.require('build_target.name', 'output_dir')
210@validate.exists('output_dir')
211@validate.validation_complete
212def BundleTestUpdatePayloads(input_proto, output_proto, _config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600213 """Generate minimal update payloads for the build target for testing.
214
215 Args:
216 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600217 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600218 _config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600219 """
220 target = input_proto.build_target.name
221 output_dir = input_proto.output_dir
222 build_root = constants.SOURCE_ROOT
223
224 # Use the first available image to create the update payload.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600225 img_dir = _GetImageDir(build_root, target)
Alex Kleind2bf1462019-10-24 16:37:04 -0600226 if img_dir is None:
227 return None
228
Alex Kleincb541e82019-06-26 15:06:11 -0600229 img_types = [constants.IMAGE_TYPE_TEST, constants.IMAGE_TYPE_DEV,
230 constants.IMAGE_TYPE_BASE]
231 img_names = [constants.IMAGE_TYPE_TO_NAME[t] for t in img_types]
Mike Frysinger66ce4132019-07-17 22:52:52 -0400232 img_paths = [os.path.join(img_dir, x) for x in img_names]
Mike Frysingera552be42018-08-17 14:39:32 -0400233 valid_images = [x for x in img_paths if os.path.exists(x)]
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600234
Alex Kleincb541e82019-06-26 15:06:11 -0600235 if not valid_images:
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600236 cros_build_lib.Die(
237 'Expected to find an image of type among %r for target "%s" '
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600238 'at path %s.', img_types, target, img_dir)
Alex Kleincb541e82019-06-26 15:06:11 -0600239 image = valid_images[0]
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600240
Alex Kleincb541e82019-06-26 15:06:11 -0600241 payloads = artifacts.BundleTestUpdatePayloads(image, output_dir)
242 for payload in payloads:
243 output_proto.artifacts.add().path = payload
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600244
245
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700246def _BundleAutotestFilesResponse(input_proto, output_proto, _config):
247 """Add test autotest files to a successful response."""
248 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
249 'autotest-a.tar.gz')
250
251
252@faux.success(_BundleAutotestFilesResponse)
253@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600254@validate.require('output_dir')
255@validate.exists('output_dir')
256def BundleAutotestFiles(input_proto, output_proto, config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600257 """Tar the autotest files for a build target.
258
259 Args:
260 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600261 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600262 config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600263 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600264 output_dir = input_proto.output_dir
Alex Klein238d8862019-05-07 11:32:46 -0600265 target = input_proto.build_target.name
Alex Kleine21a0952019-08-23 16:08:16 -0600266 chroot = controller_util.ParseChroot(input_proto.chroot)
267
Alex Klein238d8862019-05-07 11:32:46 -0600268 if target:
Alex Kleine21a0952019-08-23 16:08:16 -0600269 sysroot_path = os.path.join('/build', target)
Alex Klein238d8862019-05-07 11:32:46 -0600270 else:
271 # New style call, use chroot and sysroot.
Alex Klein238d8862019-05-07 11:32:46 -0600272 sysroot_path = input_proto.sysroot.path
273 if not sysroot_path:
274 cros_build_lib.Die('sysroot.path is required.')
275
Alex Kleine21a0952019-08-23 16:08:16 -0600276 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein238d8862019-05-07 11:32:46 -0600277
Alex Klein231d2da2019-07-22 16:44:45 -0600278 # TODO(saklein): Switch to the validate_only decorator when legacy handling
279 # is removed.
280 if config.validate_only:
281 return controller.RETURN_CODE_VALID_INPUT
282
Alex Kleine21a0952019-08-23 16:08:16 -0600283 if not sysroot.Exists(chroot=chroot):
Alex Klein238d8862019-05-07 11:32:46 -0600284 cros_build_lib.Die('Sysroot path must exist: %s', sysroot.path)
285
286 try:
287 # Note that this returns the full path to *multiple* tarballs.
Alex Kleine21a0952019-08-23 16:08:16 -0600288 archives = artifacts.BundleAutotestFiles(chroot, sysroot, output_dir)
Alex Klein238d8862019-05-07 11:32:46 -0600289 except artifacts.Error as e:
Alex Klein03890312020-06-30 09:59:50 -0600290 logging.warning(e)
291 return
Alex Klein238d8862019-05-07 11:32:46 -0600292
293 for archive in archives.values():
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600294 output_proto.artifacts.add().path = archive
295
296
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700297def _BundleTastFilesResponse(input_proto, output_proto, _config):
298 """Add test tast files to a successful response."""
299 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
300 'tast_bundles.tar.gz')
301
302
303@faux.success(_BundleTastFilesResponse)
304@faux.empty_error
Alex Kleinb9d810b2019-07-01 12:38:02 -0600305@validate.require('output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600306@validate.exists('output_dir')
307def BundleTastFiles(input_proto, output_proto, config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600308 """Tar the tast files for a build target.
309
310 Args:
311 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600312 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600313 config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600314 """
315 target = input_proto.build_target.name
316 output_dir = input_proto.output_dir
317 build_root = constants.SOURCE_ROOT
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600318
Alex Kleinb9d810b2019-07-01 12:38:02 -0600319 chroot = controller_util.ParseChroot(input_proto.chroot)
320 sysroot_path = input_proto.sysroot.path
321
322 # TODO(saklein) Cleanup legacy handling after it has been switched over.
323 if target:
324 # Legacy handling.
Alex Klein171da612019-08-06 14:00:42 -0600325 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
Alex Kleinb9d810b2019-07-01 12:38:02 -0600326 sysroot_path = os.path.join('/build', target)
327
328 # New handling - chroot & sysroot based.
329 # TODO(saklein) Switch this to the require decorator when legacy is removed.
330 if not sysroot_path:
331 cros_build_lib.Die('sysroot.path is required.')
332
Alex Klein231d2da2019-07-22 16:44:45 -0600333 # TODO(saklein): Switch to the validation_complete decorator when legacy
334 # handling is removed.
335 if config.validate_only:
336 return controller.RETURN_CODE_VALID_INPUT
337
Alex Kleinb9d810b2019-07-01 12:38:02 -0600338 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein231d2da2019-07-22 16:44:45 -0600339 if not sysroot.Exists(chroot=chroot):
Alex Kleinb9d810b2019-07-01 12:38:02 -0600340 cros_build_lib.Die('Sysroot must exist.')
341
342 archive = artifacts.BundleTastFiles(chroot, sysroot, output_dir)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600343
LaMont Jonesb9793cd2020-06-11 08:14:46 -0600344 if archive:
345 output_proto.artifacts.add().path = archive
346 else:
347 logging.warning('Found no tast files for %s.', target)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600348
349
Fergus Dall34d74e12020-09-29 15:52:32 +1000350def BundlePinnedGuestImages(_input_proto, _output_proto, _config):
351 # TODO(crbug/1034529): Remove this endpoint
352 pass
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700353
Fergus Dall34d74e12020-09-29 15:52:32 +1000354def FetchPinnedGuestImageUris(_input_proto, _output_proto, _config):
355 # TODO(crbug/1034529): Remove this endpoint
356 pass
Alex Klein7bf0ecb2019-06-25 10:04:15 -0600357
358
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700359def _BundleFirmwareResponse(input_proto, output_proto, _config):
360 """Add test firmware image files to a successful response."""
361 output_proto.artifacts.add().path = os.path.join(
362 input_proto.output_dir, 'firmware.tar.gz')
363
364
365@faux.success(_BundleFirmwareResponse)
366@faux.empty_error
Michael Mortensen38675192019-06-28 16:52:55 +0000367@validate.require('output_dir', 'sysroot.path')
Alex Klein231d2da2019-07-22 16:44:45 -0600368@validate.exists('output_dir')
369@validate.validation_complete
370def BundleFirmware(input_proto, output_proto, _config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600371 """Tar the firmware images for a build target.
372
373 Args:
374 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600375 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600376 _config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600377 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600378 output_dir = input_proto.output_dir
Michael Mortensen38675192019-06-28 16:52:55 +0000379 chroot = controller_util.ParseChroot(input_proto.chroot)
380 sysroot_path = input_proto.sysroot.path
381 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein231d2da2019-07-22 16:44:45 -0600382
383 if not chroot.exists():
384 cros_build_lib.Die('Chroot does not exist: %s', chroot.path)
385 elif not sysroot.Exists(chroot=chroot):
386 cros_build_lib.Die('Sysroot does not exist: %s',
387 chroot.full_path(sysroot.path))
388
Michael Mortensen38675192019-06-28 16:52:55 +0000389 archive = artifacts.BuildFirmwareArchive(chroot, sysroot, output_dir)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600390
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600391 if archive is None:
392 cros_build_lib.Die(
Michael Mortensen38675192019-06-28 16:52:55 +0000393 'Could not create firmware archive. No firmware found for %s.',
394 sysroot_path)
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600395
Alex Klein231d2da2019-07-22 16:44:45 -0600396 output_proto.artifacts.add().path = archive
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600397
398
Yicheng Liea1181f2020-09-22 11:51:10 -0700399def _BundleFpmcuUnittestsResponse(input_proto, output_proto, _config):
400 """Add fingerprint MCU unittest binaries to a successful response."""
401 output_proto.artifacts.add().path = os.path.join(
402 input_proto.output_dir, 'fpmcu_unittests.tar.gz')
403
404
405@faux.success(_BundleFpmcuUnittestsResponse)
406@faux.empty_error
407@validate.require('output_dir', 'sysroot.path')
408@validate.exists('output_dir')
409@validate.validation_complete
410def BundleFpmcuUnittests(input_proto, output_proto, _config):
411 """Tar the fingerprint MCU unittest binaries for a build target.
412
413 Args:
414 input_proto (BundleRequest): The input proto.
415 output_proto (BundleResponse): The output proto.
416 _config (api_config.ApiConfig): The API call config.
417 """
418 output_dir = input_proto.output_dir
419 chroot = controller_util.ParseChroot(input_proto.chroot)
420 sysroot_path = input_proto.sysroot.path
421 sysroot = sysroot_lib.Sysroot(sysroot_path)
422
423 if not chroot.exists():
424 cros_build_lib.Die('Chroot does not exist: %s', chroot.path)
425 elif not sysroot.Exists(chroot=chroot):
426 cros_build_lib.Die('Sysroot does not exist: %s',
427 chroot.full_path(sysroot.path))
428
429 archive = artifacts.BundleFpmcuUnittests(chroot, sysroot, output_dir)
430
431 if archive is None:
432 logging.warning(
433 'No fpmcu unittests found for %s.', sysroot_path)
434 return
435
436 output_proto.artifacts.add().path = archive
437
438
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700439def _BundleEbuildLogsResponse(input_proto, output_proto, _config):
440 """Add test log files to a successful response."""
441 output_proto.artifacts.add().path = os.path.join(
442 input_proto.output_dir, 'ebuild-logs.tar.gz')
443
444
445@faux.success(_BundleEbuildLogsResponse)
446@faux.empty_error
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600447@validate.exists('output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600448def BundleEbuildLogs(input_proto, output_proto, config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600449 """Tar the ebuild logs for a build target.
450
451 Args:
452 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600453 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600454 config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600455 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600456 output_dir = input_proto.output_dir
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600457 sysroot_path = input_proto.sysroot.path
458 chroot = controller_util.ParseChroot(input_proto.chroot)
Evan Hernandeza478d802019-04-08 15:08:24 -0600459
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600460 # TODO(mmortensen) Cleanup legacy handling after it has been switched over.
461 target = input_proto.build_target.name
462 if target:
463 # Legacy handling.
464 build_root = constants.SOURCE_ROOT
Alex Klein171da612019-08-06 14:00:42 -0600465 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600466 sysroot_path = os.path.join('/build', target)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600467
Alex Klein231d2da2019-07-22 16:44:45 -0600468 # TODO(saklein): Switch to validation_complete decorator after legacy
469 # handling has been cleaned up.
470 if config.validate_only:
471 return controller.RETURN_CODE_VALID_INPUT
472
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600473 sysroot = sysroot_lib.Sysroot(sysroot_path)
474 archive = artifacts.BundleEBuildLogsTarball(chroot, sysroot, output_dir)
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600475 if archive is None:
476 cros_build_lib.Die(
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600477 'Could not create ebuild logs archive. No logs found for %s.',
478 sysroot.path)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600479 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
Alex Klein6504eca2019-04-18 15:37:56 -0600480
481
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700482def _BundleChromeOSConfigResponse(input_proto, output_proto, _config):
483 """Add test config files to a successful response."""
484 output_proto.artifacts.add().path = os.path.join(
485 input_proto.output_dir, 'config.yaml')
486
487
488@faux.success(_BundleChromeOSConfigResponse)
489@faux.empty_error
Andrew Lamb811aead2019-08-12 10:25:05 -0600490@validate.exists('output_dir')
491@validate.validation_complete
492def BundleChromeOSConfig(input_proto, output_proto, _config):
493 """Output the ChromeOS Config payload for a build target.
494
495 Args:
496 input_proto (BundleRequest): The input proto.
497 output_proto (BundleResponse): The output proto.
498 _config (api_config.ApiConfig): The API call config.
499 """
500 output_dir = input_proto.output_dir
Andrew Lamb67bd68f2019-08-15 09:09:15 -0600501 sysroot_path = input_proto.sysroot.path
Andrew Lamb811aead2019-08-12 10:25:05 -0600502 chroot = controller_util.ParseChroot(input_proto.chroot)
503
Andrew Lamb67bd68f2019-08-15 09:09:15 -0600504 # TODO(mmortensen) Cleanup legacy handling after it has been switched over.
505 target = input_proto.build_target.name
506 if target:
507 # Legacy handling.
508 build_root = constants.SOURCE_ROOT
509 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
510 sysroot_path = os.path.join('/build', target)
511
512 sysroot = sysroot_lib.Sysroot(sysroot_path)
Andrew Lamb811aead2019-08-12 10:25:05 -0600513 chromeos_config = artifacts.BundleChromeOSConfig(chroot, sysroot, output_dir)
514 if chromeos_config is None:
515 cros_build_lib.Die(
516 'Could not create ChromeOS Config payload. No config found for %s.',
517 sysroot.path)
518 output_proto.artifacts.add().path = os.path.join(output_dir, chromeos_config)
519
520
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700521def _BundleSimpleChromeArtifactsResponse(input_proto, output_proto, _config):
522 """Add test simple chrome files to a successful response."""
523 output_proto.artifacts.add().path = os.path.join(
524 input_proto.output_dir, 'simple_chrome.txt')
525
526
527@faux.success(_BundleSimpleChromeArtifactsResponse)
528@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600529@validate.require('output_dir', 'sysroot.build_target.name', 'sysroot.path')
530@validate.exists('output_dir')
531@validate.validation_complete
532def BundleSimpleChromeArtifacts(input_proto, output_proto, _config):
Alex Klein2275d692019-04-23 16:04:12 -0600533 """Create the simple chrome artifacts."""
Alex Klein2275d692019-04-23 16:04:12 -0600534 sysroot_path = input_proto.sysroot.path
Alex Klein2275d692019-04-23 16:04:12 -0600535 output_dir = input_proto.output_dir
536
Alex Klein2275d692019-04-23 16:04:12 -0600537 # Build out the argument instances.
Alex Klein26e472b2020-03-10 14:35:01 -0600538 build_target = controller_util.ParseBuildTarget(
539 input_proto.sysroot.build_target)
540 chroot = controller_util.ParseChroot(input_proto.chroot)
Alex Klein2275d692019-04-23 16:04:12 -0600541 # Sysroot.path needs to be the fully qualified path, including the chroot.
542 full_sysroot_path = os.path.join(chroot.path, sysroot_path.lstrip(os.sep))
543 sysroot = sysroot_lib.Sysroot(full_sysroot_path)
544
545 # Quick sanity check that the sysroot exists before we go on.
546 if not sysroot.Exists():
547 cros_build_lib.Die('The sysroot does not exist.')
548
549 try:
550 results = artifacts.BundleSimpleChromeArtifacts(chroot, sysroot,
551 build_target, output_dir)
552 except artifacts.Error as e:
553 cros_build_lib.Die('Error %s raised in BundleSimpleChromeArtifacts: %s',
554 type(e), e)
555
556 for file_name in results:
557 output_proto.artifacts.add().path = file_name
558
559
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700560def _BundleVmFilesResponse(input_proto, output_proto, _config):
561 """Add test vm files to a successful response."""
562 output_proto.artifacts.add().path = os.path.join(
563 input_proto.output_dir, 'f1.tar')
564
565
566@faux.success(_BundleVmFilesResponse)
567@faux.empty_error
Michael Mortensen51f06722019-07-18 09:55:50 -0600568@validate.require('chroot.path', 'test_results_dir', 'output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600569@validate.exists('output_dir')
570@validate.validation_complete
571def BundleVmFiles(input_proto, output_proto, _config):
Alex Klein6504eca2019-04-18 15:37:56 -0600572 """Tar VM disk and memory files.
573
574 Args:
Trent Begin008cade2019-10-31 13:40:59 -0600575 input_proto (BundleVmFilesRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600576 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600577 _config (api_config.ApiConfig): The API call config.
Alex Klein6504eca2019-04-18 15:37:56 -0600578 """
Michael Mortensen51f06722019-07-18 09:55:50 -0600579 chroot = controller_util.ParseChroot(input_proto.chroot)
580 test_results_dir = input_proto.test_results_dir
Alex Klein6504eca2019-04-18 15:37:56 -0600581 output_dir = input_proto.output_dir
582
Michael Mortensen51f06722019-07-18 09:55:50 -0600583 archives = artifacts.BundleVmFiles(
584 chroot, test_results_dir, output_dir)
Alex Klein6504eca2019-04-18 15:37:56 -0600585 for archive in archives:
586 output_proto.artifacts.add().path = archive
Tiancong Wangc4805b72019-06-11 12:12:03 -0700587
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700588def _BundleAFDOGenerationArtifactsResponse(input_proto, output_proto, _config):
589 """Add test tarball AFDO file to a successful response."""
590 output_proto.artifacts.add().path = os.path.join(
591 input_proto.output_dir, 'artifact1')
592
Alex Klein231d2da2019-07-22 16:44:45 -0600593
Tiancong Wang24a3df72019-08-20 15:48:51 -0700594_VALID_ARTIFACT_TYPES = [toolchain_pb2.BENCHMARK_AFDO,
595 toolchain_pb2.ORDERFILE]
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700596@faux.success(_BundleAFDOGenerationArtifactsResponse)
597@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600598@validate.require('build_target.name', 'output_dir')
Tiancong Wang50b80a92019-08-01 14:46:15 -0700599@validate.is_in('artifact_type', _VALID_ARTIFACT_TYPES)
Alex Klein231d2da2019-07-22 16:44:45 -0600600@validate.exists('output_dir')
Tiancong Wang2ade7932019-09-27 14:15:40 -0700601@validate.exists('chroot.chrome_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600602@validate.validation_complete
Tiancong Wang50b80a92019-08-01 14:46:15 -0700603def BundleAFDOGenerationArtifacts(input_proto, output_proto, _config):
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700604 """Generic function for creating tarballs of both AFDO and orderfile.
Tiancong Wangc4805b72019-06-11 12:12:03 -0700605
606 Args:
Tiancong Wang50b80a92019-08-01 14:46:15 -0700607 input_proto (BundleChromeAFDORequest): The input proto.
Tiancong Wangc4805b72019-06-11 12:12:03 -0700608 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600609 _config (api_config.ApiConfig): The API call config.
Tiancong Wangc4805b72019-06-11 12:12:03 -0700610 """
Tiancong Wang2ade7932019-09-27 14:15:40 -0700611 chrome_root = input_proto.chroot.chrome_dir
Tiancong Wangc4805b72019-06-11 12:12:03 -0700612 output_dir = input_proto.output_dir
Tiancong Wang50b80a92019-08-01 14:46:15 -0700613 artifact_type = input_proto.artifact_type
Tiancong Wangc4805b72019-06-11 12:12:03 -0700614
Alex Klein26e472b2020-03-10 14:35:01 -0600615 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
Tiancong Wangc4805b72019-06-11 12:12:03 -0700616 chroot = controller_util.ParseChroot(input_proto.chroot)
617
618 try:
Tiancong Wang24a3df72019-08-20 15:48:51 -0700619 is_orderfile = bool(artifact_type is toolchain_pb2.ORDERFILE)
Tiancong Wang50b80a92019-08-01 14:46:15 -0700620 results = artifacts.BundleAFDOGenerationArtifacts(
Tiancong Wang2ade7932019-09-27 14:15:40 -0700621 is_orderfile, chroot, chrome_root,
Tiancong Wang50b80a92019-08-01 14:46:15 -0700622 build_target, output_dir)
Tiancong Wangc4805b72019-06-11 12:12:03 -0700623 except artifacts.Error as e:
624 cros_build_lib.Die('Error %s raised in BundleSimpleChromeArtifacts: %s',
625 type(e), e)
626
627 for file_name in results:
628 output_proto.artifacts.add().path = file_name
Alex Klein0b1cbfc2019-08-14 10:09:58 -0600629
630
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700631def _ExportCpeReportResponse(input_proto, output_proto, _config):
632 """Add test cpe results to a successful response."""
633 output_proto.artifacts.add().path = os.path.join(
634 input_proto.output_dir, 'cpe_report.txt')
635 output_proto.artifacts.add().path = os.path.join(
636 input_proto.output_dir, 'cpe_warnings.txt')
637
638
639@faux.success(_ExportCpeReportResponse)
640@faux.empty_error
Alex Klein0b1cbfc2019-08-14 10:09:58 -0600641@validate.exists('output_dir')
642def ExportCpeReport(input_proto, output_proto, config):
643 """Export a CPE report.
644
645 Args:
646 input_proto (BundleRequest): The input proto.
647 output_proto (BundleResponse): The output proto.
648 config (api_config.ApiConfig): The API call config.
649 """
650 chroot = controller_util.ParseChroot(input_proto.chroot)
651 output_dir = input_proto.output_dir
652
653 if input_proto.build_target.name:
654 # Legacy handling - use the default sysroot path for the build target.
655 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
656 sysroot = sysroot_lib.Sysroot(build_target.root)
657 elif input_proto.sysroot.path:
658 sysroot = sysroot_lib.Sysroot(input_proto.sysroot.path)
659 else:
660 # TODO(saklein): Switch to validate decorators once legacy handling can be
661 # cleaned up.
662 cros_build_lib.Die('sysroot.path is required.')
663
664 if config.validate_only:
665 return controller.RETURN_CODE_VALID_INPUT
666
667 cpe_result = artifacts.GenerateCpeReport(chroot, sysroot, output_dir)
668
669 output_proto.artifacts.add().path = cpe_result.report
670 output_proto.artifacts.add().path = cpe_result.warnings
Shao-Chuan Leea44dddc2020-10-30 17:16:55 +0900671
672
673def _BundleGceTarballResponse(input_proto, output_proto, _config):
674 """Add artifact tarball to a successful response."""
675 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
676 constants.TEST_IMAGE_GCE_TAR)
677
678
679@faux.success(_BundleGceTarballResponse)
680@faux.empty_error
681@validate.require('build_target.name', 'output_dir')
682@validate.exists('output_dir')
683@validate.validation_complete
684def BundleGceTarball(input_proto, output_proto, _config):
685 """Bundle the test image into a tarball suitable for importing into GCE.
686
687 Args:
688 input_proto (BundleRequest): The input proto.
689 output_proto (BundleResponse): The output proto.
690 _config (api_config.ApiConfig): The API call config.
691 """
692 target = input_proto.build_target.name
693 output_dir = input_proto.output_dir
694 image_dir = _GetImageDir(constants.SOURCE_ROOT, target)
695 if image_dir is None:
696 return None
697
698 tarball = artifacts.BundleGceTarball(output_dir, image_dir)
699 output_proto.artifacts.add().path = tarball