blob: 797d02a9fb96b1adbb7fb9d8aa7d5c39b6b20873 [file] [log] [blame]
Evan Hernandezf388cbf2019-04-01 11:15:23 -06001# Copyright 2019 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Implements ArtifactService."""
6
Chris McDonald1672ddb2021-07-21 11:48:23 -06007import logging
Evan Hernandezf388cbf2019-04-01 11:15:23 -06008import os
George Engelbrechtc9a8e812021-06-16 18:14:17 -06009from typing import Any, NamedTuple
Evan Hernandezf388cbf2019-04-01 11:15:23 -060010
Alex Klein231d2da2019-07-22 16:44:45 -060011from chromite.api import controller
Alex Klein076841b2019-08-29 15:19:39 -060012from chromite.api import faux
Alex Klein2b236722019-06-19 15:44:26 -060013from chromite.api import validate
Alex Klein238d8862019-05-07 11:32:46 -060014from chromite.api.controller import controller_util
George Engelbrechtc9a8e812021-06-16 18:14:17 -060015from chromite.api.controller import image as image_controller
16from chromite.api.controller import sysroot as sysroot_controller
David Wellingc1433c22021-06-25 16:29:48 +000017from chromite.api.controller import test as test_controller
LaMont Jones58362a42021-02-04 17:40:08 -070018from chromite.api.gen.chromite.api import artifacts_pb2
Tiancong Wang24a3df72019-08-20 15:48:51 -070019from chromite.api.gen.chromite.api import toolchain_pb2
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000020from chromite.api.gen.chromiumos import common_pb2
Alex Klein2275d692019-04-23 16:04:12 -060021from chromite.lib import chroot_lib
Evan Hernandezf388cbf2019-04-01 11:15:23 -060022from chromite.lib import constants
23from chromite.lib import cros_build_lib
Alex Klein2275d692019-04-23 16:04:12 -060024from chromite.lib import sysroot_lib
25from chromite.service import artifacts
Greg Edelstondc941072021-08-11 12:32:30 -060026from chromite.service import test
Evan Hernandezf388cbf2019-04-01 11:15:23 -060027
28
George Engelbrechtc9a8e812021-06-16 18:14:17 -060029class RegisteredGet(NamedTuple):
30 """An registered function for calling Get on an artifact type."""
31 output_proto: artifacts_pb2.GetResponse
32 artifact_dict: Any
LaMont Jones0f5171b2021-01-29 12:28:56 -070033
34
George Engelbrechtc9a8e812021-06-16 18:14:17 -060035def ExampleGetResponse(_input_proto, _output_proto, _config):
36 """Give an example GetResponse with a minimal coverage set."""
37 _output_proto = artifacts_pb2.GetResponse(
38 artifacts=common_pb2.UploadedArtifactsByService(
39 image=image_controller.ExampleGetResponse(),
40 sysroot=sysroot_controller.ExampleGetResponse(),
41 ))
42 return controller.RETURN_CODE_SUCCESS
43
44
LaMont Jones0f5171b2021-01-29 12:28:56 -070045@faux.empty_error
George Engelbrechtc9a8e812021-06-16 18:14:17 -060046@faux.success(ExampleGetResponse)
LaMont Jones0f5171b2021-01-29 12:28:56 -070047@validate.exists('result_path.path.path')
48@validate.validation_complete
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000049def Get(input_proto, output_proto, _config):
LaMont Jones0f5171b2021-01-29 12:28:56 -070050 """Get all artifacts.
51
52 Get all artifacts for the build.
53
George Engelbrechtc9a8e812021-06-16 18:14:17 -060054 Note: As the individual artifact_type bundlers are added here, they *must*
55 stop uploading it via the individual bundler function.
LaMont Jones0f5171b2021-01-29 12:28:56 -070056
57 Args:
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000058 input_proto (GetRequest): The input proto.
59 output_proto (GetResponse): The output proto.
LaMont Jones0f5171b2021-01-29 12:28:56 -070060 _config (api_config.ApiConfig): The API call config.
61 """
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000062 output_dir = input_proto.result_path.path.path
63
George Engelbrechtc9a8e812021-06-16 18:14:17 -060064 sysroot = controller_util.ParseSysroot(input_proto.sysroot)
LaMont Jones8b88e9d2021-06-28 16:37:34 -060065 # This endpoint does not currently support any artifacts that are built
66 # without a sysroot being present.
67 if not sysroot.path:
68 return controller.RETURN_CODE_SUCCESS
69
George Engelbrechtc9a8e812021-06-16 18:14:17 -060070 chroot = controller_util.ParseChroot(input_proto.chroot)
71 build_target = controller_util.ParseBuildTarget(
72 input_proto.sysroot.build_target)
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000073
George Engelbrechtc9a8e812021-06-16 18:14:17 -060074 # A list of RegisteredGet tuples (input proto, output proto, get results).
75 get_res_list = [
76 RegisteredGet(
77 output_proto.artifacts.image,
78 image_controller.GetArtifacts(
79 input_proto.artifact_info.image, chroot, sysroot, build_target,
80 output_dir)),
81 RegisteredGet(
82 output_proto.artifacts.sysroot,
83 sysroot_controller.GetArtifacts(
84 input_proto.artifact_info.sysroot, chroot, sysroot, build_target,
David Wellingc1433c22021-06-25 16:29:48 +000085 output_dir)),
86 RegisteredGet(
87 output_proto.artifacts.test,
88 test_controller.GetArtifacts(
Jack Neusc9707c32021-07-23 21:48:54 +000089 input_proto.artifact_info.test, chroot, sysroot, build_target,
90 output_dir)),
George Engelbrechtc9a8e812021-06-16 18:14:17 -060091 ]
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000092
George Engelbrechtc9a8e812021-06-16 18:14:17 -060093 for get_res in get_res_list:
94 for artifact_dict in get_res.artifact_dict:
95 get_res.output_proto.artifacts.add(
96 artifact_type=artifact_dict['type'],
97 paths=[
98 common_pb2.Path(
99 path=x, location=common_pb2.Path.Location.OUTSIDE)
100 for x in artifact_dict['paths']
101 ])
LaMont Jones0f5171b2021-01-29 12:28:56 -0700102 return controller.RETURN_CODE_SUCCESS
103
104
LaMont Jones58362a42021-02-04 17:40:08 -0700105def _BuildSetupResponse(_input_proto, output_proto, _config):
106 """Just return POINTLESS for now."""
107 # All of the artifact types we support claim that the build is POINTLESS.
108 output_proto.build_relevance = artifacts_pb2.BuildSetupResponse.POINTLESS
109
110
111@faux.success(_BuildSetupResponse)
112@faux.empty_error
113@validate.validation_complete
114def BuildSetup(_input_proto, output_proto, _config):
115 """Setup anything needed for building artifacts
116
117 If any artifact types require steps prior to building the package, they go
118 here. For example, see ToolchainService/PrepareForBuild.
119
120 Note: crbug/1034529 introduces this method as a noop. As the individual
121 artifact_type bundlers are added here, they *must* stop uploading it via the
122 individual bundler function.
123
124 Args:
125 _input_proto (GetRequest): The input proto.
126 output_proto (GetResponse): The output proto.
127 _config (api_config.ApiConfig): The API call config.
128 """
129 # If any artifact_type says "NEEDED", the return is NEEDED.
130 # Otherwise, if any artifact_type says "UNKNOWN", the return is UNKNOWN.
131 # Otherwise, the return is POINTLESS.
132 output_proto.build_relevance = artifacts_pb2.BuildSetupResponse.POINTLESS
133 return controller.RETURN_CODE_SUCCESS
134
135
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600136def _GetImageDir(build_root, target):
137 """Return path containing images for the given build target.
138
Alex Kleine2612a02019-04-18 13:51:06 -0600139 TODO(saklein) Expand image_lib.GetLatestImageLink to support this use case.
140
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600141 Args:
142 build_root (str): Path to checkout where build occurs.
143 target (str): Name of the build target.
144
145 Returns:
Alex Kleind2bf1462019-10-24 16:37:04 -0600146 Path to the latest directory containing target images or None.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600147 """
148 image_dir = os.path.join(build_root, 'src/build/images', target, 'latest')
149 if not os.path.exists(image_dir):
Alex Kleind2bf1462019-10-24 16:37:04 -0600150 logging.warning('Expected to find image output for target %s at %s, but '
151 'path does not exist', target, image_dir)
152 return None
153
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600154 return image_dir
155
156
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700157def _BundleImageArchivesResponse(input_proto, output_proto, _config):
158 """Add artifact paths to a successful response."""
159 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
160 'path0.tar.xz')
161 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
162 'path1.tar.xz')
163
164
165@faux.success(_BundleImageArchivesResponse)
166@faux.empty_error
Alex Kleind91e95a2019-09-17 10:39:02 -0600167@validate.require('build_target.name')
168@validate.exists('output_dir')
169@validate.validation_complete
170def BundleImageArchives(input_proto, output_proto, _config):
171 """Create a .tar.xz archive for each image that has been created."""
172 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
173 output_dir = input_proto.output_dir
174 image_dir = _GetImageDir(constants.SOURCE_ROOT, build_target.name)
Alex Kleind2bf1462019-10-24 16:37:04 -0600175 if image_dir is None:
176 return
Alex Kleind91e95a2019-09-17 10:39:02 -0600177
178 archives = artifacts.ArchiveImages(image_dir, output_dir)
179
180 for archive in archives:
181 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
182
183
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700184def _BundleImageZipResponse(input_proto, output_proto, _config):
185 """Add artifact zip files to a successful response."""
186 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
187 'image.zip')
188
189
190@faux.success(_BundleImageZipResponse)
191@faux.empty_error
Michael Mortensen01910922019-07-24 14:48:10 -0600192@validate.require('build_target.name', 'output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600193@validate.exists('output_dir')
194@validate.validation_complete
195def BundleImageZip(input_proto, output_proto, _config):
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600196 """Bundle image.zip.
197
198 Args:
199 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600200 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600201 _config (api_config.ApiConfig): The API call config.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600202 """
203 target = input_proto.build_target.name
204 output_dir = input_proto.output_dir
205 image_dir = _GetImageDir(constants.SOURCE_ROOT, target)
Alex Kleind2bf1462019-10-24 16:37:04 -0600206 if image_dir is None:
207 return None
Alex Klein231d2da2019-07-22 16:44:45 -0600208
Michael Mortensen01910922019-07-24 14:48:10 -0600209 archive = artifacts.BundleImageZip(output_dir, image_dir)
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600210 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
211
212
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700213def _BundleTestUpdatePayloadsResponse(input_proto, output_proto, _config):
214 """Add test payload files to a successful response."""
215 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
216 'payload1.bin')
217
218
219@faux.success(_BundleTestUpdatePayloadsResponse)
220@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600221@validate.require('build_target.name', 'output_dir')
222@validate.exists('output_dir')
223@validate.validation_complete
224def BundleTestUpdatePayloads(input_proto, output_proto, _config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600225 """Generate minimal update payloads for the build target for testing.
226
227 Args:
228 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600229 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600230 _config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600231 """
232 target = input_proto.build_target.name
233 output_dir = input_proto.output_dir
234 build_root = constants.SOURCE_ROOT
235
236 # Use the first available image to create the update payload.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600237 img_dir = _GetImageDir(build_root, target)
Alex Kleind2bf1462019-10-24 16:37:04 -0600238 if img_dir is None:
239 return None
240
Alex Kleincb541e82019-06-26 15:06:11 -0600241 img_types = [constants.IMAGE_TYPE_TEST, constants.IMAGE_TYPE_DEV,
242 constants.IMAGE_TYPE_BASE]
243 img_names = [constants.IMAGE_TYPE_TO_NAME[t] for t in img_types]
Mike Frysinger66ce4132019-07-17 22:52:52 -0400244 img_paths = [os.path.join(img_dir, x) for x in img_names]
Mike Frysingera552be42018-08-17 14:39:32 -0400245 valid_images = [x for x in img_paths if os.path.exists(x)]
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600246
Alex Kleincb541e82019-06-26 15:06:11 -0600247 if not valid_images:
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600248 cros_build_lib.Die(
249 'Expected to find an image of type among %r for target "%s" '
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600250 'at path %s.', img_types, target, img_dir)
Alex Kleincb541e82019-06-26 15:06:11 -0600251 image = valid_images[0]
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600252
Alex Kleincb541e82019-06-26 15:06:11 -0600253 payloads = artifacts.BundleTestUpdatePayloads(image, output_dir)
254 for payload in payloads:
255 output_proto.artifacts.add().path = payload
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600256
257
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700258def _BundleAutotestFilesResponse(input_proto, output_proto, _config):
259 """Add test autotest files to a successful response."""
260 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
261 'autotest-a.tar.gz')
262
263
264@faux.success(_BundleAutotestFilesResponse)
265@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600266@validate.require('output_dir')
267@validate.exists('output_dir')
268def BundleAutotestFiles(input_proto, output_proto, config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600269 """Tar the autotest files for a build target.
270
271 Args:
272 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600273 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600274 config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600275 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600276 output_dir = input_proto.output_dir
Alex Klein238d8862019-05-07 11:32:46 -0600277 target = input_proto.build_target.name
Alex Kleine21a0952019-08-23 16:08:16 -0600278 chroot = controller_util.ParseChroot(input_proto.chroot)
279
Alex Klein238d8862019-05-07 11:32:46 -0600280 if target:
Alex Kleine21a0952019-08-23 16:08:16 -0600281 sysroot_path = os.path.join('/build', target)
Alex Klein238d8862019-05-07 11:32:46 -0600282 else:
283 # New style call, use chroot and sysroot.
Alex Klein238d8862019-05-07 11:32:46 -0600284 sysroot_path = input_proto.sysroot.path
285 if not sysroot_path:
286 cros_build_lib.Die('sysroot.path is required.')
287
Alex Kleine21a0952019-08-23 16:08:16 -0600288 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein238d8862019-05-07 11:32:46 -0600289
Alex Klein231d2da2019-07-22 16:44:45 -0600290 # TODO(saklein): Switch to the validate_only decorator when legacy handling
291 # is removed.
292 if config.validate_only:
293 return controller.RETURN_CODE_VALID_INPUT
294
Alex Kleine21a0952019-08-23 16:08:16 -0600295 if not sysroot.Exists(chroot=chroot):
Alex Klein238d8862019-05-07 11:32:46 -0600296 cros_build_lib.Die('Sysroot path must exist: %s', sysroot.path)
297
298 try:
299 # Note that this returns the full path to *multiple* tarballs.
Alex Kleine21a0952019-08-23 16:08:16 -0600300 archives = artifacts.BundleAutotestFiles(chroot, sysroot, output_dir)
Alex Klein238d8862019-05-07 11:32:46 -0600301 except artifacts.Error as e:
Alex Klein03890312020-06-30 09:59:50 -0600302 logging.warning(e)
303 return
Alex Klein238d8862019-05-07 11:32:46 -0600304
305 for archive in archives.values():
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600306 output_proto.artifacts.add().path = archive
307
308
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700309def _BundleTastFilesResponse(input_proto, output_proto, _config):
310 """Add test tast files to a successful response."""
311 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
312 'tast_bundles.tar.gz')
313
314
315@faux.success(_BundleTastFilesResponse)
316@faux.empty_error
Alex Kleinb9d810b2019-07-01 12:38:02 -0600317@validate.require('output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600318@validate.exists('output_dir')
319def BundleTastFiles(input_proto, output_proto, config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600320 """Tar the tast files for a build target.
321
322 Args:
323 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600324 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600325 config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600326 """
327 target = input_proto.build_target.name
328 output_dir = input_proto.output_dir
329 build_root = constants.SOURCE_ROOT
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600330
Alex Kleinb9d810b2019-07-01 12:38:02 -0600331 chroot = controller_util.ParseChroot(input_proto.chroot)
332 sysroot_path = input_proto.sysroot.path
333
334 # TODO(saklein) Cleanup legacy handling after it has been switched over.
335 if target:
336 # Legacy handling.
Alex Klein171da612019-08-06 14:00:42 -0600337 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
Alex Kleinb9d810b2019-07-01 12:38:02 -0600338 sysroot_path = os.path.join('/build', target)
339
340 # New handling - chroot & sysroot based.
341 # TODO(saklein) Switch this to the require decorator when legacy is removed.
342 if not sysroot_path:
343 cros_build_lib.Die('sysroot.path is required.')
344
Alex Klein231d2da2019-07-22 16:44:45 -0600345 # TODO(saklein): Switch to the validation_complete decorator when legacy
346 # handling is removed.
347 if config.validate_only:
348 return controller.RETURN_CODE_VALID_INPUT
349
Alex Kleinb9d810b2019-07-01 12:38:02 -0600350 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein231d2da2019-07-22 16:44:45 -0600351 if not sysroot.Exists(chroot=chroot):
Alex Kleinb9d810b2019-07-01 12:38:02 -0600352 cros_build_lib.Die('Sysroot must exist.')
353
354 archive = artifacts.BundleTastFiles(chroot, sysroot, output_dir)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600355
LaMont Jonesb9793cd2020-06-11 08:14:46 -0600356 if archive:
357 output_proto.artifacts.add().path = archive
358 else:
359 logging.warning('Found no tast files for %s.', target)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600360
361
Fergus Dall34d74e12020-09-29 15:52:32 +1000362def BundlePinnedGuestImages(_input_proto, _output_proto, _config):
363 # TODO(crbug/1034529): Remove this endpoint
364 pass
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700365
Fergus Dall34d74e12020-09-29 15:52:32 +1000366def FetchPinnedGuestImageUris(_input_proto, _output_proto, _config):
367 # TODO(crbug/1034529): Remove this endpoint
368 pass
Alex Klein7bf0ecb2019-06-25 10:04:15 -0600369
370
Greg Edelstondc941072021-08-11 12:32:30 -0600371def _FetchMetadataResponse(_input_proto, output_proto, _config):
372 """Populate the output_proto with sample data."""
373 for fp in ('/metadata/foo.txt', '/metadata/bar.jsonproto'):
374 output_proto.filepaths.add(path=common_pb2.Path(
375 path=fp, location=common_pb2.Path.OUTSIDE))
376 return controller.RETURN_CODE_SUCCESS
377
378
379@faux.success(_FetchMetadataResponse)
380@faux.empty_error
381@validate.exists('chroot.path')
382@validate.require('sysroot.path')
383@validate.validation_complete
384def FetchMetadata(input_proto, output_proto, _config):
385 """FetchMetadata returns the paths to all build/test metadata files.
386
387 This implements ArtifactsService.FetchMetadata.
388
389 Args:
390 input_proto (FetchMetadataRequest): The input proto.
391 output_proto (FetchMetadataResponse): The output proto.
392 config (api_config.ApiConfig): The API call config.
393 """
394 chroot = controller_util.ParseChroot(input_proto.chroot)
395 sysroot = controller_util.ParseSysroot(input_proto.sysroot)
396 for path in test.FindAllMetadataFiles(chroot, sysroot):
397 output_proto.filepaths.add(
398 path=common_pb2.Path(path=path, location=common_pb2.Path.OUTSIDE))
399 return controller.RETURN_CODE_SUCCESS
400
401
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700402def _BundleFirmwareResponse(input_proto, output_proto, _config):
403 """Add test firmware image files to a successful response."""
404 output_proto.artifacts.add().path = os.path.join(
405 input_proto.output_dir, 'firmware.tar.gz')
406
407
408@faux.success(_BundleFirmwareResponse)
409@faux.empty_error
Michael Mortensen38675192019-06-28 16:52:55 +0000410@validate.require('output_dir', 'sysroot.path')
Alex Klein231d2da2019-07-22 16:44:45 -0600411@validate.exists('output_dir')
412@validate.validation_complete
413def BundleFirmware(input_proto, output_proto, _config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600414 """Tar the firmware images for a build target.
415
416 Args:
417 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600418 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600419 _config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600420 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600421 output_dir = input_proto.output_dir
Michael Mortensen38675192019-06-28 16:52:55 +0000422 chroot = controller_util.ParseChroot(input_proto.chroot)
423 sysroot_path = input_proto.sysroot.path
424 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein231d2da2019-07-22 16:44:45 -0600425
426 if not chroot.exists():
427 cros_build_lib.Die('Chroot does not exist: %s', chroot.path)
428 elif not sysroot.Exists(chroot=chroot):
429 cros_build_lib.Die('Sysroot does not exist: %s',
430 chroot.full_path(sysroot.path))
431
Michael Mortensen38675192019-06-28 16:52:55 +0000432 archive = artifacts.BuildFirmwareArchive(chroot, sysroot, output_dir)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600433
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600434 if archive is None:
435 cros_build_lib.Die(
Michael Mortensen38675192019-06-28 16:52:55 +0000436 'Could not create firmware archive. No firmware found for %s.',
437 sysroot_path)
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600438
Alex Klein231d2da2019-07-22 16:44:45 -0600439 output_proto.artifacts.add().path = archive
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600440
441
Yicheng Liea1181f2020-09-22 11:51:10 -0700442def _BundleFpmcuUnittestsResponse(input_proto, output_proto, _config):
443 """Add fingerprint MCU unittest binaries to a successful response."""
444 output_proto.artifacts.add().path = os.path.join(
445 input_proto.output_dir, 'fpmcu_unittests.tar.gz')
446
447
448@faux.success(_BundleFpmcuUnittestsResponse)
449@faux.empty_error
450@validate.require('output_dir', 'sysroot.path')
451@validate.exists('output_dir')
452@validate.validation_complete
453def BundleFpmcuUnittests(input_proto, output_proto, _config):
454 """Tar the fingerprint MCU unittest binaries for a build target.
455
456 Args:
457 input_proto (BundleRequest): The input proto.
458 output_proto (BundleResponse): The output proto.
459 _config (api_config.ApiConfig): The API call config.
460 """
461 output_dir = input_proto.output_dir
462 chroot = controller_util.ParseChroot(input_proto.chroot)
463 sysroot_path = input_proto.sysroot.path
464 sysroot = sysroot_lib.Sysroot(sysroot_path)
465
466 if not chroot.exists():
467 cros_build_lib.Die('Chroot does not exist: %s', chroot.path)
468 elif not sysroot.Exists(chroot=chroot):
469 cros_build_lib.Die('Sysroot does not exist: %s',
470 chroot.full_path(sysroot.path))
471
472 archive = artifacts.BundleFpmcuUnittests(chroot, sysroot, output_dir)
473
474 if archive is None:
475 logging.warning(
476 'No fpmcu unittests found for %s.', sysroot_path)
477 return
478
479 output_proto.artifacts.add().path = archive
480
481
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700482def _BundleEbuildLogsResponse(input_proto, output_proto, _config):
483 """Add test log files to a successful response."""
484 output_proto.artifacts.add().path = os.path.join(
485 input_proto.output_dir, 'ebuild-logs.tar.gz')
486
487
488@faux.success(_BundleEbuildLogsResponse)
489@faux.empty_error
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600490@validate.exists('output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600491def BundleEbuildLogs(input_proto, output_proto, config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600492 """Tar the ebuild logs for a build target.
493
494 Args:
495 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600496 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600497 config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600498 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600499 output_dir = input_proto.output_dir
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600500 sysroot_path = input_proto.sysroot.path
501 chroot = controller_util.ParseChroot(input_proto.chroot)
Evan Hernandeza478d802019-04-08 15:08:24 -0600502
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600503 # TODO(mmortensen) Cleanup legacy handling after it has been switched over.
504 target = input_proto.build_target.name
505 if target:
506 # Legacy handling.
507 build_root = constants.SOURCE_ROOT
Alex Klein171da612019-08-06 14:00:42 -0600508 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600509 sysroot_path = os.path.join('/build', target)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600510
Alex Klein231d2da2019-07-22 16:44:45 -0600511 # TODO(saklein): Switch to validation_complete decorator after legacy
512 # handling has been cleaned up.
513 if config.validate_only:
514 return controller.RETURN_CODE_VALID_INPUT
515
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600516 sysroot = sysroot_lib.Sysroot(sysroot_path)
517 archive = artifacts.BundleEBuildLogsTarball(chroot, sysroot, output_dir)
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600518 if archive is None:
519 cros_build_lib.Die(
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600520 'Could not create ebuild logs archive. No logs found for %s.',
521 sysroot.path)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600522 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
Alex Klein6504eca2019-04-18 15:37:56 -0600523
524
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700525def _BundleChromeOSConfigResponse(input_proto, output_proto, _config):
526 """Add test config files to a successful response."""
527 output_proto.artifacts.add().path = os.path.join(
528 input_proto.output_dir, 'config.yaml')
529
530
531@faux.success(_BundleChromeOSConfigResponse)
532@faux.empty_error
Andrew Lamb811aead2019-08-12 10:25:05 -0600533@validate.exists('output_dir')
534@validate.validation_complete
535def BundleChromeOSConfig(input_proto, output_proto, _config):
536 """Output the ChromeOS Config payload for a build target.
537
538 Args:
539 input_proto (BundleRequest): The input proto.
540 output_proto (BundleResponse): The output proto.
541 _config (api_config.ApiConfig): The API call config.
542 """
543 output_dir = input_proto.output_dir
Andrew Lamb67bd68f2019-08-15 09:09:15 -0600544 sysroot_path = input_proto.sysroot.path
Andrew Lamb811aead2019-08-12 10:25:05 -0600545 chroot = controller_util.ParseChroot(input_proto.chroot)
546
Andrew Lamb67bd68f2019-08-15 09:09:15 -0600547 # TODO(mmortensen) Cleanup legacy handling after it has been switched over.
548 target = input_proto.build_target.name
549 if target:
550 # Legacy handling.
551 build_root = constants.SOURCE_ROOT
552 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
553 sysroot_path = os.path.join('/build', target)
554
555 sysroot = sysroot_lib.Sysroot(sysroot_path)
Andrew Lamb811aead2019-08-12 10:25:05 -0600556 chromeos_config = artifacts.BundleChromeOSConfig(chroot, sysroot, output_dir)
557 if chromeos_config is None:
558 cros_build_lib.Die(
559 'Could not create ChromeOS Config payload. No config found for %s.',
560 sysroot.path)
561 output_proto.artifacts.add().path = os.path.join(output_dir, chromeos_config)
562
563
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700564def _BundleSimpleChromeArtifactsResponse(input_proto, output_proto, _config):
565 """Add test simple chrome files to a successful response."""
566 output_proto.artifacts.add().path = os.path.join(
567 input_proto.output_dir, 'simple_chrome.txt')
568
569
570@faux.success(_BundleSimpleChromeArtifactsResponse)
571@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600572@validate.require('output_dir', 'sysroot.build_target.name', 'sysroot.path')
573@validate.exists('output_dir')
574@validate.validation_complete
575def BundleSimpleChromeArtifacts(input_proto, output_proto, _config):
Alex Klein2275d692019-04-23 16:04:12 -0600576 """Create the simple chrome artifacts."""
Alex Klein2275d692019-04-23 16:04:12 -0600577 sysroot_path = input_proto.sysroot.path
Alex Klein2275d692019-04-23 16:04:12 -0600578 output_dir = input_proto.output_dir
579
Alex Klein2275d692019-04-23 16:04:12 -0600580 # Build out the argument instances.
Alex Klein26e472b2020-03-10 14:35:01 -0600581 build_target = controller_util.ParseBuildTarget(
582 input_proto.sysroot.build_target)
583 chroot = controller_util.ParseChroot(input_proto.chroot)
Alex Klein2275d692019-04-23 16:04:12 -0600584 # Sysroot.path needs to be the fully qualified path, including the chroot.
585 full_sysroot_path = os.path.join(chroot.path, sysroot_path.lstrip(os.sep))
586 sysroot = sysroot_lib.Sysroot(full_sysroot_path)
587
588 # Quick sanity check that the sysroot exists before we go on.
589 if not sysroot.Exists():
590 cros_build_lib.Die('The sysroot does not exist.')
591
592 try:
593 results = artifacts.BundleSimpleChromeArtifacts(chroot, sysroot,
594 build_target, output_dir)
595 except artifacts.Error as e:
596 cros_build_lib.Die('Error %s raised in BundleSimpleChromeArtifacts: %s',
597 type(e), e)
598
599 for file_name in results:
600 output_proto.artifacts.add().path = file_name
601
602
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700603def _BundleVmFilesResponse(input_proto, output_proto, _config):
604 """Add test vm files to a successful response."""
605 output_proto.artifacts.add().path = os.path.join(
606 input_proto.output_dir, 'f1.tar')
607
608
609@faux.success(_BundleVmFilesResponse)
610@faux.empty_error
Michael Mortensen51f06722019-07-18 09:55:50 -0600611@validate.require('chroot.path', 'test_results_dir', 'output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600612@validate.exists('output_dir')
613@validate.validation_complete
614def BundleVmFiles(input_proto, output_proto, _config):
Alex Klein6504eca2019-04-18 15:37:56 -0600615 """Tar VM disk and memory files.
616
617 Args:
Trent Begin008cade2019-10-31 13:40:59 -0600618 input_proto (BundleVmFilesRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600619 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600620 _config (api_config.ApiConfig): The API call config.
Alex Klein6504eca2019-04-18 15:37:56 -0600621 """
Michael Mortensen51f06722019-07-18 09:55:50 -0600622 chroot = controller_util.ParseChroot(input_proto.chroot)
623 test_results_dir = input_proto.test_results_dir
Alex Klein6504eca2019-04-18 15:37:56 -0600624 output_dir = input_proto.output_dir
625
Michael Mortensen51f06722019-07-18 09:55:50 -0600626 archives = artifacts.BundleVmFiles(
627 chroot, test_results_dir, output_dir)
Alex Klein6504eca2019-04-18 15:37:56 -0600628 for archive in archives:
629 output_proto.artifacts.add().path = archive
Tiancong Wangc4805b72019-06-11 12:12:03 -0700630
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700631def _BundleAFDOGenerationArtifactsResponse(input_proto, output_proto, _config):
632 """Add test tarball AFDO file to a successful response."""
633 output_proto.artifacts.add().path = os.path.join(
634 input_proto.output_dir, 'artifact1')
635
Alex Klein231d2da2019-07-22 16:44:45 -0600636
Tiancong Wang24a3df72019-08-20 15:48:51 -0700637_VALID_ARTIFACT_TYPES = [toolchain_pb2.BENCHMARK_AFDO,
638 toolchain_pb2.ORDERFILE]
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700639@faux.success(_BundleAFDOGenerationArtifactsResponse)
640@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600641@validate.require('build_target.name', 'output_dir')
Tiancong Wang50b80a92019-08-01 14:46:15 -0700642@validate.is_in('artifact_type', _VALID_ARTIFACT_TYPES)
Alex Klein231d2da2019-07-22 16:44:45 -0600643@validate.exists('output_dir')
Tiancong Wang2ade7932019-09-27 14:15:40 -0700644@validate.exists('chroot.chrome_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600645@validate.validation_complete
Tiancong Wang50b80a92019-08-01 14:46:15 -0700646def BundleAFDOGenerationArtifacts(input_proto, output_proto, _config):
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700647 """Generic function for creating tarballs of both AFDO and orderfile.
Tiancong Wangc4805b72019-06-11 12:12:03 -0700648
649 Args:
Tiancong Wang50b80a92019-08-01 14:46:15 -0700650 input_proto (BundleChromeAFDORequest): The input proto.
Tiancong Wangc4805b72019-06-11 12:12:03 -0700651 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600652 _config (api_config.ApiConfig): The API call config.
Tiancong Wangc4805b72019-06-11 12:12:03 -0700653 """
Tiancong Wang2ade7932019-09-27 14:15:40 -0700654 chrome_root = input_proto.chroot.chrome_dir
Tiancong Wangc4805b72019-06-11 12:12:03 -0700655 output_dir = input_proto.output_dir
Tiancong Wang50b80a92019-08-01 14:46:15 -0700656 artifact_type = input_proto.artifact_type
Tiancong Wangc4805b72019-06-11 12:12:03 -0700657
Alex Klein26e472b2020-03-10 14:35:01 -0600658 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
Tiancong Wangc4805b72019-06-11 12:12:03 -0700659 chroot = controller_util.ParseChroot(input_proto.chroot)
660
661 try:
Tiancong Wang24a3df72019-08-20 15:48:51 -0700662 is_orderfile = bool(artifact_type is toolchain_pb2.ORDERFILE)
Tiancong Wang50b80a92019-08-01 14:46:15 -0700663 results = artifacts.BundleAFDOGenerationArtifacts(
Tiancong Wang2ade7932019-09-27 14:15:40 -0700664 is_orderfile, chroot, chrome_root,
Tiancong Wang50b80a92019-08-01 14:46:15 -0700665 build_target, output_dir)
Tiancong Wangc4805b72019-06-11 12:12:03 -0700666 except artifacts.Error as e:
667 cros_build_lib.Die('Error %s raised in BundleSimpleChromeArtifacts: %s',
668 type(e), e)
669
670 for file_name in results:
671 output_proto.artifacts.add().path = file_name
Alex Klein0b1cbfc2019-08-14 10:09:58 -0600672
673
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700674def _ExportCpeReportResponse(input_proto, output_proto, _config):
675 """Add test cpe results to a successful response."""
676 output_proto.artifacts.add().path = os.path.join(
677 input_proto.output_dir, 'cpe_report.txt')
678 output_proto.artifacts.add().path = os.path.join(
679 input_proto.output_dir, 'cpe_warnings.txt')
680
681
682@faux.success(_ExportCpeReportResponse)
683@faux.empty_error
Alex Klein0b1cbfc2019-08-14 10:09:58 -0600684@validate.exists('output_dir')
685def ExportCpeReport(input_proto, output_proto, config):
686 """Export a CPE report.
687
688 Args:
689 input_proto (BundleRequest): The input proto.
690 output_proto (BundleResponse): The output proto.
691 config (api_config.ApiConfig): The API call config.
692 """
693 chroot = controller_util.ParseChroot(input_proto.chroot)
694 output_dir = input_proto.output_dir
695
696 if input_proto.build_target.name:
697 # Legacy handling - use the default sysroot path for the build target.
698 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
699 sysroot = sysroot_lib.Sysroot(build_target.root)
700 elif input_proto.sysroot.path:
701 sysroot = sysroot_lib.Sysroot(input_proto.sysroot.path)
702 else:
703 # TODO(saklein): Switch to validate decorators once legacy handling can be
704 # cleaned up.
705 cros_build_lib.Die('sysroot.path is required.')
706
707 if config.validate_only:
708 return controller.RETURN_CODE_VALID_INPUT
709
710 cpe_result = artifacts.GenerateCpeReport(chroot, sysroot, output_dir)
711
712 output_proto.artifacts.add().path = cpe_result.report
713 output_proto.artifacts.add().path = cpe_result.warnings
Shao-Chuan Leea44dddc2020-10-30 17:16:55 +0900714
715
716def _BundleGceTarballResponse(input_proto, output_proto, _config):
717 """Add artifact tarball to a successful response."""
718 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
719 constants.TEST_IMAGE_GCE_TAR)
720
721
722@faux.success(_BundleGceTarballResponse)
723@faux.empty_error
724@validate.require('build_target.name', 'output_dir')
725@validate.exists('output_dir')
726@validate.validation_complete
727def BundleGceTarball(input_proto, output_proto, _config):
728 """Bundle the test image into a tarball suitable for importing into GCE.
729
730 Args:
731 input_proto (BundleRequest): The input proto.
732 output_proto (BundleResponse): The output proto.
733 _config (api_config.ApiConfig): The API call config.
734 """
735 target = input_proto.build_target.name
736 output_dir = input_proto.output_dir
737 image_dir = _GetImageDir(constants.SOURCE_ROOT, target)
738 if image_dir is None:
739 return None
740
741 tarball = artifacts.BundleGceTarball(output_dir, image_dir)
742 output_proto.artifacts.add().path = tarball