blob: f46bb4fadc1e8e18258a43ac11d52610c991c34e [file] [log] [blame]
Evan Hernandezf388cbf2019-04-01 11:15:23 -06001# Copyright 2019 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Implements ArtifactService."""
6
Evan Hernandezf388cbf2019-04-01 11:15:23 -06007import os
George Engelbrechtc9a8e812021-06-16 18:14:17 -06008from typing import Any, NamedTuple
Evan Hernandezf388cbf2019-04-01 11:15:23 -06009
Alex Klein231d2da2019-07-22 16:44:45 -060010from chromite.api import controller
Alex Klein076841b2019-08-29 15:19:39 -060011from chromite.api import faux
Alex Klein2b236722019-06-19 15:44:26 -060012from chromite.api import validate
Alex Klein238d8862019-05-07 11:32:46 -060013from chromite.api.controller import controller_util
George Engelbrechtc9a8e812021-06-16 18:14:17 -060014from chromite.api.controller import image as image_controller
15from chromite.api.controller import sysroot as sysroot_controller
David Wellingc1433c22021-06-25 16:29:48 +000016from chromite.api.controller import test as test_controller
LaMont Jones58362a42021-02-04 17:40:08 -070017from chromite.api.gen.chromite.api import artifacts_pb2
Tiancong Wang24a3df72019-08-20 15:48:51 -070018from chromite.api.gen.chromite.api import toolchain_pb2
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000019from chromite.api.gen.chromiumos import common_pb2
Alex Klein2275d692019-04-23 16:04:12 -060020from chromite.lib import chroot_lib
Evan Hernandezf388cbf2019-04-01 11:15:23 -060021from chromite.lib import constants
22from chromite.lib import cros_build_lib
Evan Hernandezde445982019-04-22 13:42:34 -060023from chromite.lib import cros_logging as logging
Alex Klein2275d692019-04-23 16:04:12 -060024from chromite.lib import sysroot_lib
25from chromite.service import artifacts
Evan Hernandezf388cbf2019-04-01 11:15:23 -060026
27
George Engelbrechtc9a8e812021-06-16 18:14:17 -060028class RegisteredGet(NamedTuple):
29 """An registered function for calling Get on an artifact type."""
30 output_proto: artifacts_pb2.GetResponse
31 artifact_dict: Any
LaMont Jones0f5171b2021-01-29 12:28:56 -070032
33
George Engelbrechtc9a8e812021-06-16 18:14:17 -060034def ExampleGetResponse(_input_proto, _output_proto, _config):
35 """Give an example GetResponse with a minimal coverage set."""
36 _output_proto = artifacts_pb2.GetResponse(
37 artifacts=common_pb2.UploadedArtifactsByService(
38 image=image_controller.ExampleGetResponse(),
39 sysroot=sysroot_controller.ExampleGetResponse(),
40 ))
41 return controller.RETURN_CODE_SUCCESS
42
43
LaMont Jones0f5171b2021-01-29 12:28:56 -070044@faux.empty_error
George Engelbrechtc9a8e812021-06-16 18:14:17 -060045@faux.success(ExampleGetResponse)
LaMont Jones0f5171b2021-01-29 12:28:56 -070046@validate.exists('result_path.path.path')
47@validate.validation_complete
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000048def Get(input_proto, output_proto, _config):
LaMont Jones0f5171b2021-01-29 12:28:56 -070049 """Get all artifacts.
50
51 Get all artifacts for the build.
52
George Engelbrechtc9a8e812021-06-16 18:14:17 -060053 Note: As the individual artifact_type bundlers are added here, they *must*
54 stop uploading it via the individual bundler function.
LaMont Jones0f5171b2021-01-29 12:28:56 -070055
56 Args:
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000057 input_proto (GetRequest): The input proto.
58 output_proto (GetResponse): The output proto.
LaMont Jones0f5171b2021-01-29 12:28:56 -070059 _config (api_config.ApiConfig): The API call config.
60 """
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000061 output_dir = input_proto.result_path.path.path
62
George Engelbrechtc9a8e812021-06-16 18:14:17 -060063 sysroot = controller_util.ParseSysroot(input_proto.sysroot)
LaMont Jones8b88e9d2021-06-28 16:37:34 -060064 # This endpoint does not currently support any artifacts that are built
65 # without a sysroot being present.
66 if not sysroot.path:
67 return controller.RETURN_CODE_SUCCESS
68
George Engelbrechtc9a8e812021-06-16 18:14:17 -060069 chroot = controller_util.ParseChroot(input_proto.chroot)
70 build_target = controller_util.ParseBuildTarget(
71 input_proto.sysroot.build_target)
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000072
George Engelbrechtc9a8e812021-06-16 18:14:17 -060073 # A list of RegisteredGet tuples (input proto, output proto, get results).
74 get_res_list = [
75 RegisteredGet(
76 output_proto.artifacts.image,
77 image_controller.GetArtifacts(
78 input_proto.artifact_info.image, chroot, sysroot, build_target,
79 output_dir)),
80 RegisteredGet(
81 output_proto.artifacts.sysroot,
82 sysroot_controller.GetArtifacts(
83 input_proto.artifact_info.sysroot, chroot, sysroot, build_target,
David Wellingc1433c22021-06-25 16:29:48 +000084 output_dir)),
85 RegisteredGet(
86 output_proto.artifacts.test,
87 test_controller.GetArtifacts(
88 input_proto.artifact_info.test, chroot, sysroot, output_dir)),
George Engelbrechtc9a8e812021-06-16 18:14:17 -060089 ]
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000090
George Engelbrechtc9a8e812021-06-16 18:14:17 -060091 for get_res in get_res_list:
92 for artifact_dict in get_res.artifact_dict:
93 get_res.output_proto.artifacts.add(
94 artifact_type=artifact_dict['type'],
95 paths=[
96 common_pb2.Path(
97 path=x, location=common_pb2.Path.Location.OUTSIDE)
98 for x in artifact_dict['paths']
99 ])
LaMont Jones0f5171b2021-01-29 12:28:56 -0700100 return controller.RETURN_CODE_SUCCESS
101
102
LaMont Jones58362a42021-02-04 17:40:08 -0700103def _BuildSetupResponse(_input_proto, output_proto, _config):
104 """Just return POINTLESS for now."""
105 # All of the artifact types we support claim that the build is POINTLESS.
106 output_proto.build_relevance = artifacts_pb2.BuildSetupResponse.POINTLESS
107
108
109@faux.success(_BuildSetupResponse)
110@faux.empty_error
111@validate.validation_complete
112def BuildSetup(_input_proto, output_proto, _config):
113 """Setup anything needed for building artifacts
114
115 If any artifact types require steps prior to building the package, they go
116 here. For example, see ToolchainService/PrepareForBuild.
117
118 Note: crbug/1034529 introduces this method as a noop. As the individual
119 artifact_type bundlers are added here, they *must* stop uploading it via the
120 individual bundler function.
121
122 Args:
123 _input_proto (GetRequest): The input proto.
124 output_proto (GetResponse): The output proto.
125 _config (api_config.ApiConfig): The API call config.
126 """
127 # If any artifact_type says "NEEDED", the return is NEEDED.
128 # Otherwise, if any artifact_type says "UNKNOWN", the return is UNKNOWN.
129 # Otherwise, the return is POINTLESS.
130 output_proto.build_relevance = artifacts_pb2.BuildSetupResponse.POINTLESS
131 return controller.RETURN_CODE_SUCCESS
132
133
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600134def _GetImageDir(build_root, target):
135 """Return path containing images for the given build target.
136
Alex Kleine2612a02019-04-18 13:51:06 -0600137 TODO(saklein) Expand image_lib.GetLatestImageLink to support this use case.
138
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600139 Args:
140 build_root (str): Path to checkout where build occurs.
141 target (str): Name of the build target.
142
143 Returns:
Alex Kleind2bf1462019-10-24 16:37:04 -0600144 Path to the latest directory containing target images or None.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600145 """
146 image_dir = os.path.join(build_root, 'src/build/images', target, 'latest')
147 if not os.path.exists(image_dir):
Alex Kleind2bf1462019-10-24 16:37:04 -0600148 logging.warning('Expected to find image output for target %s at %s, but '
149 'path does not exist', target, image_dir)
150 return None
151
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600152 return image_dir
153
154
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700155def _BundleImageArchivesResponse(input_proto, output_proto, _config):
156 """Add artifact paths to a successful response."""
157 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
158 'path0.tar.xz')
159 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
160 'path1.tar.xz')
161
162
163@faux.success(_BundleImageArchivesResponse)
164@faux.empty_error
Alex Kleind91e95a2019-09-17 10:39:02 -0600165@validate.require('build_target.name')
166@validate.exists('output_dir')
167@validate.validation_complete
168def BundleImageArchives(input_proto, output_proto, _config):
169 """Create a .tar.xz archive for each image that has been created."""
170 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
171 output_dir = input_proto.output_dir
172 image_dir = _GetImageDir(constants.SOURCE_ROOT, build_target.name)
Alex Kleind2bf1462019-10-24 16:37:04 -0600173 if image_dir is None:
174 return
Alex Kleind91e95a2019-09-17 10:39:02 -0600175
176 archives = artifacts.ArchiveImages(image_dir, output_dir)
177
178 for archive in archives:
179 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
180
181
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700182def _BundleImageZipResponse(input_proto, output_proto, _config):
183 """Add artifact zip files to a successful response."""
184 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
185 'image.zip')
186
187
188@faux.success(_BundleImageZipResponse)
189@faux.empty_error
Michael Mortensen01910922019-07-24 14:48:10 -0600190@validate.require('build_target.name', 'output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600191@validate.exists('output_dir')
192@validate.validation_complete
193def BundleImageZip(input_proto, output_proto, _config):
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600194 """Bundle image.zip.
195
196 Args:
197 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600198 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600199 _config (api_config.ApiConfig): The API call config.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600200 """
201 target = input_proto.build_target.name
202 output_dir = input_proto.output_dir
203 image_dir = _GetImageDir(constants.SOURCE_ROOT, target)
Alex Kleind2bf1462019-10-24 16:37:04 -0600204 if image_dir is None:
205 return None
Alex Klein231d2da2019-07-22 16:44:45 -0600206
Michael Mortensen01910922019-07-24 14:48:10 -0600207 archive = artifacts.BundleImageZip(output_dir, image_dir)
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600208 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
209
210
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700211def _BundleTestUpdatePayloadsResponse(input_proto, output_proto, _config):
212 """Add test payload files to a successful response."""
213 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
214 'payload1.bin')
215
216
217@faux.success(_BundleTestUpdatePayloadsResponse)
218@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600219@validate.require('build_target.name', 'output_dir')
220@validate.exists('output_dir')
221@validate.validation_complete
222def BundleTestUpdatePayloads(input_proto, output_proto, _config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600223 """Generate minimal update payloads for the build target for testing.
224
225 Args:
226 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600227 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600228 _config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600229 """
230 target = input_proto.build_target.name
231 output_dir = input_proto.output_dir
232 build_root = constants.SOURCE_ROOT
233
234 # Use the first available image to create the update payload.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600235 img_dir = _GetImageDir(build_root, target)
Alex Kleind2bf1462019-10-24 16:37:04 -0600236 if img_dir is None:
237 return None
238
Alex Kleincb541e82019-06-26 15:06:11 -0600239 img_types = [constants.IMAGE_TYPE_TEST, constants.IMAGE_TYPE_DEV,
240 constants.IMAGE_TYPE_BASE]
241 img_names = [constants.IMAGE_TYPE_TO_NAME[t] for t in img_types]
Mike Frysinger66ce4132019-07-17 22:52:52 -0400242 img_paths = [os.path.join(img_dir, x) for x in img_names]
Mike Frysingera552be42018-08-17 14:39:32 -0400243 valid_images = [x for x in img_paths if os.path.exists(x)]
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600244
Alex Kleincb541e82019-06-26 15:06:11 -0600245 if not valid_images:
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600246 cros_build_lib.Die(
247 'Expected to find an image of type among %r for target "%s" '
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600248 'at path %s.', img_types, target, img_dir)
Alex Kleincb541e82019-06-26 15:06:11 -0600249 image = valid_images[0]
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600250
Alex Kleincb541e82019-06-26 15:06:11 -0600251 payloads = artifacts.BundleTestUpdatePayloads(image, output_dir)
252 for payload in payloads:
253 output_proto.artifacts.add().path = payload
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600254
255
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700256def _BundleAutotestFilesResponse(input_proto, output_proto, _config):
257 """Add test autotest files to a successful response."""
258 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
259 'autotest-a.tar.gz')
260
261
262@faux.success(_BundleAutotestFilesResponse)
263@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600264@validate.require('output_dir')
265@validate.exists('output_dir')
266def BundleAutotestFiles(input_proto, output_proto, config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600267 """Tar the autotest files for a build target.
268
269 Args:
270 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600271 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600272 config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600273 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600274 output_dir = input_proto.output_dir
Alex Klein238d8862019-05-07 11:32:46 -0600275 target = input_proto.build_target.name
Alex Kleine21a0952019-08-23 16:08:16 -0600276 chroot = controller_util.ParseChroot(input_proto.chroot)
277
Alex Klein238d8862019-05-07 11:32:46 -0600278 if target:
Alex Kleine21a0952019-08-23 16:08:16 -0600279 sysroot_path = os.path.join('/build', target)
Alex Klein238d8862019-05-07 11:32:46 -0600280 else:
281 # New style call, use chroot and sysroot.
Alex Klein238d8862019-05-07 11:32:46 -0600282 sysroot_path = input_proto.sysroot.path
283 if not sysroot_path:
284 cros_build_lib.Die('sysroot.path is required.')
285
Alex Kleine21a0952019-08-23 16:08:16 -0600286 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein238d8862019-05-07 11:32:46 -0600287
Alex Klein231d2da2019-07-22 16:44:45 -0600288 # TODO(saklein): Switch to the validate_only decorator when legacy handling
289 # is removed.
290 if config.validate_only:
291 return controller.RETURN_CODE_VALID_INPUT
292
Alex Kleine21a0952019-08-23 16:08:16 -0600293 if not sysroot.Exists(chroot=chroot):
Alex Klein238d8862019-05-07 11:32:46 -0600294 cros_build_lib.Die('Sysroot path must exist: %s', sysroot.path)
295
296 try:
297 # Note that this returns the full path to *multiple* tarballs.
Alex Kleine21a0952019-08-23 16:08:16 -0600298 archives = artifacts.BundleAutotestFiles(chroot, sysroot, output_dir)
Alex Klein238d8862019-05-07 11:32:46 -0600299 except artifacts.Error as e:
Alex Klein03890312020-06-30 09:59:50 -0600300 logging.warning(e)
301 return
Alex Klein238d8862019-05-07 11:32:46 -0600302
303 for archive in archives.values():
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600304 output_proto.artifacts.add().path = archive
305
306
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700307def _BundleTastFilesResponse(input_proto, output_proto, _config):
308 """Add test tast files to a successful response."""
309 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
310 'tast_bundles.tar.gz')
311
312
313@faux.success(_BundleTastFilesResponse)
314@faux.empty_error
Alex Kleinb9d810b2019-07-01 12:38:02 -0600315@validate.require('output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600316@validate.exists('output_dir')
317def BundleTastFiles(input_proto, output_proto, config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600318 """Tar the tast files for a build target.
319
320 Args:
321 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600322 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600323 config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600324 """
325 target = input_proto.build_target.name
326 output_dir = input_proto.output_dir
327 build_root = constants.SOURCE_ROOT
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600328
Alex Kleinb9d810b2019-07-01 12:38:02 -0600329 chroot = controller_util.ParseChroot(input_proto.chroot)
330 sysroot_path = input_proto.sysroot.path
331
332 # TODO(saklein) Cleanup legacy handling after it has been switched over.
333 if target:
334 # Legacy handling.
Alex Klein171da612019-08-06 14:00:42 -0600335 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
Alex Kleinb9d810b2019-07-01 12:38:02 -0600336 sysroot_path = os.path.join('/build', target)
337
338 # New handling - chroot & sysroot based.
339 # TODO(saklein) Switch this to the require decorator when legacy is removed.
340 if not sysroot_path:
341 cros_build_lib.Die('sysroot.path is required.')
342
Alex Klein231d2da2019-07-22 16:44:45 -0600343 # TODO(saklein): Switch to the validation_complete decorator when legacy
344 # handling is removed.
345 if config.validate_only:
346 return controller.RETURN_CODE_VALID_INPUT
347
Alex Kleinb9d810b2019-07-01 12:38:02 -0600348 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein231d2da2019-07-22 16:44:45 -0600349 if not sysroot.Exists(chroot=chroot):
Alex Kleinb9d810b2019-07-01 12:38:02 -0600350 cros_build_lib.Die('Sysroot must exist.')
351
352 archive = artifacts.BundleTastFiles(chroot, sysroot, output_dir)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600353
LaMont Jonesb9793cd2020-06-11 08:14:46 -0600354 if archive:
355 output_proto.artifacts.add().path = archive
356 else:
357 logging.warning('Found no tast files for %s.', target)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600358
359
Fergus Dall34d74e12020-09-29 15:52:32 +1000360def BundlePinnedGuestImages(_input_proto, _output_proto, _config):
361 # TODO(crbug/1034529): Remove this endpoint
362 pass
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700363
Fergus Dall34d74e12020-09-29 15:52:32 +1000364def FetchPinnedGuestImageUris(_input_proto, _output_proto, _config):
365 # TODO(crbug/1034529): Remove this endpoint
366 pass
Alex Klein7bf0ecb2019-06-25 10:04:15 -0600367
368
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700369def _BundleFirmwareResponse(input_proto, output_proto, _config):
370 """Add test firmware image files to a successful response."""
371 output_proto.artifacts.add().path = os.path.join(
372 input_proto.output_dir, 'firmware.tar.gz')
373
374
375@faux.success(_BundleFirmwareResponse)
376@faux.empty_error
Michael Mortensen38675192019-06-28 16:52:55 +0000377@validate.require('output_dir', 'sysroot.path')
Alex Klein231d2da2019-07-22 16:44:45 -0600378@validate.exists('output_dir')
379@validate.validation_complete
380def BundleFirmware(input_proto, output_proto, _config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600381 """Tar the firmware images for a build target.
382
383 Args:
384 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600385 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600386 _config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600387 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600388 output_dir = input_proto.output_dir
Michael Mortensen38675192019-06-28 16:52:55 +0000389 chroot = controller_util.ParseChroot(input_proto.chroot)
390 sysroot_path = input_proto.sysroot.path
391 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein231d2da2019-07-22 16:44:45 -0600392
393 if not chroot.exists():
394 cros_build_lib.Die('Chroot does not exist: %s', chroot.path)
395 elif not sysroot.Exists(chroot=chroot):
396 cros_build_lib.Die('Sysroot does not exist: %s',
397 chroot.full_path(sysroot.path))
398
Michael Mortensen38675192019-06-28 16:52:55 +0000399 archive = artifacts.BuildFirmwareArchive(chroot, sysroot, output_dir)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600400
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600401 if archive is None:
402 cros_build_lib.Die(
Michael Mortensen38675192019-06-28 16:52:55 +0000403 'Could not create firmware archive. No firmware found for %s.',
404 sysroot_path)
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600405
Alex Klein231d2da2019-07-22 16:44:45 -0600406 output_proto.artifacts.add().path = archive
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600407
408
Yicheng Liea1181f2020-09-22 11:51:10 -0700409def _BundleFpmcuUnittestsResponse(input_proto, output_proto, _config):
410 """Add fingerprint MCU unittest binaries to a successful response."""
411 output_proto.artifacts.add().path = os.path.join(
412 input_proto.output_dir, 'fpmcu_unittests.tar.gz')
413
414
415@faux.success(_BundleFpmcuUnittestsResponse)
416@faux.empty_error
417@validate.require('output_dir', 'sysroot.path')
418@validate.exists('output_dir')
419@validate.validation_complete
420def BundleFpmcuUnittests(input_proto, output_proto, _config):
421 """Tar the fingerprint MCU unittest binaries for a build target.
422
423 Args:
424 input_proto (BundleRequest): The input proto.
425 output_proto (BundleResponse): The output proto.
426 _config (api_config.ApiConfig): The API call config.
427 """
428 output_dir = input_proto.output_dir
429 chroot = controller_util.ParseChroot(input_proto.chroot)
430 sysroot_path = input_proto.sysroot.path
431 sysroot = sysroot_lib.Sysroot(sysroot_path)
432
433 if not chroot.exists():
434 cros_build_lib.Die('Chroot does not exist: %s', chroot.path)
435 elif not sysroot.Exists(chroot=chroot):
436 cros_build_lib.Die('Sysroot does not exist: %s',
437 chroot.full_path(sysroot.path))
438
439 archive = artifacts.BundleFpmcuUnittests(chroot, sysroot, output_dir)
440
441 if archive is None:
442 logging.warning(
443 'No fpmcu unittests found for %s.', sysroot_path)
444 return
445
446 output_proto.artifacts.add().path = archive
447
448
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700449def _BundleEbuildLogsResponse(input_proto, output_proto, _config):
450 """Add test log files to a successful response."""
451 output_proto.artifacts.add().path = os.path.join(
452 input_proto.output_dir, 'ebuild-logs.tar.gz')
453
454
455@faux.success(_BundleEbuildLogsResponse)
456@faux.empty_error
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600457@validate.exists('output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600458def BundleEbuildLogs(input_proto, output_proto, config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600459 """Tar the ebuild logs for a build target.
460
461 Args:
462 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600463 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600464 config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600465 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600466 output_dir = input_proto.output_dir
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600467 sysroot_path = input_proto.sysroot.path
468 chroot = controller_util.ParseChroot(input_proto.chroot)
Evan Hernandeza478d802019-04-08 15:08:24 -0600469
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600470 # TODO(mmortensen) Cleanup legacy handling after it has been switched over.
471 target = input_proto.build_target.name
472 if target:
473 # Legacy handling.
474 build_root = constants.SOURCE_ROOT
Alex Klein171da612019-08-06 14:00:42 -0600475 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600476 sysroot_path = os.path.join('/build', target)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600477
Alex Klein231d2da2019-07-22 16:44:45 -0600478 # TODO(saklein): Switch to validation_complete decorator after legacy
479 # handling has been cleaned up.
480 if config.validate_only:
481 return controller.RETURN_CODE_VALID_INPUT
482
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600483 sysroot = sysroot_lib.Sysroot(sysroot_path)
484 archive = artifacts.BundleEBuildLogsTarball(chroot, sysroot, output_dir)
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600485 if archive is None:
486 cros_build_lib.Die(
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600487 'Could not create ebuild logs archive. No logs found for %s.',
488 sysroot.path)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600489 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
Alex Klein6504eca2019-04-18 15:37:56 -0600490
491
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700492def _BundleChromeOSConfigResponse(input_proto, output_proto, _config):
493 """Add test config files to a successful response."""
494 output_proto.artifacts.add().path = os.path.join(
495 input_proto.output_dir, 'config.yaml')
496
497
498@faux.success(_BundleChromeOSConfigResponse)
499@faux.empty_error
Andrew Lamb811aead2019-08-12 10:25:05 -0600500@validate.exists('output_dir')
501@validate.validation_complete
502def BundleChromeOSConfig(input_proto, output_proto, _config):
503 """Output the ChromeOS Config payload for a build target.
504
505 Args:
506 input_proto (BundleRequest): The input proto.
507 output_proto (BundleResponse): The output proto.
508 _config (api_config.ApiConfig): The API call config.
509 """
510 output_dir = input_proto.output_dir
Andrew Lamb67bd68f2019-08-15 09:09:15 -0600511 sysroot_path = input_proto.sysroot.path
Andrew Lamb811aead2019-08-12 10:25:05 -0600512 chroot = controller_util.ParseChroot(input_proto.chroot)
513
Andrew Lamb67bd68f2019-08-15 09:09:15 -0600514 # TODO(mmortensen) Cleanup legacy handling after it has been switched over.
515 target = input_proto.build_target.name
516 if target:
517 # Legacy handling.
518 build_root = constants.SOURCE_ROOT
519 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
520 sysroot_path = os.path.join('/build', target)
521
522 sysroot = sysroot_lib.Sysroot(sysroot_path)
Andrew Lamb811aead2019-08-12 10:25:05 -0600523 chromeos_config = artifacts.BundleChromeOSConfig(chroot, sysroot, output_dir)
524 if chromeos_config is None:
525 cros_build_lib.Die(
526 'Could not create ChromeOS Config payload. No config found for %s.',
527 sysroot.path)
528 output_proto.artifacts.add().path = os.path.join(output_dir, chromeos_config)
529
530
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700531def _BundleSimpleChromeArtifactsResponse(input_proto, output_proto, _config):
532 """Add test simple chrome files to a successful response."""
533 output_proto.artifacts.add().path = os.path.join(
534 input_proto.output_dir, 'simple_chrome.txt')
535
536
537@faux.success(_BundleSimpleChromeArtifactsResponse)
538@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600539@validate.require('output_dir', 'sysroot.build_target.name', 'sysroot.path')
540@validate.exists('output_dir')
541@validate.validation_complete
542def BundleSimpleChromeArtifacts(input_proto, output_proto, _config):
Alex Klein2275d692019-04-23 16:04:12 -0600543 """Create the simple chrome artifacts."""
Alex Klein2275d692019-04-23 16:04:12 -0600544 sysroot_path = input_proto.sysroot.path
Alex Klein2275d692019-04-23 16:04:12 -0600545 output_dir = input_proto.output_dir
546
Alex Klein2275d692019-04-23 16:04:12 -0600547 # Build out the argument instances.
Alex Klein26e472b2020-03-10 14:35:01 -0600548 build_target = controller_util.ParseBuildTarget(
549 input_proto.sysroot.build_target)
550 chroot = controller_util.ParseChroot(input_proto.chroot)
Alex Klein2275d692019-04-23 16:04:12 -0600551 # Sysroot.path needs to be the fully qualified path, including the chroot.
552 full_sysroot_path = os.path.join(chroot.path, sysroot_path.lstrip(os.sep))
553 sysroot = sysroot_lib.Sysroot(full_sysroot_path)
554
555 # Quick sanity check that the sysroot exists before we go on.
556 if not sysroot.Exists():
557 cros_build_lib.Die('The sysroot does not exist.')
558
559 try:
560 results = artifacts.BundleSimpleChromeArtifacts(chroot, sysroot,
561 build_target, output_dir)
562 except artifacts.Error as e:
563 cros_build_lib.Die('Error %s raised in BundleSimpleChromeArtifacts: %s',
564 type(e), e)
565
566 for file_name in results:
567 output_proto.artifacts.add().path = file_name
568
569
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700570def _BundleVmFilesResponse(input_proto, output_proto, _config):
571 """Add test vm files to a successful response."""
572 output_proto.artifacts.add().path = os.path.join(
573 input_proto.output_dir, 'f1.tar')
574
575
576@faux.success(_BundleVmFilesResponse)
577@faux.empty_error
Michael Mortensen51f06722019-07-18 09:55:50 -0600578@validate.require('chroot.path', 'test_results_dir', 'output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600579@validate.exists('output_dir')
580@validate.validation_complete
581def BundleVmFiles(input_proto, output_proto, _config):
Alex Klein6504eca2019-04-18 15:37:56 -0600582 """Tar VM disk and memory files.
583
584 Args:
Trent Begin008cade2019-10-31 13:40:59 -0600585 input_proto (BundleVmFilesRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600586 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600587 _config (api_config.ApiConfig): The API call config.
Alex Klein6504eca2019-04-18 15:37:56 -0600588 """
Michael Mortensen51f06722019-07-18 09:55:50 -0600589 chroot = controller_util.ParseChroot(input_proto.chroot)
590 test_results_dir = input_proto.test_results_dir
Alex Klein6504eca2019-04-18 15:37:56 -0600591 output_dir = input_proto.output_dir
592
Michael Mortensen51f06722019-07-18 09:55:50 -0600593 archives = artifacts.BundleVmFiles(
594 chroot, test_results_dir, output_dir)
Alex Klein6504eca2019-04-18 15:37:56 -0600595 for archive in archives:
596 output_proto.artifacts.add().path = archive
Tiancong Wangc4805b72019-06-11 12:12:03 -0700597
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700598def _BundleAFDOGenerationArtifactsResponse(input_proto, output_proto, _config):
599 """Add test tarball AFDO file to a successful response."""
600 output_proto.artifacts.add().path = os.path.join(
601 input_proto.output_dir, 'artifact1')
602
Alex Klein231d2da2019-07-22 16:44:45 -0600603
Tiancong Wang24a3df72019-08-20 15:48:51 -0700604_VALID_ARTIFACT_TYPES = [toolchain_pb2.BENCHMARK_AFDO,
605 toolchain_pb2.ORDERFILE]
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700606@faux.success(_BundleAFDOGenerationArtifactsResponse)
607@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600608@validate.require('build_target.name', 'output_dir')
Tiancong Wang50b80a92019-08-01 14:46:15 -0700609@validate.is_in('artifact_type', _VALID_ARTIFACT_TYPES)
Alex Klein231d2da2019-07-22 16:44:45 -0600610@validate.exists('output_dir')
Tiancong Wang2ade7932019-09-27 14:15:40 -0700611@validate.exists('chroot.chrome_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600612@validate.validation_complete
Tiancong Wang50b80a92019-08-01 14:46:15 -0700613def BundleAFDOGenerationArtifacts(input_proto, output_proto, _config):
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700614 """Generic function for creating tarballs of both AFDO and orderfile.
Tiancong Wangc4805b72019-06-11 12:12:03 -0700615
616 Args:
Tiancong Wang50b80a92019-08-01 14:46:15 -0700617 input_proto (BundleChromeAFDORequest): The input proto.
Tiancong Wangc4805b72019-06-11 12:12:03 -0700618 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600619 _config (api_config.ApiConfig): The API call config.
Tiancong Wangc4805b72019-06-11 12:12:03 -0700620 """
Tiancong Wang2ade7932019-09-27 14:15:40 -0700621 chrome_root = input_proto.chroot.chrome_dir
Tiancong Wangc4805b72019-06-11 12:12:03 -0700622 output_dir = input_proto.output_dir
Tiancong Wang50b80a92019-08-01 14:46:15 -0700623 artifact_type = input_proto.artifact_type
Tiancong Wangc4805b72019-06-11 12:12:03 -0700624
Alex Klein26e472b2020-03-10 14:35:01 -0600625 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
Tiancong Wangc4805b72019-06-11 12:12:03 -0700626 chroot = controller_util.ParseChroot(input_proto.chroot)
627
628 try:
Tiancong Wang24a3df72019-08-20 15:48:51 -0700629 is_orderfile = bool(artifact_type is toolchain_pb2.ORDERFILE)
Tiancong Wang50b80a92019-08-01 14:46:15 -0700630 results = artifacts.BundleAFDOGenerationArtifacts(
Tiancong Wang2ade7932019-09-27 14:15:40 -0700631 is_orderfile, chroot, chrome_root,
Tiancong Wang50b80a92019-08-01 14:46:15 -0700632 build_target, output_dir)
Tiancong Wangc4805b72019-06-11 12:12:03 -0700633 except artifacts.Error as e:
634 cros_build_lib.Die('Error %s raised in BundleSimpleChromeArtifacts: %s',
635 type(e), e)
636
637 for file_name in results:
638 output_proto.artifacts.add().path = file_name
Alex Klein0b1cbfc2019-08-14 10:09:58 -0600639
640
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700641def _ExportCpeReportResponse(input_proto, output_proto, _config):
642 """Add test cpe results to a successful response."""
643 output_proto.artifacts.add().path = os.path.join(
644 input_proto.output_dir, 'cpe_report.txt')
645 output_proto.artifacts.add().path = os.path.join(
646 input_proto.output_dir, 'cpe_warnings.txt')
647
648
649@faux.success(_ExportCpeReportResponse)
650@faux.empty_error
Alex Klein0b1cbfc2019-08-14 10:09:58 -0600651@validate.exists('output_dir')
652def ExportCpeReport(input_proto, output_proto, config):
653 """Export a CPE report.
654
655 Args:
656 input_proto (BundleRequest): The input proto.
657 output_proto (BundleResponse): The output proto.
658 config (api_config.ApiConfig): The API call config.
659 """
660 chroot = controller_util.ParseChroot(input_proto.chroot)
661 output_dir = input_proto.output_dir
662
663 if input_proto.build_target.name:
664 # Legacy handling - use the default sysroot path for the build target.
665 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
666 sysroot = sysroot_lib.Sysroot(build_target.root)
667 elif input_proto.sysroot.path:
668 sysroot = sysroot_lib.Sysroot(input_proto.sysroot.path)
669 else:
670 # TODO(saklein): Switch to validate decorators once legacy handling can be
671 # cleaned up.
672 cros_build_lib.Die('sysroot.path is required.')
673
674 if config.validate_only:
675 return controller.RETURN_CODE_VALID_INPUT
676
677 cpe_result = artifacts.GenerateCpeReport(chroot, sysroot, output_dir)
678
679 output_proto.artifacts.add().path = cpe_result.report
680 output_proto.artifacts.add().path = cpe_result.warnings
Shao-Chuan Leea44dddc2020-10-30 17:16:55 +0900681
682
683def _BundleGceTarballResponse(input_proto, output_proto, _config):
684 """Add artifact tarball to a successful response."""
685 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
686 constants.TEST_IMAGE_GCE_TAR)
687
688
689@faux.success(_BundleGceTarballResponse)
690@faux.empty_error
691@validate.require('build_target.name', 'output_dir')
692@validate.exists('output_dir')
693@validate.validation_complete
694def BundleGceTarball(input_proto, output_proto, _config):
695 """Bundle the test image into a tarball suitable for importing into GCE.
696
697 Args:
698 input_proto (BundleRequest): The input proto.
699 output_proto (BundleResponse): The output proto.
700 _config (api_config.ApiConfig): The API call config.
701 """
702 target = input_proto.build_target.name
703 output_dir = input_proto.output_dir
704 image_dir = _GetImageDir(constants.SOURCE_ROOT, target)
705 if image_dir is None:
706 return None
707
708 tarball = artifacts.BundleGceTarball(output_dir, image_dir)
709 output_proto.artifacts.add().path = tarball