blob: a7e6bf5d4f2d9db2b673f0690b3a92a0a40d48ec [file] [log] [blame]
Evan Hernandezf388cbf2019-04-01 11:15:23 -06001# Copyright 2019 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Implements ArtifactService."""
6
Evan Hernandezf388cbf2019-04-01 11:15:23 -06007import os
8
Alex Klein231d2da2019-07-22 16:44:45 -06009from chromite.api import controller
Alex Klein076841b2019-08-29 15:19:39 -060010from chromite.api import faux
Alex Klein2b236722019-06-19 15:44:26 -060011from chromite.api import validate
Alex Klein238d8862019-05-07 11:32:46 -060012from chromite.api.controller import controller_util
LaMont Jones58362a42021-02-04 17:40:08 -070013from chromite.api.gen.chromite.api import artifacts_pb2
Tiancong Wang24a3df72019-08-20 15:48:51 -070014from chromite.api.gen.chromite.api import toolchain_pb2
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000015from chromite.api.gen.chromiumos import common_pb2
Alex Klein2275d692019-04-23 16:04:12 -060016from chromite.lib import chroot_lib
Evan Hernandezf388cbf2019-04-01 11:15:23 -060017from chromite.lib import constants
18from chromite.lib import cros_build_lib
Evan Hernandezde445982019-04-22 13:42:34 -060019from chromite.lib import cros_logging as logging
Alex Klein2275d692019-04-23 16:04:12 -060020from chromite.lib import sysroot_lib
21from chromite.service import artifacts
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000022from chromite.service import image as image_service
Evan Hernandezf388cbf2019-04-01 11:15:23 -060023
24
LaMont Jones0f5171b2021-01-29 12:28:56 -070025def _GetResponse(_input_proto, _output_proto, _config):
26 """Currently bundles nothing."""
27 # TODO(crbug/1034529): As methods migrate, begin populating them based on what
28 # input_proto has defined.
29
30
31@faux.success(_GetResponse)
32@faux.empty_error
33@validate.exists('result_path.path.path')
34@validate.validation_complete
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000035def Get(input_proto, output_proto, _config):
LaMont Jones0f5171b2021-01-29 12:28:56 -070036 """Get all artifacts.
37
38 Get all artifacts for the build.
39
40 Note: crbug/1034529 introduces this method as a noop. As the individual
41 artifact_type bundlers are added here, they *must* stop uploading it via the
42 individual bundler function.
43
44 Args:
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000045 input_proto (GetRequest): The input proto.
46 output_proto (GetResponse): The output proto.
LaMont Jones0f5171b2021-01-29 12:28:56 -070047 _config (api_config.ApiConfig): The API call config.
48 """
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000049
50 image_proto = input_proto.artifact_info.image
51 base_path = os.path.join(input_proto.chroot.path,
52 input_proto.sysroot.path[1:])
53 output_dir = input_proto.result_path.path.path
54
55 images_list = image_service.Get(image_proto, base_path, output_dir)
56
57 for artifact_dict in images_list:
58 output_proto.artifacts.image.artifacts.add(
59 artifact_type=artifact_dict['type'],
60 paths=[
61 common_pb2.Path(
62 path=x,
63 location=common_pb2.Path.Location.OUTSIDE)
64 for x in artifact_dict['paths']
65 ])
66
LaMont Jones0f5171b2021-01-29 12:28:56 -070067 return controller.RETURN_CODE_SUCCESS
68
69
LaMont Jones58362a42021-02-04 17:40:08 -070070def _BuildSetupResponse(_input_proto, output_proto, _config):
71 """Just return POINTLESS for now."""
72 # All of the artifact types we support claim that the build is POINTLESS.
73 output_proto.build_relevance = artifacts_pb2.BuildSetupResponse.POINTLESS
74
75
76@faux.success(_BuildSetupResponse)
77@faux.empty_error
78@validate.validation_complete
79def BuildSetup(_input_proto, output_proto, _config):
80 """Setup anything needed for building artifacts
81
82 If any artifact types require steps prior to building the package, they go
83 here. For example, see ToolchainService/PrepareForBuild.
84
85 Note: crbug/1034529 introduces this method as a noop. As the individual
86 artifact_type bundlers are added here, they *must* stop uploading it via the
87 individual bundler function.
88
89 Args:
90 _input_proto (GetRequest): The input proto.
91 output_proto (GetResponse): The output proto.
92 _config (api_config.ApiConfig): The API call config.
93 """
94 # If any artifact_type says "NEEDED", the return is NEEDED.
95 # Otherwise, if any artifact_type says "UNKNOWN", the return is UNKNOWN.
96 # Otherwise, the return is POINTLESS.
97 output_proto.build_relevance = artifacts_pb2.BuildSetupResponse.POINTLESS
98 return controller.RETURN_CODE_SUCCESS
99
100
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600101def _GetImageDir(build_root, target):
102 """Return path containing images for the given build target.
103
Alex Kleine2612a02019-04-18 13:51:06 -0600104 TODO(saklein) Expand image_lib.GetLatestImageLink to support this use case.
105
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600106 Args:
107 build_root (str): Path to checkout where build occurs.
108 target (str): Name of the build target.
109
110 Returns:
Alex Kleind2bf1462019-10-24 16:37:04 -0600111 Path to the latest directory containing target images or None.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600112 """
113 image_dir = os.path.join(build_root, 'src/build/images', target, 'latest')
114 if not os.path.exists(image_dir):
Alex Kleind2bf1462019-10-24 16:37:04 -0600115 logging.warning('Expected to find image output for target %s at %s, but '
116 'path does not exist', target, image_dir)
117 return None
118
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600119 return image_dir
120
121
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700122def _BundleImageArchivesResponse(input_proto, output_proto, _config):
123 """Add artifact paths to a successful response."""
124 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
125 'path0.tar.xz')
126 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
127 'path1.tar.xz')
128
129
130@faux.success(_BundleImageArchivesResponse)
131@faux.empty_error
Alex Kleind91e95a2019-09-17 10:39:02 -0600132@validate.require('build_target.name')
133@validate.exists('output_dir')
134@validate.validation_complete
135def BundleImageArchives(input_proto, output_proto, _config):
136 """Create a .tar.xz archive for each image that has been created."""
137 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
138 output_dir = input_proto.output_dir
139 image_dir = _GetImageDir(constants.SOURCE_ROOT, build_target.name)
Alex Kleind2bf1462019-10-24 16:37:04 -0600140 if image_dir is None:
141 return
Alex Kleind91e95a2019-09-17 10:39:02 -0600142
143 archives = artifacts.ArchiveImages(image_dir, output_dir)
144
145 for archive in archives:
146 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
147
148
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700149def _BundleImageZipResponse(input_proto, output_proto, _config):
150 """Add artifact zip files to a successful response."""
151 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
152 'image.zip')
153
154
155@faux.success(_BundleImageZipResponse)
156@faux.empty_error
Michael Mortensen01910922019-07-24 14:48:10 -0600157@validate.require('build_target.name', 'output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600158@validate.exists('output_dir')
159@validate.validation_complete
160def BundleImageZip(input_proto, output_proto, _config):
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600161 """Bundle image.zip.
162
163 Args:
164 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600165 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600166 _config (api_config.ApiConfig): The API call config.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600167 """
168 target = input_proto.build_target.name
169 output_dir = input_proto.output_dir
170 image_dir = _GetImageDir(constants.SOURCE_ROOT, target)
Alex Kleind2bf1462019-10-24 16:37:04 -0600171 if image_dir is None:
172 return None
Alex Klein231d2da2019-07-22 16:44:45 -0600173
Michael Mortensen01910922019-07-24 14:48:10 -0600174 archive = artifacts.BundleImageZip(output_dir, image_dir)
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600175 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
176
177
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700178def _BundleTestUpdatePayloadsResponse(input_proto, output_proto, _config):
179 """Add test payload files to a successful response."""
180 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
181 'payload1.bin')
182
183
184@faux.success(_BundleTestUpdatePayloadsResponse)
185@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600186@validate.require('build_target.name', 'output_dir')
187@validate.exists('output_dir')
188@validate.validation_complete
189def BundleTestUpdatePayloads(input_proto, output_proto, _config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600190 """Generate minimal update payloads for the build target for testing.
191
192 Args:
193 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600194 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600195 _config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600196 """
197 target = input_proto.build_target.name
198 output_dir = input_proto.output_dir
199 build_root = constants.SOURCE_ROOT
200
201 # Use the first available image to create the update payload.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600202 img_dir = _GetImageDir(build_root, target)
Alex Kleind2bf1462019-10-24 16:37:04 -0600203 if img_dir is None:
204 return None
205
Alex Kleincb541e82019-06-26 15:06:11 -0600206 img_types = [constants.IMAGE_TYPE_TEST, constants.IMAGE_TYPE_DEV,
207 constants.IMAGE_TYPE_BASE]
208 img_names = [constants.IMAGE_TYPE_TO_NAME[t] for t in img_types]
Mike Frysinger66ce4132019-07-17 22:52:52 -0400209 img_paths = [os.path.join(img_dir, x) for x in img_names]
Mike Frysingera552be42018-08-17 14:39:32 -0400210 valid_images = [x for x in img_paths if os.path.exists(x)]
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600211
Alex Kleincb541e82019-06-26 15:06:11 -0600212 if not valid_images:
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600213 cros_build_lib.Die(
214 'Expected to find an image of type among %r for target "%s" '
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600215 'at path %s.', img_types, target, img_dir)
Alex Kleincb541e82019-06-26 15:06:11 -0600216 image = valid_images[0]
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600217
Alex Kleincb541e82019-06-26 15:06:11 -0600218 payloads = artifacts.BundleTestUpdatePayloads(image, output_dir)
219 for payload in payloads:
220 output_proto.artifacts.add().path = payload
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600221
222
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700223def _BundleAutotestFilesResponse(input_proto, output_proto, _config):
224 """Add test autotest files to a successful response."""
225 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
226 'autotest-a.tar.gz')
227
228
229@faux.success(_BundleAutotestFilesResponse)
230@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600231@validate.require('output_dir')
232@validate.exists('output_dir')
233def BundleAutotestFiles(input_proto, output_proto, config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600234 """Tar the autotest files for a build target.
235
236 Args:
237 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600238 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600239 config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600240 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600241 output_dir = input_proto.output_dir
Alex Klein238d8862019-05-07 11:32:46 -0600242 target = input_proto.build_target.name
Alex Kleine21a0952019-08-23 16:08:16 -0600243 chroot = controller_util.ParseChroot(input_proto.chroot)
244
Alex Klein238d8862019-05-07 11:32:46 -0600245 if target:
Alex Kleine21a0952019-08-23 16:08:16 -0600246 sysroot_path = os.path.join('/build', target)
Alex Klein238d8862019-05-07 11:32:46 -0600247 else:
248 # New style call, use chroot and sysroot.
Alex Klein238d8862019-05-07 11:32:46 -0600249 sysroot_path = input_proto.sysroot.path
250 if not sysroot_path:
251 cros_build_lib.Die('sysroot.path is required.')
252
Alex Kleine21a0952019-08-23 16:08:16 -0600253 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein238d8862019-05-07 11:32:46 -0600254
Alex Klein231d2da2019-07-22 16:44:45 -0600255 # TODO(saklein): Switch to the validate_only decorator when legacy handling
256 # is removed.
257 if config.validate_only:
258 return controller.RETURN_CODE_VALID_INPUT
259
Alex Kleine21a0952019-08-23 16:08:16 -0600260 if not sysroot.Exists(chroot=chroot):
Alex Klein238d8862019-05-07 11:32:46 -0600261 cros_build_lib.Die('Sysroot path must exist: %s', sysroot.path)
262
263 try:
264 # Note that this returns the full path to *multiple* tarballs.
Alex Kleine21a0952019-08-23 16:08:16 -0600265 archives = artifacts.BundleAutotestFiles(chroot, sysroot, output_dir)
Alex Klein238d8862019-05-07 11:32:46 -0600266 except artifacts.Error as e:
Alex Klein03890312020-06-30 09:59:50 -0600267 logging.warning(e)
268 return
Alex Klein238d8862019-05-07 11:32:46 -0600269
270 for archive in archives.values():
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600271 output_proto.artifacts.add().path = archive
272
273
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700274def _BundleTastFilesResponse(input_proto, output_proto, _config):
275 """Add test tast files to a successful response."""
276 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
277 'tast_bundles.tar.gz')
278
279
280@faux.success(_BundleTastFilesResponse)
281@faux.empty_error
Alex Kleinb9d810b2019-07-01 12:38:02 -0600282@validate.require('output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600283@validate.exists('output_dir')
284def BundleTastFiles(input_proto, output_proto, config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600285 """Tar the tast files for a build target.
286
287 Args:
288 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600289 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600290 config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600291 """
292 target = input_proto.build_target.name
293 output_dir = input_proto.output_dir
294 build_root = constants.SOURCE_ROOT
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600295
Alex Kleinb9d810b2019-07-01 12:38:02 -0600296 chroot = controller_util.ParseChroot(input_proto.chroot)
297 sysroot_path = input_proto.sysroot.path
298
299 # TODO(saklein) Cleanup legacy handling after it has been switched over.
300 if target:
301 # Legacy handling.
Alex Klein171da612019-08-06 14:00:42 -0600302 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
Alex Kleinb9d810b2019-07-01 12:38:02 -0600303 sysroot_path = os.path.join('/build', target)
304
305 # New handling - chroot & sysroot based.
306 # TODO(saklein) Switch this to the require decorator when legacy is removed.
307 if not sysroot_path:
308 cros_build_lib.Die('sysroot.path is required.')
309
Alex Klein231d2da2019-07-22 16:44:45 -0600310 # TODO(saklein): Switch to the validation_complete decorator when legacy
311 # handling is removed.
312 if config.validate_only:
313 return controller.RETURN_CODE_VALID_INPUT
314
Alex Kleinb9d810b2019-07-01 12:38:02 -0600315 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein231d2da2019-07-22 16:44:45 -0600316 if not sysroot.Exists(chroot=chroot):
Alex Kleinb9d810b2019-07-01 12:38:02 -0600317 cros_build_lib.Die('Sysroot must exist.')
318
319 archive = artifacts.BundleTastFiles(chroot, sysroot, output_dir)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600320
LaMont Jonesb9793cd2020-06-11 08:14:46 -0600321 if archive:
322 output_proto.artifacts.add().path = archive
323 else:
324 logging.warning('Found no tast files for %s.', target)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600325
326
Fergus Dall34d74e12020-09-29 15:52:32 +1000327def BundlePinnedGuestImages(_input_proto, _output_proto, _config):
328 # TODO(crbug/1034529): Remove this endpoint
329 pass
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700330
Fergus Dall34d74e12020-09-29 15:52:32 +1000331def FetchPinnedGuestImageUris(_input_proto, _output_proto, _config):
332 # TODO(crbug/1034529): Remove this endpoint
333 pass
Alex Klein7bf0ecb2019-06-25 10:04:15 -0600334
335
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700336def _BundleFirmwareResponse(input_proto, output_proto, _config):
337 """Add test firmware image files to a successful response."""
338 output_proto.artifacts.add().path = os.path.join(
339 input_proto.output_dir, 'firmware.tar.gz')
340
341
342@faux.success(_BundleFirmwareResponse)
343@faux.empty_error
Michael Mortensen38675192019-06-28 16:52:55 +0000344@validate.require('output_dir', 'sysroot.path')
Alex Klein231d2da2019-07-22 16:44:45 -0600345@validate.exists('output_dir')
346@validate.validation_complete
347def BundleFirmware(input_proto, output_proto, _config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600348 """Tar the firmware images for a build target.
349
350 Args:
351 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600352 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600353 _config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600354 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600355 output_dir = input_proto.output_dir
Michael Mortensen38675192019-06-28 16:52:55 +0000356 chroot = controller_util.ParseChroot(input_proto.chroot)
357 sysroot_path = input_proto.sysroot.path
358 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein231d2da2019-07-22 16:44:45 -0600359
360 if not chroot.exists():
361 cros_build_lib.Die('Chroot does not exist: %s', chroot.path)
362 elif not sysroot.Exists(chroot=chroot):
363 cros_build_lib.Die('Sysroot does not exist: %s',
364 chroot.full_path(sysroot.path))
365
Michael Mortensen38675192019-06-28 16:52:55 +0000366 archive = artifacts.BuildFirmwareArchive(chroot, sysroot, output_dir)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600367
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600368 if archive is None:
369 cros_build_lib.Die(
Michael Mortensen38675192019-06-28 16:52:55 +0000370 'Could not create firmware archive. No firmware found for %s.',
371 sysroot_path)
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600372
Alex Klein231d2da2019-07-22 16:44:45 -0600373 output_proto.artifacts.add().path = archive
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600374
375
Yicheng Liea1181f2020-09-22 11:51:10 -0700376def _BundleFpmcuUnittestsResponse(input_proto, output_proto, _config):
377 """Add fingerprint MCU unittest binaries to a successful response."""
378 output_proto.artifacts.add().path = os.path.join(
379 input_proto.output_dir, 'fpmcu_unittests.tar.gz')
380
381
382@faux.success(_BundleFpmcuUnittestsResponse)
383@faux.empty_error
384@validate.require('output_dir', 'sysroot.path')
385@validate.exists('output_dir')
386@validate.validation_complete
387def BundleFpmcuUnittests(input_proto, output_proto, _config):
388 """Tar the fingerprint MCU unittest binaries for a build target.
389
390 Args:
391 input_proto (BundleRequest): The input proto.
392 output_proto (BundleResponse): The output proto.
393 _config (api_config.ApiConfig): The API call config.
394 """
395 output_dir = input_proto.output_dir
396 chroot = controller_util.ParseChroot(input_proto.chroot)
397 sysroot_path = input_proto.sysroot.path
398 sysroot = sysroot_lib.Sysroot(sysroot_path)
399
400 if not chroot.exists():
401 cros_build_lib.Die('Chroot does not exist: %s', chroot.path)
402 elif not sysroot.Exists(chroot=chroot):
403 cros_build_lib.Die('Sysroot does not exist: %s',
404 chroot.full_path(sysroot.path))
405
406 archive = artifacts.BundleFpmcuUnittests(chroot, sysroot, output_dir)
407
408 if archive is None:
409 logging.warning(
410 'No fpmcu unittests found for %s.', sysroot_path)
411 return
412
413 output_proto.artifacts.add().path = archive
414
415
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700416def _BundleEbuildLogsResponse(input_proto, output_proto, _config):
417 """Add test log files to a successful response."""
418 output_proto.artifacts.add().path = os.path.join(
419 input_proto.output_dir, 'ebuild-logs.tar.gz')
420
421
422@faux.success(_BundleEbuildLogsResponse)
423@faux.empty_error
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600424@validate.exists('output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600425def BundleEbuildLogs(input_proto, output_proto, config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600426 """Tar the ebuild logs for a build target.
427
428 Args:
429 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600430 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600431 config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600432 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600433 output_dir = input_proto.output_dir
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600434 sysroot_path = input_proto.sysroot.path
435 chroot = controller_util.ParseChroot(input_proto.chroot)
Evan Hernandeza478d802019-04-08 15:08:24 -0600436
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600437 # TODO(mmortensen) Cleanup legacy handling after it has been switched over.
438 target = input_proto.build_target.name
439 if target:
440 # Legacy handling.
441 build_root = constants.SOURCE_ROOT
Alex Klein171da612019-08-06 14:00:42 -0600442 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600443 sysroot_path = os.path.join('/build', target)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600444
Alex Klein231d2da2019-07-22 16:44:45 -0600445 # TODO(saklein): Switch to validation_complete decorator after legacy
446 # handling has been cleaned up.
447 if config.validate_only:
448 return controller.RETURN_CODE_VALID_INPUT
449
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600450 sysroot = sysroot_lib.Sysroot(sysroot_path)
451 archive = artifacts.BundleEBuildLogsTarball(chroot, sysroot, output_dir)
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600452 if archive is None:
453 cros_build_lib.Die(
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600454 'Could not create ebuild logs archive. No logs found for %s.',
455 sysroot.path)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600456 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
Alex Klein6504eca2019-04-18 15:37:56 -0600457
458
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700459def _BundleChromeOSConfigResponse(input_proto, output_proto, _config):
460 """Add test config files to a successful response."""
461 output_proto.artifacts.add().path = os.path.join(
462 input_proto.output_dir, 'config.yaml')
463
464
465@faux.success(_BundleChromeOSConfigResponse)
466@faux.empty_error
Andrew Lamb811aead2019-08-12 10:25:05 -0600467@validate.exists('output_dir')
468@validate.validation_complete
469def BundleChromeOSConfig(input_proto, output_proto, _config):
470 """Output the ChromeOS Config payload for a build target.
471
472 Args:
473 input_proto (BundleRequest): The input proto.
474 output_proto (BundleResponse): The output proto.
475 _config (api_config.ApiConfig): The API call config.
476 """
477 output_dir = input_proto.output_dir
Andrew Lamb67bd68f2019-08-15 09:09:15 -0600478 sysroot_path = input_proto.sysroot.path
Andrew Lamb811aead2019-08-12 10:25:05 -0600479 chroot = controller_util.ParseChroot(input_proto.chroot)
480
Andrew Lamb67bd68f2019-08-15 09:09:15 -0600481 # TODO(mmortensen) Cleanup legacy handling after it has been switched over.
482 target = input_proto.build_target.name
483 if target:
484 # Legacy handling.
485 build_root = constants.SOURCE_ROOT
486 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
487 sysroot_path = os.path.join('/build', target)
488
489 sysroot = sysroot_lib.Sysroot(sysroot_path)
Andrew Lamb811aead2019-08-12 10:25:05 -0600490 chromeos_config = artifacts.BundleChromeOSConfig(chroot, sysroot, output_dir)
491 if chromeos_config is None:
492 cros_build_lib.Die(
493 'Could not create ChromeOS Config payload. No config found for %s.',
494 sysroot.path)
495 output_proto.artifacts.add().path = os.path.join(output_dir, chromeos_config)
496
497
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700498def _BundleSimpleChromeArtifactsResponse(input_proto, output_proto, _config):
499 """Add test simple chrome files to a successful response."""
500 output_proto.artifacts.add().path = os.path.join(
501 input_proto.output_dir, 'simple_chrome.txt')
502
503
504@faux.success(_BundleSimpleChromeArtifactsResponse)
505@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600506@validate.require('output_dir', 'sysroot.build_target.name', 'sysroot.path')
507@validate.exists('output_dir')
508@validate.validation_complete
509def BundleSimpleChromeArtifacts(input_proto, output_proto, _config):
Alex Klein2275d692019-04-23 16:04:12 -0600510 """Create the simple chrome artifacts."""
Alex Klein2275d692019-04-23 16:04:12 -0600511 sysroot_path = input_proto.sysroot.path
Alex Klein2275d692019-04-23 16:04:12 -0600512 output_dir = input_proto.output_dir
513
Alex Klein2275d692019-04-23 16:04:12 -0600514 # Build out the argument instances.
Alex Klein26e472b2020-03-10 14:35:01 -0600515 build_target = controller_util.ParseBuildTarget(
516 input_proto.sysroot.build_target)
517 chroot = controller_util.ParseChroot(input_proto.chroot)
Alex Klein2275d692019-04-23 16:04:12 -0600518 # Sysroot.path needs to be the fully qualified path, including the chroot.
519 full_sysroot_path = os.path.join(chroot.path, sysroot_path.lstrip(os.sep))
520 sysroot = sysroot_lib.Sysroot(full_sysroot_path)
521
522 # Quick sanity check that the sysroot exists before we go on.
523 if not sysroot.Exists():
524 cros_build_lib.Die('The sysroot does not exist.')
525
526 try:
527 results = artifacts.BundleSimpleChromeArtifacts(chroot, sysroot,
528 build_target, output_dir)
529 except artifacts.Error as e:
530 cros_build_lib.Die('Error %s raised in BundleSimpleChromeArtifacts: %s',
531 type(e), e)
532
533 for file_name in results:
534 output_proto.artifacts.add().path = file_name
535
536
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700537def _BundleVmFilesResponse(input_proto, output_proto, _config):
538 """Add test vm files to a successful response."""
539 output_proto.artifacts.add().path = os.path.join(
540 input_proto.output_dir, 'f1.tar')
541
542
543@faux.success(_BundleVmFilesResponse)
544@faux.empty_error
Michael Mortensen51f06722019-07-18 09:55:50 -0600545@validate.require('chroot.path', 'test_results_dir', 'output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600546@validate.exists('output_dir')
547@validate.validation_complete
548def BundleVmFiles(input_proto, output_proto, _config):
Alex Klein6504eca2019-04-18 15:37:56 -0600549 """Tar VM disk and memory files.
550
551 Args:
Trent Begin008cade2019-10-31 13:40:59 -0600552 input_proto (BundleVmFilesRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600553 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600554 _config (api_config.ApiConfig): The API call config.
Alex Klein6504eca2019-04-18 15:37:56 -0600555 """
Michael Mortensen51f06722019-07-18 09:55:50 -0600556 chroot = controller_util.ParseChroot(input_proto.chroot)
557 test_results_dir = input_proto.test_results_dir
Alex Klein6504eca2019-04-18 15:37:56 -0600558 output_dir = input_proto.output_dir
559
Michael Mortensen51f06722019-07-18 09:55:50 -0600560 archives = artifacts.BundleVmFiles(
561 chroot, test_results_dir, output_dir)
Alex Klein6504eca2019-04-18 15:37:56 -0600562 for archive in archives:
563 output_proto.artifacts.add().path = archive
Tiancong Wangc4805b72019-06-11 12:12:03 -0700564
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700565def _BundleAFDOGenerationArtifactsResponse(input_proto, output_proto, _config):
566 """Add test tarball AFDO file to a successful response."""
567 output_proto.artifacts.add().path = os.path.join(
568 input_proto.output_dir, 'artifact1')
569
Alex Klein231d2da2019-07-22 16:44:45 -0600570
Tiancong Wang24a3df72019-08-20 15:48:51 -0700571_VALID_ARTIFACT_TYPES = [toolchain_pb2.BENCHMARK_AFDO,
572 toolchain_pb2.ORDERFILE]
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700573@faux.success(_BundleAFDOGenerationArtifactsResponse)
574@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600575@validate.require('build_target.name', 'output_dir')
Tiancong Wang50b80a92019-08-01 14:46:15 -0700576@validate.is_in('artifact_type', _VALID_ARTIFACT_TYPES)
Alex Klein231d2da2019-07-22 16:44:45 -0600577@validate.exists('output_dir')
Tiancong Wang2ade7932019-09-27 14:15:40 -0700578@validate.exists('chroot.chrome_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600579@validate.validation_complete
Tiancong Wang50b80a92019-08-01 14:46:15 -0700580def BundleAFDOGenerationArtifacts(input_proto, output_proto, _config):
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700581 """Generic function for creating tarballs of both AFDO and orderfile.
Tiancong Wangc4805b72019-06-11 12:12:03 -0700582
583 Args:
Tiancong Wang50b80a92019-08-01 14:46:15 -0700584 input_proto (BundleChromeAFDORequest): The input proto.
Tiancong Wangc4805b72019-06-11 12:12:03 -0700585 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600586 _config (api_config.ApiConfig): The API call config.
Tiancong Wangc4805b72019-06-11 12:12:03 -0700587 """
Tiancong Wang2ade7932019-09-27 14:15:40 -0700588 chrome_root = input_proto.chroot.chrome_dir
Tiancong Wangc4805b72019-06-11 12:12:03 -0700589 output_dir = input_proto.output_dir
Tiancong Wang50b80a92019-08-01 14:46:15 -0700590 artifact_type = input_proto.artifact_type
Tiancong Wangc4805b72019-06-11 12:12:03 -0700591
Alex Klein26e472b2020-03-10 14:35:01 -0600592 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
Tiancong Wangc4805b72019-06-11 12:12:03 -0700593 chroot = controller_util.ParseChroot(input_proto.chroot)
594
595 try:
Tiancong Wang24a3df72019-08-20 15:48:51 -0700596 is_orderfile = bool(artifact_type is toolchain_pb2.ORDERFILE)
Tiancong Wang50b80a92019-08-01 14:46:15 -0700597 results = artifacts.BundleAFDOGenerationArtifacts(
Tiancong Wang2ade7932019-09-27 14:15:40 -0700598 is_orderfile, chroot, chrome_root,
Tiancong Wang50b80a92019-08-01 14:46:15 -0700599 build_target, output_dir)
Tiancong Wangc4805b72019-06-11 12:12:03 -0700600 except artifacts.Error as e:
601 cros_build_lib.Die('Error %s raised in BundleSimpleChromeArtifacts: %s',
602 type(e), e)
603
604 for file_name in results:
605 output_proto.artifacts.add().path = file_name
Alex Klein0b1cbfc2019-08-14 10:09:58 -0600606
607
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700608def _ExportCpeReportResponse(input_proto, output_proto, _config):
609 """Add test cpe results to a successful response."""
610 output_proto.artifacts.add().path = os.path.join(
611 input_proto.output_dir, 'cpe_report.txt')
612 output_proto.artifacts.add().path = os.path.join(
613 input_proto.output_dir, 'cpe_warnings.txt')
614
615
616@faux.success(_ExportCpeReportResponse)
617@faux.empty_error
Alex Klein0b1cbfc2019-08-14 10:09:58 -0600618@validate.exists('output_dir')
619def ExportCpeReport(input_proto, output_proto, config):
620 """Export a CPE report.
621
622 Args:
623 input_proto (BundleRequest): The input proto.
624 output_proto (BundleResponse): The output proto.
625 config (api_config.ApiConfig): The API call config.
626 """
627 chroot = controller_util.ParseChroot(input_proto.chroot)
628 output_dir = input_proto.output_dir
629
630 if input_proto.build_target.name:
631 # Legacy handling - use the default sysroot path for the build target.
632 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
633 sysroot = sysroot_lib.Sysroot(build_target.root)
634 elif input_proto.sysroot.path:
635 sysroot = sysroot_lib.Sysroot(input_proto.sysroot.path)
636 else:
637 # TODO(saklein): Switch to validate decorators once legacy handling can be
638 # cleaned up.
639 cros_build_lib.Die('sysroot.path is required.')
640
641 if config.validate_only:
642 return controller.RETURN_CODE_VALID_INPUT
643
644 cpe_result = artifacts.GenerateCpeReport(chroot, sysroot, output_dir)
645
646 output_proto.artifacts.add().path = cpe_result.report
647 output_proto.artifacts.add().path = cpe_result.warnings
Shao-Chuan Leea44dddc2020-10-30 17:16:55 +0900648
649
650def _BundleGceTarballResponse(input_proto, output_proto, _config):
651 """Add artifact tarball to a successful response."""
652 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
653 constants.TEST_IMAGE_GCE_TAR)
654
655
656@faux.success(_BundleGceTarballResponse)
657@faux.empty_error
658@validate.require('build_target.name', 'output_dir')
659@validate.exists('output_dir')
660@validate.validation_complete
661def BundleGceTarball(input_proto, output_proto, _config):
662 """Bundle the test image into a tarball suitable for importing into GCE.
663
664 Args:
665 input_proto (BundleRequest): The input proto.
666 output_proto (BundleResponse): The output proto.
667 _config (api_config.ApiConfig): The API call config.
668 """
669 target = input_proto.build_target.name
670 output_dir = input_proto.output_dir
671 image_dir = _GetImageDir(constants.SOURCE_ROOT, target)
672 if image_dir is None:
673 return None
674
675 tarball = artifacts.BundleGceTarball(output_dir, image_dir)
676 output_proto.artifacts.add().path = tarball
Michael Mortensen6667b542020-12-12 11:09:55 -0700677
678
679def _BundleDebugSymbolsResponse(input_proto, output_proto, _config):
680 """Add artifact tarball to a successful response."""
681 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
682 constants.DEBUG_SYMBOLS_TAR)
683
684
685@faux.success(_BundleDebugSymbolsResponse)
686@faux.empty_error
687@validate.require('build_target.name', 'output_dir')
688@validate.exists('output_dir')
689@validate.validation_complete
690def BundleDebugSymbols(input_proto, output_proto, _config):
691 """Bundle the debug symbols into a tarball suitable for importing into GCE.
692
693 Args:
694 input_proto (BundleRequest): The input proto.
695 output_proto (BundleResponse): The output proto.
696 _config (api_config.ApiConfig): The API call config.
697 """
Michael Mortensen6667b542020-12-12 11:09:55 -0700698 output_dir = input_proto.output_dir
699
700 chroot = controller_util.ParseChroot(input_proto.chroot)
Michael Mortensen7da39cc2021-03-09 10:28:45 -0700701 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
Michael Mortensen6667b542020-12-12 11:09:55 -0700702 result = artifacts.GenerateBreakpadSymbols(chroot,
Michael Mortensen7da39cc2021-03-09 10:28:45 -0700703 build_target,
Michael Mortensen6667b542020-12-12 11:09:55 -0700704 debug=True)
705
706 # Verify breakpad symbol generation before gathering the sym files.
707 if result.returncode != 0:
708 return controller.RETURN_CODE_COMPLETED_UNSUCCESSFULLY
709
710 with chroot.tempdir() as symbol_tmpdir, chroot.tempdir() as dest_tmpdir:
George Engelbrechtca8bbe02021-06-10 17:56:30 -0600711 breakpad_dir = os.path.join(chroot.path, 'build', build_target.name,
Michael Mortensen7da39cc2021-03-09 10:28:45 -0700712 'usr/lib/debug/breakpad')
Michael Mortensen5044d272021-04-09 16:15:16 -0600713 # Call list on the atifacts.GatherSymbolFiles generator function to
714 # materialize and consume all entries so that all are copied to
715 # dest dir and complete list of all symbol files is returned.
716 sym_file_list = list(artifacts.GatherSymbolFiles(tempdir=symbol_tmpdir,
717 destdir=dest_tmpdir,
718 paths=[breakpad_dir]))
719 if not sym_file_list:
Michael Mortensen6667b542020-12-12 11:09:55 -0700720 logging.warning('No sym files found in %s.', breakpad_dir)
721 # Create tarball from destination_tmp, then copy it...
722 tarball_path = os.path.join(output_dir, constants.DEBUG_SYMBOLS_TAR)
723 result = cros_build_lib.CreateTarball(tarball_path, dest_tmpdir)
724 if result.returncode != 0:
725 logging.error('Error (%d) when creating tarball %s from %s',
726 result.returncode,
727 tarball_path,
728 dest_tmpdir)
729 return controller.RETURN_CODE_COMPLETED_UNSUCCESSFULLY
730 output_proto.artifacts.add().path = tarball_path
731
732 return controller.RETURN_CODE_SUCCESS