blob: 609b2f744b98365721ffe0efe630deac97dfd4a9 [file] [log] [blame]
Evan Hernandezf388cbf2019-04-01 11:15:23 -06001# -*- coding: utf-8 -*-
2# Copyright 2019 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Implements ArtifactService."""
7
8from __future__ import print_function
9
10import os
11
Alex Klein231d2da2019-07-22 16:44:45 -060012from chromite.api import controller
Alex Klein076841b2019-08-29 15:19:39 -060013from chromite.api import faux
Alex Klein2b236722019-06-19 15:44:26 -060014from chromite.api import validate
Alex Klein238d8862019-05-07 11:32:46 -060015from chromite.api.controller import controller_util
LaMont Jones58362a42021-02-04 17:40:08 -070016from chromite.api.gen.chromite.api import artifacts_pb2
Tiancong Wang24a3df72019-08-20 15:48:51 -070017from chromite.api.gen.chromite.api import toolchain_pb2
Alex Klein2275d692019-04-23 16:04:12 -060018from chromite.lib import chroot_lib
Evan Hernandezf388cbf2019-04-01 11:15:23 -060019from chromite.lib import constants
20from chromite.lib import cros_build_lib
Evan Hernandezde445982019-04-22 13:42:34 -060021from chromite.lib import cros_logging as logging
Alex Klein2275d692019-04-23 16:04:12 -060022from chromite.lib import sysroot_lib
23from chromite.service import artifacts
Evan Hernandezf388cbf2019-04-01 11:15:23 -060024
25
LaMont Jones0f5171b2021-01-29 12:28:56 -070026def _GetResponse(_input_proto, _output_proto, _config):
27 """Currently bundles nothing."""
28 # TODO(crbug/1034529): As methods migrate, begin populating them based on what
29 # input_proto has defined.
30
31
32@faux.success(_GetResponse)
33@faux.empty_error
34@validate.exists('result_path.path.path')
35@validate.validation_complete
36def Get(_input_proto, _output_proto, _config):
37 """Get all artifacts.
38
39 Get all artifacts for the build.
40
41 Note: crbug/1034529 introduces this method as a noop. As the individual
42 artifact_type bundlers are added here, they *must* stop uploading it via the
43 individual bundler function.
44
45 Args:
46 _input_proto (GetRequest): The input proto.
47 _output_proto (GetResponse): The output proto.
48 _config (api_config.ApiConfig): The API call config.
49 """
50 return controller.RETURN_CODE_SUCCESS
51
52
LaMont Jones58362a42021-02-04 17:40:08 -070053def _BuildSetupResponse(_input_proto, output_proto, _config):
54 """Just return POINTLESS for now."""
55 # All of the artifact types we support claim that the build is POINTLESS.
56 output_proto.build_relevance = artifacts_pb2.BuildSetupResponse.POINTLESS
57
58
59@faux.success(_BuildSetupResponse)
60@faux.empty_error
61@validate.validation_complete
62def BuildSetup(_input_proto, output_proto, _config):
63 """Setup anything needed for building artifacts
64
65 If any artifact types require steps prior to building the package, they go
66 here. For example, see ToolchainService/PrepareForBuild.
67
68 Note: crbug/1034529 introduces this method as a noop. As the individual
69 artifact_type bundlers are added here, they *must* stop uploading it via the
70 individual bundler function.
71
72 Args:
73 _input_proto (GetRequest): The input proto.
74 output_proto (GetResponse): The output proto.
75 _config (api_config.ApiConfig): The API call config.
76 """
77 # If any artifact_type says "NEEDED", the return is NEEDED.
78 # Otherwise, if any artifact_type says "UNKNOWN", the return is UNKNOWN.
79 # Otherwise, the return is POINTLESS.
80 output_proto.build_relevance = artifacts_pb2.BuildSetupResponse.POINTLESS
81 return controller.RETURN_CODE_SUCCESS
82
83
Evan Hernandez9f125ac2019-04-08 17:18:47 -060084def _GetImageDir(build_root, target):
85 """Return path containing images for the given build target.
86
Alex Kleine2612a02019-04-18 13:51:06 -060087 TODO(saklein) Expand image_lib.GetLatestImageLink to support this use case.
88
Evan Hernandez9f125ac2019-04-08 17:18:47 -060089 Args:
90 build_root (str): Path to checkout where build occurs.
91 target (str): Name of the build target.
92
93 Returns:
Alex Kleind2bf1462019-10-24 16:37:04 -060094 Path to the latest directory containing target images or None.
Evan Hernandez9f125ac2019-04-08 17:18:47 -060095 """
96 image_dir = os.path.join(build_root, 'src/build/images', target, 'latest')
97 if not os.path.exists(image_dir):
Alex Kleind2bf1462019-10-24 16:37:04 -060098 logging.warning('Expected to find image output for target %s at %s, but '
99 'path does not exist', target, image_dir)
100 return None
101
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600102 return image_dir
103
104
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700105def _BundleImageArchivesResponse(input_proto, output_proto, _config):
106 """Add artifact paths to a successful response."""
107 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
108 'path0.tar.xz')
109 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
110 'path1.tar.xz')
111
112
113@faux.success(_BundleImageArchivesResponse)
114@faux.empty_error
Alex Kleind91e95a2019-09-17 10:39:02 -0600115@validate.require('build_target.name')
116@validate.exists('output_dir')
117@validate.validation_complete
118def BundleImageArchives(input_proto, output_proto, _config):
119 """Create a .tar.xz archive for each image that has been created."""
120 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
121 output_dir = input_proto.output_dir
122 image_dir = _GetImageDir(constants.SOURCE_ROOT, build_target.name)
Alex Kleind2bf1462019-10-24 16:37:04 -0600123 if image_dir is None:
124 return
Alex Kleind91e95a2019-09-17 10:39:02 -0600125
126 archives = artifacts.ArchiveImages(image_dir, output_dir)
127
128 for archive in archives:
129 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
130
131
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700132def _BundleImageZipResponse(input_proto, output_proto, _config):
133 """Add artifact zip files to a successful response."""
134 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
135 'image.zip')
136
137
138@faux.success(_BundleImageZipResponse)
139@faux.empty_error
Michael Mortensen01910922019-07-24 14:48:10 -0600140@validate.require('build_target.name', 'output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600141@validate.exists('output_dir')
142@validate.validation_complete
143def BundleImageZip(input_proto, output_proto, _config):
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600144 """Bundle image.zip.
145
146 Args:
147 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600148 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600149 _config (api_config.ApiConfig): The API call config.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600150 """
151 target = input_proto.build_target.name
152 output_dir = input_proto.output_dir
153 image_dir = _GetImageDir(constants.SOURCE_ROOT, target)
Alex Kleind2bf1462019-10-24 16:37:04 -0600154 if image_dir is None:
155 return None
Alex Klein231d2da2019-07-22 16:44:45 -0600156
Michael Mortensen01910922019-07-24 14:48:10 -0600157 archive = artifacts.BundleImageZip(output_dir, image_dir)
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600158 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
159
160
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700161def _BundleTestUpdatePayloadsResponse(input_proto, output_proto, _config):
162 """Add test payload files to a successful response."""
163 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
164 'payload1.bin')
165
166
167@faux.success(_BundleTestUpdatePayloadsResponse)
168@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600169@validate.require('build_target.name', 'output_dir')
170@validate.exists('output_dir')
171@validate.validation_complete
172def BundleTestUpdatePayloads(input_proto, output_proto, _config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600173 """Generate minimal update payloads for the build target for testing.
174
175 Args:
176 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600177 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600178 _config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600179 """
180 target = input_proto.build_target.name
181 output_dir = input_proto.output_dir
182 build_root = constants.SOURCE_ROOT
183
184 # Use the first available image to create the update payload.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600185 img_dir = _GetImageDir(build_root, target)
Alex Kleind2bf1462019-10-24 16:37:04 -0600186 if img_dir is None:
187 return None
188
Alex Kleincb541e82019-06-26 15:06:11 -0600189 img_types = [constants.IMAGE_TYPE_TEST, constants.IMAGE_TYPE_DEV,
190 constants.IMAGE_TYPE_BASE]
191 img_names = [constants.IMAGE_TYPE_TO_NAME[t] for t in img_types]
Mike Frysinger66ce4132019-07-17 22:52:52 -0400192 img_paths = [os.path.join(img_dir, x) for x in img_names]
Mike Frysingera552be42018-08-17 14:39:32 -0400193 valid_images = [x for x in img_paths if os.path.exists(x)]
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600194
Alex Kleincb541e82019-06-26 15:06:11 -0600195 if not valid_images:
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600196 cros_build_lib.Die(
197 'Expected to find an image of type among %r for target "%s" '
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600198 'at path %s.', img_types, target, img_dir)
Alex Kleincb541e82019-06-26 15:06:11 -0600199 image = valid_images[0]
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600200
Alex Kleincb541e82019-06-26 15:06:11 -0600201 payloads = artifacts.BundleTestUpdatePayloads(image, output_dir)
202 for payload in payloads:
203 output_proto.artifacts.add().path = payload
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600204
205
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700206def _BundleAutotestFilesResponse(input_proto, output_proto, _config):
207 """Add test autotest files to a successful response."""
208 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
209 'autotest-a.tar.gz')
210
211
212@faux.success(_BundleAutotestFilesResponse)
213@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600214@validate.require('output_dir')
215@validate.exists('output_dir')
216def BundleAutotestFiles(input_proto, output_proto, config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600217 """Tar the autotest files for a build target.
218
219 Args:
220 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600221 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600222 config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600223 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600224 output_dir = input_proto.output_dir
Alex Klein238d8862019-05-07 11:32:46 -0600225 target = input_proto.build_target.name
Alex Kleine21a0952019-08-23 16:08:16 -0600226 chroot = controller_util.ParseChroot(input_proto.chroot)
227
Alex Klein238d8862019-05-07 11:32:46 -0600228 if target:
Alex Kleine21a0952019-08-23 16:08:16 -0600229 sysroot_path = os.path.join('/build', target)
Alex Klein238d8862019-05-07 11:32:46 -0600230 else:
231 # New style call, use chroot and sysroot.
Alex Klein238d8862019-05-07 11:32:46 -0600232 sysroot_path = input_proto.sysroot.path
233 if not sysroot_path:
234 cros_build_lib.Die('sysroot.path is required.')
235
Alex Kleine21a0952019-08-23 16:08:16 -0600236 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein238d8862019-05-07 11:32:46 -0600237
Alex Klein231d2da2019-07-22 16:44:45 -0600238 # TODO(saklein): Switch to the validate_only decorator when legacy handling
239 # is removed.
240 if config.validate_only:
241 return controller.RETURN_CODE_VALID_INPUT
242
Alex Kleine21a0952019-08-23 16:08:16 -0600243 if not sysroot.Exists(chroot=chroot):
Alex Klein238d8862019-05-07 11:32:46 -0600244 cros_build_lib.Die('Sysroot path must exist: %s', sysroot.path)
245
246 try:
247 # Note that this returns the full path to *multiple* tarballs.
Alex Kleine21a0952019-08-23 16:08:16 -0600248 archives = artifacts.BundleAutotestFiles(chroot, sysroot, output_dir)
Alex Klein238d8862019-05-07 11:32:46 -0600249 except artifacts.Error as e:
Alex Klein03890312020-06-30 09:59:50 -0600250 logging.warning(e)
251 return
Alex Klein238d8862019-05-07 11:32:46 -0600252
253 for archive in archives.values():
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600254 output_proto.artifacts.add().path = archive
255
256
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700257def _BundleTastFilesResponse(input_proto, output_proto, _config):
258 """Add test tast files to a successful response."""
259 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
260 'tast_bundles.tar.gz')
261
262
263@faux.success(_BundleTastFilesResponse)
264@faux.empty_error
Alex Kleinb9d810b2019-07-01 12:38:02 -0600265@validate.require('output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600266@validate.exists('output_dir')
267def BundleTastFiles(input_proto, output_proto, config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600268 """Tar the tast files for a build target.
269
270 Args:
271 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600272 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600273 config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600274 """
275 target = input_proto.build_target.name
276 output_dir = input_proto.output_dir
277 build_root = constants.SOURCE_ROOT
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600278
Alex Kleinb9d810b2019-07-01 12:38:02 -0600279 chroot = controller_util.ParseChroot(input_proto.chroot)
280 sysroot_path = input_proto.sysroot.path
281
282 # TODO(saklein) Cleanup legacy handling after it has been switched over.
283 if target:
284 # Legacy handling.
Alex Klein171da612019-08-06 14:00:42 -0600285 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
Alex Kleinb9d810b2019-07-01 12:38:02 -0600286 sysroot_path = os.path.join('/build', target)
287
288 # New handling - chroot & sysroot based.
289 # TODO(saklein) Switch this to the require decorator when legacy is removed.
290 if not sysroot_path:
291 cros_build_lib.Die('sysroot.path is required.')
292
Alex Klein231d2da2019-07-22 16:44:45 -0600293 # TODO(saklein): Switch to the validation_complete decorator when legacy
294 # handling is removed.
295 if config.validate_only:
296 return controller.RETURN_CODE_VALID_INPUT
297
Alex Kleinb9d810b2019-07-01 12:38:02 -0600298 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein231d2da2019-07-22 16:44:45 -0600299 if not sysroot.Exists(chroot=chroot):
Alex Kleinb9d810b2019-07-01 12:38:02 -0600300 cros_build_lib.Die('Sysroot must exist.')
301
302 archive = artifacts.BundleTastFiles(chroot, sysroot, output_dir)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600303
LaMont Jonesb9793cd2020-06-11 08:14:46 -0600304 if archive:
305 output_proto.artifacts.add().path = archive
306 else:
307 logging.warning('Found no tast files for %s.', target)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600308
309
Fergus Dall34d74e12020-09-29 15:52:32 +1000310def BundlePinnedGuestImages(_input_proto, _output_proto, _config):
311 # TODO(crbug/1034529): Remove this endpoint
312 pass
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700313
Fergus Dall34d74e12020-09-29 15:52:32 +1000314def FetchPinnedGuestImageUris(_input_proto, _output_proto, _config):
315 # TODO(crbug/1034529): Remove this endpoint
316 pass
Alex Klein7bf0ecb2019-06-25 10:04:15 -0600317
318
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700319def _BundleFirmwareResponse(input_proto, output_proto, _config):
320 """Add test firmware image files to a successful response."""
321 output_proto.artifacts.add().path = os.path.join(
322 input_proto.output_dir, 'firmware.tar.gz')
323
324
325@faux.success(_BundleFirmwareResponse)
326@faux.empty_error
Michael Mortensen38675192019-06-28 16:52:55 +0000327@validate.require('output_dir', 'sysroot.path')
Alex Klein231d2da2019-07-22 16:44:45 -0600328@validate.exists('output_dir')
329@validate.validation_complete
330def BundleFirmware(input_proto, output_proto, _config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600331 """Tar the firmware images for a build target.
332
333 Args:
334 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600335 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600336 _config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600337 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600338 output_dir = input_proto.output_dir
Michael Mortensen38675192019-06-28 16:52:55 +0000339 chroot = controller_util.ParseChroot(input_proto.chroot)
340 sysroot_path = input_proto.sysroot.path
341 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein231d2da2019-07-22 16:44:45 -0600342
343 if not chroot.exists():
344 cros_build_lib.Die('Chroot does not exist: %s', chroot.path)
345 elif not sysroot.Exists(chroot=chroot):
346 cros_build_lib.Die('Sysroot does not exist: %s',
347 chroot.full_path(sysroot.path))
348
Michael Mortensen38675192019-06-28 16:52:55 +0000349 archive = artifacts.BuildFirmwareArchive(chroot, sysroot, output_dir)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600350
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600351 if archive is None:
352 cros_build_lib.Die(
Michael Mortensen38675192019-06-28 16:52:55 +0000353 'Could not create firmware archive. No firmware found for %s.',
354 sysroot_path)
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600355
Alex Klein231d2da2019-07-22 16:44:45 -0600356 output_proto.artifacts.add().path = archive
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600357
358
Yicheng Liea1181f2020-09-22 11:51:10 -0700359def _BundleFpmcuUnittestsResponse(input_proto, output_proto, _config):
360 """Add fingerprint MCU unittest binaries to a successful response."""
361 output_proto.artifacts.add().path = os.path.join(
362 input_proto.output_dir, 'fpmcu_unittests.tar.gz')
363
364
365@faux.success(_BundleFpmcuUnittestsResponse)
366@faux.empty_error
367@validate.require('output_dir', 'sysroot.path')
368@validate.exists('output_dir')
369@validate.validation_complete
370def BundleFpmcuUnittests(input_proto, output_proto, _config):
371 """Tar the fingerprint MCU unittest binaries for a build target.
372
373 Args:
374 input_proto (BundleRequest): The input proto.
375 output_proto (BundleResponse): The output proto.
376 _config (api_config.ApiConfig): The API call config.
377 """
378 output_dir = input_proto.output_dir
379 chroot = controller_util.ParseChroot(input_proto.chroot)
380 sysroot_path = input_proto.sysroot.path
381 sysroot = sysroot_lib.Sysroot(sysroot_path)
382
383 if not chroot.exists():
384 cros_build_lib.Die('Chroot does not exist: %s', chroot.path)
385 elif not sysroot.Exists(chroot=chroot):
386 cros_build_lib.Die('Sysroot does not exist: %s',
387 chroot.full_path(sysroot.path))
388
389 archive = artifacts.BundleFpmcuUnittests(chroot, sysroot, output_dir)
390
391 if archive is None:
392 logging.warning(
393 'No fpmcu unittests found for %s.', sysroot_path)
394 return
395
396 output_proto.artifacts.add().path = archive
397
398
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700399def _BundleEbuildLogsResponse(input_proto, output_proto, _config):
400 """Add test log files to a successful response."""
401 output_proto.artifacts.add().path = os.path.join(
402 input_proto.output_dir, 'ebuild-logs.tar.gz')
403
404
405@faux.success(_BundleEbuildLogsResponse)
406@faux.empty_error
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600407@validate.exists('output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600408def BundleEbuildLogs(input_proto, output_proto, config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600409 """Tar the ebuild logs for a build target.
410
411 Args:
412 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600413 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600414 config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600415 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600416 output_dir = input_proto.output_dir
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600417 sysroot_path = input_proto.sysroot.path
418 chroot = controller_util.ParseChroot(input_proto.chroot)
Evan Hernandeza478d802019-04-08 15:08:24 -0600419
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600420 # TODO(mmortensen) Cleanup legacy handling after it has been switched over.
421 target = input_proto.build_target.name
422 if target:
423 # Legacy handling.
424 build_root = constants.SOURCE_ROOT
Alex Klein171da612019-08-06 14:00:42 -0600425 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600426 sysroot_path = os.path.join('/build', target)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600427
Alex Klein231d2da2019-07-22 16:44:45 -0600428 # TODO(saklein): Switch to validation_complete decorator after legacy
429 # handling has been cleaned up.
430 if config.validate_only:
431 return controller.RETURN_CODE_VALID_INPUT
432
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600433 sysroot = sysroot_lib.Sysroot(sysroot_path)
434 archive = artifacts.BundleEBuildLogsTarball(chroot, sysroot, output_dir)
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600435 if archive is None:
436 cros_build_lib.Die(
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600437 'Could not create ebuild logs archive. No logs found for %s.',
438 sysroot.path)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600439 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
Alex Klein6504eca2019-04-18 15:37:56 -0600440
441
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700442def _BundleChromeOSConfigResponse(input_proto, output_proto, _config):
443 """Add test config files to a successful response."""
444 output_proto.artifacts.add().path = os.path.join(
445 input_proto.output_dir, 'config.yaml')
446
447
448@faux.success(_BundleChromeOSConfigResponse)
449@faux.empty_error
Andrew Lamb811aead2019-08-12 10:25:05 -0600450@validate.exists('output_dir')
451@validate.validation_complete
452def BundleChromeOSConfig(input_proto, output_proto, _config):
453 """Output the ChromeOS Config payload for a build target.
454
455 Args:
456 input_proto (BundleRequest): The input proto.
457 output_proto (BundleResponse): The output proto.
458 _config (api_config.ApiConfig): The API call config.
459 """
460 output_dir = input_proto.output_dir
Andrew Lamb67bd68f2019-08-15 09:09:15 -0600461 sysroot_path = input_proto.sysroot.path
Andrew Lamb811aead2019-08-12 10:25:05 -0600462 chroot = controller_util.ParseChroot(input_proto.chroot)
463
Andrew Lamb67bd68f2019-08-15 09:09:15 -0600464 # TODO(mmortensen) Cleanup legacy handling after it has been switched over.
465 target = input_proto.build_target.name
466 if target:
467 # Legacy handling.
468 build_root = constants.SOURCE_ROOT
469 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
470 sysroot_path = os.path.join('/build', target)
471
472 sysroot = sysroot_lib.Sysroot(sysroot_path)
Andrew Lamb811aead2019-08-12 10:25:05 -0600473 chromeos_config = artifacts.BundleChromeOSConfig(chroot, sysroot, output_dir)
474 if chromeos_config is None:
475 cros_build_lib.Die(
476 'Could not create ChromeOS Config payload. No config found for %s.',
477 sysroot.path)
478 output_proto.artifacts.add().path = os.path.join(output_dir, chromeos_config)
479
480
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700481def _BundleSimpleChromeArtifactsResponse(input_proto, output_proto, _config):
482 """Add test simple chrome files to a successful response."""
483 output_proto.artifacts.add().path = os.path.join(
484 input_proto.output_dir, 'simple_chrome.txt')
485
486
487@faux.success(_BundleSimpleChromeArtifactsResponse)
488@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600489@validate.require('output_dir', 'sysroot.build_target.name', 'sysroot.path')
490@validate.exists('output_dir')
491@validate.validation_complete
492def BundleSimpleChromeArtifacts(input_proto, output_proto, _config):
Alex Klein2275d692019-04-23 16:04:12 -0600493 """Create the simple chrome artifacts."""
Alex Klein2275d692019-04-23 16:04:12 -0600494 sysroot_path = input_proto.sysroot.path
Alex Klein2275d692019-04-23 16:04:12 -0600495 output_dir = input_proto.output_dir
496
Alex Klein2275d692019-04-23 16:04:12 -0600497 # Build out the argument instances.
Alex Klein26e472b2020-03-10 14:35:01 -0600498 build_target = controller_util.ParseBuildTarget(
499 input_proto.sysroot.build_target)
500 chroot = controller_util.ParseChroot(input_proto.chroot)
Alex Klein2275d692019-04-23 16:04:12 -0600501 # Sysroot.path needs to be the fully qualified path, including the chroot.
502 full_sysroot_path = os.path.join(chroot.path, sysroot_path.lstrip(os.sep))
503 sysroot = sysroot_lib.Sysroot(full_sysroot_path)
504
505 # Quick sanity check that the sysroot exists before we go on.
506 if not sysroot.Exists():
507 cros_build_lib.Die('The sysroot does not exist.')
508
509 try:
510 results = artifacts.BundleSimpleChromeArtifacts(chroot, sysroot,
511 build_target, output_dir)
512 except artifacts.Error as e:
513 cros_build_lib.Die('Error %s raised in BundleSimpleChromeArtifacts: %s',
514 type(e), e)
515
516 for file_name in results:
517 output_proto.artifacts.add().path = file_name
518
519
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700520def _BundleVmFilesResponse(input_proto, output_proto, _config):
521 """Add test vm files to a successful response."""
522 output_proto.artifacts.add().path = os.path.join(
523 input_proto.output_dir, 'f1.tar')
524
525
526@faux.success(_BundleVmFilesResponse)
527@faux.empty_error
Michael Mortensen51f06722019-07-18 09:55:50 -0600528@validate.require('chroot.path', 'test_results_dir', 'output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600529@validate.exists('output_dir')
530@validate.validation_complete
531def BundleVmFiles(input_proto, output_proto, _config):
Alex Klein6504eca2019-04-18 15:37:56 -0600532 """Tar VM disk and memory files.
533
534 Args:
Trent Begin008cade2019-10-31 13:40:59 -0600535 input_proto (BundleVmFilesRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600536 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600537 _config (api_config.ApiConfig): The API call config.
Alex Klein6504eca2019-04-18 15:37:56 -0600538 """
Michael Mortensen51f06722019-07-18 09:55:50 -0600539 chroot = controller_util.ParseChroot(input_proto.chroot)
540 test_results_dir = input_proto.test_results_dir
Alex Klein6504eca2019-04-18 15:37:56 -0600541 output_dir = input_proto.output_dir
542
Michael Mortensen51f06722019-07-18 09:55:50 -0600543 archives = artifacts.BundleVmFiles(
544 chroot, test_results_dir, output_dir)
Alex Klein6504eca2019-04-18 15:37:56 -0600545 for archive in archives:
546 output_proto.artifacts.add().path = archive
Tiancong Wangc4805b72019-06-11 12:12:03 -0700547
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700548def _BundleAFDOGenerationArtifactsResponse(input_proto, output_proto, _config):
549 """Add test tarball AFDO file to a successful response."""
550 output_proto.artifacts.add().path = os.path.join(
551 input_proto.output_dir, 'artifact1')
552
Alex Klein231d2da2019-07-22 16:44:45 -0600553
Tiancong Wang24a3df72019-08-20 15:48:51 -0700554_VALID_ARTIFACT_TYPES = [toolchain_pb2.BENCHMARK_AFDO,
555 toolchain_pb2.ORDERFILE]
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700556@faux.success(_BundleAFDOGenerationArtifactsResponse)
557@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600558@validate.require('build_target.name', 'output_dir')
Tiancong Wang50b80a92019-08-01 14:46:15 -0700559@validate.is_in('artifact_type', _VALID_ARTIFACT_TYPES)
Alex Klein231d2da2019-07-22 16:44:45 -0600560@validate.exists('output_dir')
Tiancong Wang2ade7932019-09-27 14:15:40 -0700561@validate.exists('chroot.chrome_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600562@validate.validation_complete
Tiancong Wang50b80a92019-08-01 14:46:15 -0700563def BundleAFDOGenerationArtifacts(input_proto, output_proto, _config):
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700564 """Generic function for creating tarballs of both AFDO and orderfile.
Tiancong Wangc4805b72019-06-11 12:12:03 -0700565
566 Args:
Tiancong Wang50b80a92019-08-01 14:46:15 -0700567 input_proto (BundleChromeAFDORequest): The input proto.
Tiancong Wangc4805b72019-06-11 12:12:03 -0700568 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600569 _config (api_config.ApiConfig): The API call config.
Tiancong Wangc4805b72019-06-11 12:12:03 -0700570 """
Tiancong Wang2ade7932019-09-27 14:15:40 -0700571 chrome_root = input_proto.chroot.chrome_dir
Tiancong Wangc4805b72019-06-11 12:12:03 -0700572 output_dir = input_proto.output_dir
Tiancong Wang50b80a92019-08-01 14:46:15 -0700573 artifact_type = input_proto.artifact_type
Tiancong Wangc4805b72019-06-11 12:12:03 -0700574
Alex Klein26e472b2020-03-10 14:35:01 -0600575 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
Tiancong Wangc4805b72019-06-11 12:12:03 -0700576 chroot = controller_util.ParseChroot(input_proto.chroot)
577
578 try:
Tiancong Wang24a3df72019-08-20 15:48:51 -0700579 is_orderfile = bool(artifact_type is toolchain_pb2.ORDERFILE)
Tiancong Wang50b80a92019-08-01 14:46:15 -0700580 results = artifacts.BundleAFDOGenerationArtifacts(
Tiancong Wang2ade7932019-09-27 14:15:40 -0700581 is_orderfile, chroot, chrome_root,
Tiancong Wang50b80a92019-08-01 14:46:15 -0700582 build_target, output_dir)
Tiancong Wangc4805b72019-06-11 12:12:03 -0700583 except artifacts.Error as e:
584 cros_build_lib.Die('Error %s raised in BundleSimpleChromeArtifacts: %s',
585 type(e), e)
586
587 for file_name in results:
588 output_proto.artifacts.add().path = file_name
Alex Klein0b1cbfc2019-08-14 10:09:58 -0600589
590
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700591def _ExportCpeReportResponse(input_proto, output_proto, _config):
592 """Add test cpe results to a successful response."""
593 output_proto.artifacts.add().path = os.path.join(
594 input_proto.output_dir, 'cpe_report.txt')
595 output_proto.artifacts.add().path = os.path.join(
596 input_proto.output_dir, 'cpe_warnings.txt')
597
598
599@faux.success(_ExportCpeReportResponse)
600@faux.empty_error
Alex Klein0b1cbfc2019-08-14 10:09:58 -0600601@validate.exists('output_dir')
602def ExportCpeReport(input_proto, output_proto, config):
603 """Export a CPE report.
604
605 Args:
606 input_proto (BundleRequest): The input proto.
607 output_proto (BundleResponse): The output proto.
608 config (api_config.ApiConfig): The API call config.
609 """
610 chroot = controller_util.ParseChroot(input_proto.chroot)
611 output_dir = input_proto.output_dir
612
613 if input_proto.build_target.name:
614 # Legacy handling - use the default sysroot path for the build target.
615 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
616 sysroot = sysroot_lib.Sysroot(build_target.root)
617 elif input_proto.sysroot.path:
618 sysroot = sysroot_lib.Sysroot(input_proto.sysroot.path)
619 else:
620 # TODO(saklein): Switch to validate decorators once legacy handling can be
621 # cleaned up.
622 cros_build_lib.Die('sysroot.path is required.')
623
624 if config.validate_only:
625 return controller.RETURN_CODE_VALID_INPUT
626
627 cpe_result = artifacts.GenerateCpeReport(chroot, sysroot, output_dir)
628
629 output_proto.artifacts.add().path = cpe_result.report
630 output_proto.artifacts.add().path = cpe_result.warnings
Shao-Chuan Leea44dddc2020-10-30 17:16:55 +0900631
632
633def _BundleGceTarballResponse(input_proto, output_proto, _config):
634 """Add artifact tarball to a successful response."""
635 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
636 constants.TEST_IMAGE_GCE_TAR)
637
638
639@faux.success(_BundleGceTarballResponse)
640@faux.empty_error
641@validate.require('build_target.name', 'output_dir')
642@validate.exists('output_dir')
643@validate.validation_complete
644def BundleGceTarball(input_proto, output_proto, _config):
645 """Bundle the test image into a tarball suitable for importing into GCE.
646
647 Args:
648 input_proto (BundleRequest): The input proto.
649 output_proto (BundleResponse): The output proto.
650 _config (api_config.ApiConfig): The API call config.
651 """
652 target = input_proto.build_target.name
653 output_dir = input_proto.output_dir
654 image_dir = _GetImageDir(constants.SOURCE_ROOT, target)
655 if image_dir is None:
656 return None
657
658 tarball = artifacts.BundleGceTarball(output_dir, image_dir)
659 output_proto.artifacts.add().path = tarball
Michael Mortensen6667b542020-12-12 11:09:55 -0700660
661
662def _BundleDebugSymbolsResponse(input_proto, output_proto, _config):
663 """Add artifact tarball to a successful response."""
664 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
665 constants.DEBUG_SYMBOLS_TAR)
666
667
668@faux.success(_BundleDebugSymbolsResponse)
669@faux.empty_error
670@validate.require('build_target.name', 'output_dir')
671@validate.exists('output_dir')
672@validate.validation_complete
673def BundleDebugSymbols(input_proto, output_proto, _config):
674 """Bundle the debug symbols into a tarball suitable for importing into GCE.
675
676 Args:
677 input_proto (BundleRequest): The input proto.
678 output_proto (BundleResponse): The output proto.
679 _config (api_config.ApiConfig): The API call config.
680 """
Michael Mortensen6667b542020-12-12 11:09:55 -0700681 output_dir = input_proto.output_dir
682
683 chroot = controller_util.ParseChroot(input_proto.chroot)
Michael Mortensen7da39cc2021-03-09 10:28:45 -0700684 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
Michael Mortensen6667b542020-12-12 11:09:55 -0700685 result = artifacts.GenerateBreakpadSymbols(chroot,
Michael Mortensen7da39cc2021-03-09 10:28:45 -0700686 build_target,
Michael Mortensen6667b542020-12-12 11:09:55 -0700687 debug=True)
688
689 # Verify breakpad symbol generation before gathering the sym files.
690 if result.returncode != 0:
691 return controller.RETURN_CODE_COMPLETED_UNSUCCESSFULLY
692
693 with chroot.tempdir() as symbol_tmpdir, chroot.tempdir() as dest_tmpdir:
Michael Mortensen7da39cc2021-03-09 10:28:45 -0700694 breakpad_dir = os.path.join('/build', build_target.name,
695 'usr/lib/debug/breakpad')
Michael Mortensen6667b542020-12-12 11:09:55 -0700696 sym_file_list = artifacts.GatherSymbolFiles(tempdir=symbol_tmpdir,
697 destdir=dest_tmpdir,
698 paths=[breakpad_dir])
699 if len(sym_file_list) == 0:
700 logging.warning('No sym files found in %s.', breakpad_dir)
701 # Create tarball from destination_tmp, then copy it...
702 tarball_path = os.path.join(output_dir, constants.DEBUG_SYMBOLS_TAR)
703 result = cros_build_lib.CreateTarball(tarball_path, dest_tmpdir)
704 if result.returncode != 0:
705 logging.error('Error (%d) when creating tarball %s from %s',
706 result.returncode,
707 tarball_path,
708 dest_tmpdir)
709 return controller.RETURN_CODE_COMPLETED_UNSUCCESSFULLY
710 output_proto.artifacts.add().path = tarball_path
711
712 return controller.RETURN_CODE_SUCCESS