blob: fefc1726ea95ac02315bb5ee4bff32038784ae6c [file] [log] [blame]
Evan Hernandezf388cbf2019-04-01 11:15:23 -06001# -*- coding: utf-8 -*-
2# Copyright 2019 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Implements ArtifactService."""
7
8from __future__ import print_function
9
10import os
11
Alex Klein231d2da2019-07-22 16:44:45 -060012from chromite.api import controller
Alex Klein076841b2019-08-29 15:19:39 -060013from chromite.api import faux
Alex Klein2b236722019-06-19 15:44:26 -060014from chromite.api import validate
Alex Klein238d8862019-05-07 11:32:46 -060015from chromite.api.controller import controller_util
LaMont Jones58362a42021-02-04 17:40:08 -070016from chromite.api.gen.chromite.api import artifacts_pb2
Tiancong Wang24a3df72019-08-20 15:48:51 -070017from chromite.api.gen.chromite.api import toolchain_pb2
Jaques Clapaucheaa67462021-04-09 20:14:40 +000018from chromite.api.gen.chromiumos import common_pb2
Alex Klein2275d692019-04-23 16:04:12 -060019from chromite.lib import chroot_lib
Evan Hernandezf388cbf2019-04-01 11:15:23 -060020from chromite.lib import constants
21from chromite.lib import cros_build_lib
Evan Hernandezde445982019-04-22 13:42:34 -060022from chromite.lib import cros_logging as logging
Alex Klein2275d692019-04-23 16:04:12 -060023from chromite.lib import sysroot_lib
24from chromite.service import artifacts
Jaques Clapaucheaa67462021-04-09 20:14:40 +000025from chromite.service import image as image_service
Evan Hernandezf388cbf2019-04-01 11:15:23 -060026
27
LaMont Jones0f5171b2021-01-29 12:28:56 -070028def _GetResponse(_input_proto, _output_proto, _config):
29 """Currently bundles nothing."""
30 # TODO(crbug/1034529): As methods migrate, begin populating them based on what
31 # input_proto has defined.
32
33
34@faux.success(_GetResponse)
35@faux.empty_error
36@validate.exists('result_path.path.path')
37@validate.validation_complete
Jaques Clapaucheaa67462021-04-09 20:14:40 +000038def Get(input_proto, output_proto, _config):
LaMont Jones0f5171b2021-01-29 12:28:56 -070039 """Get all artifacts.
40
41 Get all artifacts for the build.
42
43 Note: crbug/1034529 introduces this method as a noop. As the individual
44 artifact_type bundlers are added here, they *must* stop uploading it via the
45 individual bundler function.
46
47 Args:
Jaques Clapaucheaa67462021-04-09 20:14:40 +000048 input_proto (GetRequest): The input proto.
49 output_proto (GetResponse): The output proto.
LaMont Jones0f5171b2021-01-29 12:28:56 -070050 _config (api_config.ApiConfig): The API call config.
51 """
Jaques Clapaucheaa67462021-04-09 20:14:40 +000052
53 image_proto = input_proto.artifact_info.image
54 base_path = os.path.join(input_proto.chroot.path,
55 input_proto.sysroot.path[1:])
56 output_dir = input_proto.result_path.path.path
57
58 images_list = image_service.Get(image_proto, base_path, output_dir)
59
60 for artifact_dict in images_list:
61 output_proto.artifacts.image.artifacts.add(
62 artifact_type=artifact_dict['type'],
63 paths=[
64 common_pb2.Path(
65 path=x,
66 location=common_pb2.Path.Location.OUTSIDE)
67 for x in artifact_dict['paths']
68 ])
69
LaMont Jones0f5171b2021-01-29 12:28:56 -070070 return controller.RETURN_CODE_SUCCESS
71
72
LaMont Jones58362a42021-02-04 17:40:08 -070073def _BuildSetupResponse(_input_proto, output_proto, _config):
74 """Just return POINTLESS for now."""
75 # All of the artifact types we support claim that the build is POINTLESS.
76 output_proto.build_relevance = artifacts_pb2.BuildSetupResponse.POINTLESS
77
78
79@faux.success(_BuildSetupResponse)
80@faux.empty_error
81@validate.validation_complete
82def BuildSetup(_input_proto, output_proto, _config):
83 """Setup anything needed for building artifacts
84
85 If any artifact types require steps prior to building the package, they go
86 here. For example, see ToolchainService/PrepareForBuild.
87
88 Note: crbug/1034529 introduces this method as a noop. As the individual
89 artifact_type bundlers are added here, they *must* stop uploading it via the
90 individual bundler function.
91
92 Args:
93 _input_proto (GetRequest): The input proto.
94 output_proto (GetResponse): The output proto.
95 _config (api_config.ApiConfig): The API call config.
96 """
97 # If any artifact_type says "NEEDED", the return is NEEDED.
98 # Otherwise, if any artifact_type says "UNKNOWN", the return is UNKNOWN.
99 # Otherwise, the return is POINTLESS.
100 output_proto.build_relevance = artifacts_pb2.BuildSetupResponse.POINTLESS
101 return controller.RETURN_CODE_SUCCESS
102
103
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600104def _GetImageDir(build_root, target):
105 """Return path containing images for the given build target.
106
Alex Kleine2612a02019-04-18 13:51:06 -0600107 TODO(saklein) Expand image_lib.GetLatestImageLink to support this use case.
108
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600109 Args:
110 build_root (str): Path to checkout where build occurs.
111 target (str): Name of the build target.
112
113 Returns:
Alex Kleind2bf1462019-10-24 16:37:04 -0600114 Path to the latest directory containing target images or None.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600115 """
116 image_dir = os.path.join(build_root, 'src/build/images', target, 'latest')
117 if not os.path.exists(image_dir):
Alex Kleind2bf1462019-10-24 16:37:04 -0600118 logging.warning('Expected to find image output for target %s at %s, but '
119 'path does not exist', target, image_dir)
120 return None
121
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600122 return image_dir
123
124
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700125def _BundleImageArchivesResponse(input_proto, output_proto, _config):
126 """Add artifact paths to a successful response."""
127 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
128 'path0.tar.xz')
129 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
130 'path1.tar.xz')
131
132
133@faux.success(_BundleImageArchivesResponse)
134@faux.empty_error
Alex Kleind91e95a2019-09-17 10:39:02 -0600135@validate.require('build_target.name')
136@validate.exists('output_dir')
137@validate.validation_complete
138def BundleImageArchives(input_proto, output_proto, _config):
139 """Create a .tar.xz archive for each image that has been created."""
140 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
141 output_dir = input_proto.output_dir
142 image_dir = _GetImageDir(constants.SOURCE_ROOT, build_target.name)
Alex Kleind2bf1462019-10-24 16:37:04 -0600143 if image_dir is None:
144 return
Alex Kleind91e95a2019-09-17 10:39:02 -0600145
146 archives = artifacts.ArchiveImages(image_dir, output_dir)
147
148 for archive in archives:
149 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
150
151
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700152def _BundleImageZipResponse(input_proto, output_proto, _config):
153 """Add artifact zip files to a successful response."""
154 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
155 'image.zip')
156
157
158@faux.success(_BundleImageZipResponse)
159@faux.empty_error
Michael Mortensen01910922019-07-24 14:48:10 -0600160@validate.require('build_target.name', 'output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600161@validate.exists('output_dir')
162@validate.validation_complete
163def BundleImageZip(input_proto, output_proto, _config):
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600164 """Bundle image.zip.
165
166 Args:
167 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600168 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600169 _config (api_config.ApiConfig): The API call config.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600170 """
171 target = input_proto.build_target.name
172 output_dir = input_proto.output_dir
173 image_dir = _GetImageDir(constants.SOURCE_ROOT, target)
Alex Kleind2bf1462019-10-24 16:37:04 -0600174 if image_dir is None:
175 return None
Alex Klein231d2da2019-07-22 16:44:45 -0600176
Michael Mortensen01910922019-07-24 14:48:10 -0600177 archive = artifacts.BundleImageZip(output_dir, image_dir)
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600178 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
179
180
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700181def _BundleTestUpdatePayloadsResponse(input_proto, output_proto, _config):
182 """Add test payload files to a successful response."""
183 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
184 'payload1.bin')
185
186
187@faux.success(_BundleTestUpdatePayloadsResponse)
188@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600189@validate.require('build_target.name', 'output_dir')
190@validate.exists('output_dir')
191@validate.validation_complete
192def BundleTestUpdatePayloads(input_proto, output_proto, _config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600193 """Generate minimal update payloads for the build target for testing.
194
195 Args:
196 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600197 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600198 _config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600199 """
200 target = input_proto.build_target.name
201 output_dir = input_proto.output_dir
202 build_root = constants.SOURCE_ROOT
203
204 # Use the first available image to create the update payload.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600205 img_dir = _GetImageDir(build_root, target)
Alex Kleind2bf1462019-10-24 16:37:04 -0600206 if img_dir is None:
207 return None
208
Alex Kleincb541e82019-06-26 15:06:11 -0600209 img_types = [constants.IMAGE_TYPE_TEST, constants.IMAGE_TYPE_DEV,
210 constants.IMAGE_TYPE_BASE]
211 img_names = [constants.IMAGE_TYPE_TO_NAME[t] for t in img_types]
Mike Frysinger66ce4132019-07-17 22:52:52 -0400212 img_paths = [os.path.join(img_dir, x) for x in img_names]
Mike Frysingera552be42018-08-17 14:39:32 -0400213 valid_images = [x for x in img_paths if os.path.exists(x)]
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600214
Alex Kleincb541e82019-06-26 15:06:11 -0600215 if not valid_images:
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600216 cros_build_lib.Die(
217 'Expected to find an image of type among %r for target "%s" '
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600218 'at path %s.', img_types, target, img_dir)
Alex Kleincb541e82019-06-26 15:06:11 -0600219 image = valid_images[0]
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600220
Alex Kleincb541e82019-06-26 15:06:11 -0600221 payloads = artifacts.BundleTestUpdatePayloads(image, output_dir)
222 for payload in payloads:
223 output_proto.artifacts.add().path = payload
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600224
225
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700226def _BundleAutotestFilesResponse(input_proto, output_proto, _config):
227 """Add test autotest files to a successful response."""
228 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
229 'autotest-a.tar.gz')
230
231
232@faux.success(_BundleAutotestFilesResponse)
233@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600234@validate.require('output_dir')
235@validate.exists('output_dir')
236def BundleAutotestFiles(input_proto, output_proto, config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600237 """Tar the autotest files for a build target.
238
239 Args:
240 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600241 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600242 config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600243 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600244 output_dir = input_proto.output_dir
Alex Klein238d8862019-05-07 11:32:46 -0600245 target = input_proto.build_target.name
Alex Kleine21a0952019-08-23 16:08:16 -0600246 chroot = controller_util.ParseChroot(input_proto.chroot)
247
Alex Klein238d8862019-05-07 11:32:46 -0600248 if target:
Alex Kleine21a0952019-08-23 16:08:16 -0600249 sysroot_path = os.path.join('/build', target)
Alex Klein238d8862019-05-07 11:32:46 -0600250 else:
251 # New style call, use chroot and sysroot.
Alex Klein238d8862019-05-07 11:32:46 -0600252 sysroot_path = input_proto.sysroot.path
253 if not sysroot_path:
254 cros_build_lib.Die('sysroot.path is required.')
255
Alex Kleine21a0952019-08-23 16:08:16 -0600256 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein238d8862019-05-07 11:32:46 -0600257
Alex Klein231d2da2019-07-22 16:44:45 -0600258 # TODO(saklein): Switch to the validate_only decorator when legacy handling
259 # is removed.
260 if config.validate_only:
261 return controller.RETURN_CODE_VALID_INPUT
262
Alex Kleine21a0952019-08-23 16:08:16 -0600263 if not sysroot.Exists(chroot=chroot):
Alex Klein238d8862019-05-07 11:32:46 -0600264 cros_build_lib.Die('Sysroot path must exist: %s', sysroot.path)
265
266 try:
267 # Note that this returns the full path to *multiple* tarballs.
Alex Kleine21a0952019-08-23 16:08:16 -0600268 archives = artifacts.BundleAutotestFiles(chroot, sysroot, output_dir)
Alex Klein238d8862019-05-07 11:32:46 -0600269 except artifacts.Error as e:
Alex Klein03890312020-06-30 09:59:50 -0600270 logging.warning(e)
271 return
Alex Klein238d8862019-05-07 11:32:46 -0600272
273 for archive in archives.values():
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600274 output_proto.artifacts.add().path = archive
275
276
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700277def _BundleTastFilesResponse(input_proto, output_proto, _config):
278 """Add test tast files to a successful response."""
279 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
280 'tast_bundles.tar.gz')
281
282
283@faux.success(_BundleTastFilesResponse)
284@faux.empty_error
Alex Kleinb9d810b2019-07-01 12:38:02 -0600285@validate.require('output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600286@validate.exists('output_dir')
287def BundleTastFiles(input_proto, output_proto, config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600288 """Tar the tast files for a build target.
289
290 Args:
291 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600292 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600293 config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600294 """
295 target = input_proto.build_target.name
296 output_dir = input_proto.output_dir
297 build_root = constants.SOURCE_ROOT
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600298
Alex Kleinb9d810b2019-07-01 12:38:02 -0600299 chroot = controller_util.ParseChroot(input_proto.chroot)
300 sysroot_path = input_proto.sysroot.path
301
302 # TODO(saklein) Cleanup legacy handling after it has been switched over.
303 if target:
304 # Legacy handling.
Alex Klein171da612019-08-06 14:00:42 -0600305 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
Alex Kleinb9d810b2019-07-01 12:38:02 -0600306 sysroot_path = os.path.join('/build', target)
307
308 # New handling - chroot & sysroot based.
309 # TODO(saklein) Switch this to the require decorator when legacy is removed.
310 if not sysroot_path:
311 cros_build_lib.Die('sysroot.path is required.')
312
Alex Klein231d2da2019-07-22 16:44:45 -0600313 # TODO(saklein): Switch to the validation_complete decorator when legacy
314 # handling is removed.
315 if config.validate_only:
316 return controller.RETURN_CODE_VALID_INPUT
317
Alex Kleinb9d810b2019-07-01 12:38:02 -0600318 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein231d2da2019-07-22 16:44:45 -0600319 if not sysroot.Exists(chroot=chroot):
Alex Kleinb9d810b2019-07-01 12:38:02 -0600320 cros_build_lib.Die('Sysroot must exist.')
321
322 archive = artifacts.BundleTastFiles(chroot, sysroot, output_dir)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600323
LaMont Jonesb9793cd2020-06-11 08:14:46 -0600324 if archive:
325 output_proto.artifacts.add().path = archive
326 else:
327 logging.warning('Found no tast files for %s.', target)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600328
329
Fergus Dall34d74e12020-09-29 15:52:32 +1000330def BundlePinnedGuestImages(_input_proto, _output_proto, _config):
331 # TODO(crbug/1034529): Remove this endpoint
332 pass
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700333
Fergus Dall34d74e12020-09-29 15:52:32 +1000334def FetchPinnedGuestImageUris(_input_proto, _output_proto, _config):
335 # TODO(crbug/1034529): Remove this endpoint
336 pass
Alex Klein7bf0ecb2019-06-25 10:04:15 -0600337
338
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700339def _BundleFirmwareResponse(input_proto, output_proto, _config):
340 """Add test firmware image files to a successful response."""
341 output_proto.artifacts.add().path = os.path.join(
342 input_proto.output_dir, 'firmware.tar.gz')
343
344
345@faux.success(_BundleFirmwareResponse)
346@faux.empty_error
Michael Mortensen38675192019-06-28 16:52:55 +0000347@validate.require('output_dir', 'sysroot.path')
Alex Klein231d2da2019-07-22 16:44:45 -0600348@validate.exists('output_dir')
349@validate.validation_complete
350def BundleFirmware(input_proto, output_proto, _config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600351 """Tar the firmware images for a build target.
352
353 Args:
354 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600355 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600356 _config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600357 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600358 output_dir = input_proto.output_dir
Michael Mortensen38675192019-06-28 16:52:55 +0000359 chroot = controller_util.ParseChroot(input_proto.chroot)
360 sysroot_path = input_proto.sysroot.path
361 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein231d2da2019-07-22 16:44:45 -0600362
363 if not chroot.exists():
364 cros_build_lib.Die('Chroot does not exist: %s', chroot.path)
365 elif not sysroot.Exists(chroot=chroot):
366 cros_build_lib.Die('Sysroot does not exist: %s',
367 chroot.full_path(sysroot.path))
368
Michael Mortensen38675192019-06-28 16:52:55 +0000369 archive = artifacts.BuildFirmwareArchive(chroot, sysroot, output_dir)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600370
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600371 if archive is None:
372 cros_build_lib.Die(
Michael Mortensen38675192019-06-28 16:52:55 +0000373 'Could not create firmware archive. No firmware found for %s.',
374 sysroot_path)
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600375
Alex Klein231d2da2019-07-22 16:44:45 -0600376 output_proto.artifacts.add().path = archive
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600377
378
Yicheng Liea1181f2020-09-22 11:51:10 -0700379def _BundleFpmcuUnittestsResponse(input_proto, output_proto, _config):
380 """Add fingerprint MCU unittest binaries to a successful response."""
381 output_proto.artifacts.add().path = os.path.join(
382 input_proto.output_dir, 'fpmcu_unittests.tar.gz')
383
384
385@faux.success(_BundleFpmcuUnittestsResponse)
386@faux.empty_error
387@validate.require('output_dir', 'sysroot.path')
388@validate.exists('output_dir')
389@validate.validation_complete
390def BundleFpmcuUnittests(input_proto, output_proto, _config):
391 """Tar the fingerprint MCU unittest binaries for a build target.
392
393 Args:
394 input_proto (BundleRequest): The input proto.
395 output_proto (BundleResponse): The output proto.
396 _config (api_config.ApiConfig): The API call config.
397 """
398 output_dir = input_proto.output_dir
399 chroot = controller_util.ParseChroot(input_proto.chroot)
400 sysroot_path = input_proto.sysroot.path
401 sysroot = sysroot_lib.Sysroot(sysroot_path)
402
403 if not chroot.exists():
404 cros_build_lib.Die('Chroot does not exist: %s', chroot.path)
405 elif not sysroot.Exists(chroot=chroot):
406 cros_build_lib.Die('Sysroot does not exist: %s',
407 chroot.full_path(sysroot.path))
408
409 archive = artifacts.BundleFpmcuUnittests(chroot, sysroot, output_dir)
410
411 if archive is None:
412 logging.warning(
413 'No fpmcu unittests found for %s.', sysroot_path)
414 return
415
416 output_proto.artifacts.add().path = archive
417
418
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700419def _BundleEbuildLogsResponse(input_proto, output_proto, _config):
420 """Add test log files to a successful response."""
421 output_proto.artifacts.add().path = os.path.join(
422 input_proto.output_dir, 'ebuild-logs.tar.gz')
423
424
425@faux.success(_BundleEbuildLogsResponse)
426@faux.empty_error
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600427@validate.exists('output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600428def BundleEbuildLogs(input_proto, output_proto, config):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600429 """Tar the ebuild logs for a build target.
430
431 Args:
432 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600433 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600434 config (api_config.ApiConfig): The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600435 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600436 output_dir = input_proto.output_dir
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600437 sysroot_path = input_proto.sysroot.path
438 chroot = controller_util.ParseChroot(input_proto.chroot)
Evan Hernandeza478d802019-04-08 15:08:24 -0600439
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600440 # TODO(mmortensen) Cleanup legacy handling after it has been switched over.
441 target = input_proto.build_target.name
442 if target:
443 # Legacy handling.
444 build_root = constants.SOURCE_ROOT
Alex Klein171da612019-08-06 14:00:42 -0600445 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600446 sysroot_path = os.path.join('/build', target)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600447
Alex Klein231d2da2019-07-22 16:44:45 -0600448 # TODO(saklein): Switch to validation_complete decorator after legacy
449 # handling has been cleaned up.
450 if config.validate_only:
451 return controller.RETURN_CODE_VALID_INPUT
452
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600453 sysroot = sysroot_lib.Sysroot(sysroot_path)
454 archive = artifacts.BundleEBuildLogsTarball(chroot, sysroot, output_dir)
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600455 if archive is None:
456 cros_build_lib.Die(
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600457 'Could not create ebuild logs archive. No logs found for %s.',
458 sysroot.path)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600459 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
Alex Klein6504eca2019-04-18 15:37:56 -0600460
461
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700462def _BundleChromeOSConfigResponse(input_proto, output_proto, _config):
463 """Add test config files to a successful response."""
464 output_proto.artifacts.add().path = os.path.join(
465 input_proto.output_dir, 'config.yaml')
466
467
468@faux.success(_BundleChromeOSConfigResponse)
469@faux.empty_error
Andrew Lamb811aead2019-08-12 10:25:05 -0600470@validate.exists('output_dir')
471@validate.validation_complete
472def BundleChromeOSConfig(input_proto, output_proto, _config):
473 """Output the ChromeOS Config payload for a build target.
474
475 Args:
476 input_proto (BundleRequest): The input proto.
477 output_proto (BundleResponse): The output proto.
478 _config (api_config.ApiConfig): The API call config.
479 """
480 output_dir = input_proto.output_dir
Andrew Lamb67bd68f2019-08-15 09:09:15 -0600481 sysroot_path = input_proto.sysroot.path
Andrew Lamb811aead2019-08-12 10:25:05 -0600482 chroot = controller_util.ParseChroot(input_proto.chroot)
483
Andrew Lamb67bd68f2019-08-15 09:09:15 -0600484 # TODO(mmortensen) Cleanup legacy handling after it has been switched over.
485 target = input_proto.build_target.name
486 if target:
487 # Legacy handling.
488 build_root = constants.SOURCE_ROOT
489 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
490 sysroot_path = os.path.join('/build', target)
491
492 sysroot = sysroot_lib.Sysroot(sysroot_path)
Andrew Lamb811aead2019-08-12 10:25:05 -0600493 chromeos_config = artifacts.BundleChromeOSConfig(chroot, sysroot, output_dir)
494 if chromeos_config is None:
495 cros_build_lib.Die(
496 'Could not create ChromeOS Config payload. No config found for %s.',
497 sysroot.path)
498 output_proto.artifacts.add().path = os.path.join(output_dir, chromeos_config)
499
500
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700501def _BundleSimpleChromeArtifactsResponse(input_proto, output_proto, _config):
502 """Add test simple chrome files to a successful response."""
503 output_proto.artifacts.add().path = os.path.join(
504 input_proto.output_dir, 'simple_chrome.txt')
505
506
507@faux.success(_BundleSimpleChromeArtifactsResponse)
508@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600509@validate.require('output_dir', 'sysroot.build_target.name', 'sysroot.path')
510@validate.exists('output_dir')
511@validate.validation_complete
512def BundleSimpleChromeArtifacts(input_proto, output_proto, _config):
Alex Klein2275d692019-04-23 16:04:12 -0600513 """Create the simple chrome artifacts."""
Alex Klein2275d692019-04-23 16:04:12 -0600514 sysroot_path = input_proto.sysroot.path
Alex Klein2275d692019-04-23 16:04:12 -0600515 output_dir = input_proto.output_dir
516
Alex Klein2275d692019-04-23 16:04:12 -0600517 # Build out the argument instances.
Alex Klein26e472b2020-03-10 14:35:01 -0600518 build_target = controller_util.ParseBuildTarget(
519 input_proto.sysroot.build_target)
520 chroot = controller_util.ParseChroot(input_proto.chroot)
Alex Klein2275d692019-04-23 16:04:12 -0600521 # Sysroot.path needs to be the fully qualified path, including the chroot.
522 full_sysroot_path = os.path.join(chroot.path, sysroot_path.lstrip(os.sep))
523 sysroot = sysroot_lib.Sysroot(full_sysroot_path)
524
525 # Quick sanity check that the sysroot exists before we go on.
526 if not sysroot.Exists():
527 cros_build_lib.Die('The sysroot does not exist.')
528
529 try:
530 results = artifacts.BundleSimpleChromeArtifacts(chroot, sysroot,
531 build_target, output_dir)
532 except artifacts.Error as e:
533 cros_build_lib.Die('Error %s raised in BundleSimpleChromeArtifacts: %s',
534 type(e), e)
535
536 for file_name in results:
537 output_proto.artifacts.add().path = file_name
538
539
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700540def _BundleVmFilesResponse(input_proto, output_proto, _config):
541 """Add test vm files to a successful response."""
542 output_proto.artifacts.add().path = os.path.join(
543 input_proto.output_dir, 'f1.tar')
544
545
546@faux.success(_BundleVmFilesResponse)
547@faux.empty_error
Michael Mortensen51f06722019-07-18 09:55:50 -0600548@validate.require('chroot.path', 'test_results_dir', 'output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600549@validate.exists('output_dir')
550@validate.validation_complete
551def BundleVmFiles(input_proto, output_proto, _config):
Alex Klein6504eca2019-04-18 15:37:56 -0600552 """Tar VM disk and memory files.
553
554 Args:
Trent Begin008cade2019-10-31 13:40:59 -0600555 input_proto (BundleVmFilesRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600556 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600557 _config (api_config.ApiConfig): The API call config.
Alex Klein6504eca2019-04-18 15:37:56 -0600558 """
Michael Mortensen51f06722019-07-18 09:55:50 -0600559 chroot = controller_util.ParseChroot(input_proto.chroot)
560 test_results_dir = input_proto.test_results_dir
Alex Klein6504eca2019-04-18 15:37:56 -0600561 output_dir = input_proto.output_dir
562
Michael Mortensen51f06722019-07-18 09:55:50 -0600563 archives = artifacts.BundleVmFiles(
564 chroot, test_results_dir, output_dir)
Alex Klein6504eca2019-04-18 15:37:56 -0600565 for archive in archives:
566 output_proto.artifacts.add().path = archive
Tiancong Wangc4805b72019-06-11 12:12:03 -0700567
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700568def _BundleAFDOGenerationArtifactsResponse(input_proto, output_proto, _config):
569 """Add test tarball AFDO file to a successful response."""
570 output_proto.artifacts.add().path = os.path.join(
571 input_proto.output_dir, 'artifact1')
572
Alex Klein231d2da2019-07-22 16:44:45 -0600573
Tiancong Wang24a3df72019-08-20 15:48:51 -0700574_VALID_ARTIFACT_TYPES = [toolchain_pb2.BENCHMARK_AFDO,
575 toolchain_pb2.ORDERFILE]
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700576@faux.success(_BundleAFDOGenerationArtifactsResponse)
577@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600578@validate.require('build_target.name', 'output_dir')
Tiancong Wang50b80a92019-08-01 14:46:15 -0700579@validate.is_in('artifact_type', _VALID_ARTIFACT_TYPES)
Alex Klein231d2da2019-07-22 16:44:45 -0600580@validate.exists('output_dir')
Tiancong Wang2ade7932019-09-27 14:15:40 -0700581@validate.exists('chroot.chrome_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600582@validate.validation_complete
Tiancong Wang50b80a92019-08-01 14:46:15 -0700583def BundleAFDOGenerationArtifacts(input_proto, output_proto, _config):
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700584 """Generic function for creating tarballs of both AFDO and orderfile.
Tiancong Wangc4805b72019-06-11 12:12:03 -0700585
586 Args:
Tiancong Wang50b80a92019-08-01 14:46:15 -0700587 input_proto (BundleChromeAFDORequest): The input proto.
Tiancong Wangc4805b72019-06-11 12:12:03 -0700588 output_proto (BundleResponse): The output proto.
Alex Klein231d2da2019-07-22 16:44:45 -0600589 _config (api_config.ApiConfig): The API call config.
Tiancong Wangc4805b72019-06-11 12:12:03 -0700590 """
Tiancong Wang2ade7932019-09-27 14:15:40 -0700591 chrome_root = input_proto.chroot.chrome_dir
Tiancong Wangc4805b72019-06-11 12:12:03 -0700592 output_dir = input_proto.output_dir
Tiancong Wang50b80a92019-08-01 14:46:15 -0700593 artifact_type = input_proto.artifact_type
Tiancong Wangc4805b72019-06-11 12:12:03 -0700594
Alex Klein26e472b2020-03-10 14:35:01 -0600595 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
Tiancong Wangc4805b72019-06-11 12:12:03 -0700596 chroot = controller_util.ParseChroot(input_proto.chroot)
597
598 try:
Tiancong Wang24a3df72019-08-20 15:48:51 -0700599 is_orderfile = bool(artifact_type is toolchain_pb2.ORDERFILE)
Tiancong Wang50b80a92019-08-01 14:46:15 -0700600 results = artifacts.BundleAFDOGenerationArtifacts(
Tiancong Wang2ade7932019-09-27 14:15:40 -0700601 is_orderfile, chroot, chrome_root,
Tiancong Wang50b80a92019-08-01 14:46:15 -0700602 build_target, output_dir)
Tiancong Wangc4805b72019-06-11 12:12:03 -0700603 except artifacts.Error as e:
604 cros_build_lib.Die('Error %s raised in BundleSimpleChromeArtifacts: %s',
605 type(e), e)
606
607 for file_name in results:
608 output_proto.artifacts.add().path = file_name
Alex Klein0b1cbfc2019-08-14 10:09:58 -0600609
610
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700611def _ExportCpeReportResponse(input_proto, output_proto, _config):
612 """Add test cpe results to a successful response."""
613 output_proto.artifacts.add().path = os.path.join(
614 input_proto.output_dir, 'cpe_report.txt')
615 output_proto.artifacts.add().path = os.path.join(
616 input_proto.output_dir, 'cpe_warnings.txt')
617
618
619@faux.success(_ExportCpeReportResponse)
620@faux.empty_error
Alex Klein0b1cbfc2019-08-14 10:09:58 -0600621@validate.exists('output_dir')
622def ExportCpeReport(input_proto, output_proto, config):
623 """Export a CPE report.
624
625 Args:
626 input_proto (BundleRequest): The input proto.
627 output_proto (BundleResponse): The output proto.
628 config (api_config.ApiConfig): The API call config.
629 """
630 chroot = controller_util.ParseChroot(input_proto.chroot)
631 output_dir = input_proto.output_dir
632
633 if input_proto.build_target.name:
634 # Legacy handling - use the default sysroot path for the build target.
635 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
636 sysroot = sysroot_lib.Sysroot(build_target.root)
637 elif input_proto.sysroot.path:
638 sysroot = sysroot_lib.Sysroot(input_proto.sysroot.path)
639 else:
640 # TODO(saklein): Switch to validate decorators once legacy handling can be
641 # cleaned up.
642 cros_build_lib.Die('sysroot.path is required.')
643
644 if config.validate_only:
645 return controller.RETURN_CODE_VALID_INPUT
646
647 cpe_result = artifacts.GenerateCpeReport(chroot, sysroot, output_dir)
648
649 output_proto.artifacts.add().path = cpe_result.report
650 output_proto.artifacts.add().path = cpe_result.warnings
Shao-Chuan Leea44dddc2020-10-30 17:16:55 +0900651
652
653def _BundleGceTarballResponse(input_proto, output_proto, _config):
654 """Add artifact tarball to a successful response."""
655 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
656 constants.TEST_IMAGE_GCE_TAR)
657
658
659@faux.success(_BundleGceTarballResponse)
660@faux.empty_error
661@validate.require('build_target.name', 'output_dir')
662@validate.exists('output_dir')
663@validate.validation_complete
664def BundleGceTarball(input_proto, output_proto, _config):
665 """Bundle the test image into a tarball suitable for importing into GCE.
666
667 Args:
668 input_proto (BundleRequest): The input proto.
669 output_proto (BundleResponse): The output proto.
670 _config (api_config.ApiConfig): The API call config.
671 """
672 target = input_proto.build_target.name
673 output_dir = input_proto.output_dir
674 image_dir = _GetImageDir(constants.SOURCE_ROOT, target)
675 if image_dir is None:
676 return None
677
678 tarball = artifacts.BundleGceTarball(output_dir, image_dir)
679 output_proto.artifacts.add().path = tarball
Michael Mortensen6667b542020-12-12 11:09:55 -0700680
681
682def _BundleDebugSymbolsResponse(input_proto, output_proto, _config):
683 """Add artifact tarball to a successful response."""
684 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
685 constants.DEBUG_SYMBOLS_TAR)
686
687
688@faux.success(_BundleDebugSymbolsResponse)
689@faux.empty_error
690@validate.require('build_target.name', 'output_dir')
691@validate.exists('output_dir')
692@validate.validation_complete
693def BundleDebugSymbols(input_proto, output_proto, _config):
694 """Bundle the debug symbols into a tarball suitable for importing into GCE.
695
696 Args:
697 input_proto (BundleRequest): The input proto.
698 output_proto (BundleResponse): The output proto.
699 _config (api_config.ApiConfig): The API call config.
700 """
Michael Mortensen6667b542020-12-12 11:09:55 -0700701 output_dir = input_proto.output_dir
702
703 chroot = controller_util.ParseChroot(input_proto.chroot)
Michael Mortensen7da39cc2021-03-09 10:28:45 -0700704 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
Michael Mortensen6667b542020-12-12 11:09:55 -0700705 result = artifacts.GenerateBreakpadSymbols(chroot,
Michael Mortensen7da39cc2021-03-09 10:28:45 -0700706 build_target,
Michael Mortensen6667b542020-12-12 11:09:55 -0700707 debug=True)
708
709 # Verify breakpad symbol generation before gathering the sym files.
710 if result.returncode != 0:
711 return controller.RETURN_CODE_COMPLETED_UNSUCCESSFULLY
712
713 with chroot.tempdir() as symbol_tmpdir, chroot.tempdir() as dest_tmpdir:
Michael Mortensen7da39cc2021-03-09 10:28:45 -0700714 breakpad_dir = os.path.join('/build', build_target.name,
715 'usr/lib/debug/breakpad')
Michael Mortensen5044d272021-04-09 16:15:16 -0600716 # Call list on the atifacts.GatherSymbolFiles generator function to
717 # materialize and consume all entries so that all are copied to
718 # dest dir and complete list of all symbol files is returned.
719 sym_file_list = list(artifacts.GatherSymbolFiles(tempdir=symbol_tmpdir,
720 destdir=dest_tmpdir,
721 paths=[breakpad_dir]))
722 if not sym_file_list:
Michael Mortensen6667b542020-12-12 11:09:55 -0700723 logging.warning('No sym files found in %s.', breakpad_dir)
724 # Create tarball from destination_tmp, then copy it...
725 tarball_path = os.path.join(output_dir, constants.DEBUG_SYMBOLS_TAR)
726 result = cros_build_lib.CreateTarball(tarball_path, dest_tmpdir)
727 if result.returncode != 0:
728 logging.error('Error (%d) when creating tarball %s from %s',
729 result.returncode,
730 tarball_path,
731 dest_tmpdir)
732 return controller.RETURN_CODE_COMPLETED_UNSUCCESSFULLY
733 output_proto.artifacts.add().path = tarball_path
734
735 return controller.RETURN_CODE_SUCCESS