blob: cfa3773319db0e0256ac1e475f9322c1f037c8d8 [file] [log] [blame]
Evan Hernandezf388cbf2019-04-01 11:15:23 -06001# Copyright 2019 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Implements ArtifactService."""
6
Chris McDonald1672ddb2021-07-21 11:48:23 -06007import logging
Evan Hernandezf388cbf2019-04-01 11:15:23 -06008import os
Varun Somani04dccd72021-10-09 01:06:11 +00009from typing import Any, NamedTuple, Optional, TYPE_CHECKING
Evan Hernandezf388cbf2019-04-01 11:15:23 -060010
Alex Klein231d2da2019-07-22 16:44:45 -060011from chromite.api import controller
Alex Klein076841b2019-08-29 15:19:39 -060012from chromite.api import faux
Alex Klein2b236722019-06-19 15:44:26 -060013from chromite.api import validate
Alex Klein238d8862019-05-07 11:32:46 -060014from chromite.api.controller import controller_util
George Engelbrechtc9a8e812021-06-16 18:14:17 -060015from chromite.api.controller import image as image_controller
16from chromite.api.controller import sysroot as sysroot_controller
David Wellingc1433c22021-06-25 16:29:48 +000017from chromite.api.controller import test as test_controller
LaMont Jones58362a42021-02-04 17:40:08 -070018from chromite.api.gen.chromite.api import artifacts_pb2
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000019from chromite.api.gen.chromiumos import common_pb2
Alex Klein2275d692019-04-23 16:04:12 -060020from chromite.lib import chroot_lib
Evan Hernandezf388cbf2019-04-01 11:15:23 -060021from chromite.lib import constants
22from chromite.lib import cros_build_lib
Alex Klein2275d692019-04-23 16:04:12 -060023from chromite.lib import sysroot_lib
24from chromite.service import artifacts
Greg Edelstondc941072021-08-11 12:32:30 -060025from chromite.service import test
Evan Hernandezf388cbf2019-04-01 11:15:23 -060026
Varun Somani04dccd72021-10-09 01:06:11 +000027if TYPE_CHECKING:
28 from chromite.api import api_config
Evan Hernandezf388cbf2019-04-01 11:15:23 -060029
George Engelbrechtc9a8e812021-06-16 18:14:17 -060030class RegisteredGet(NamedTuple):
31 """An registered function for calling Get on an artifact type."""
32 output_proto: artifacts_pb2.GetResponse
33 artifact_dict: Any
LaMont Jones0f5171b2021-01-29 12:28:56 -070034
35
George Engelbrechtc9a8e812021-06-16 18:14:17 -060036def ExampleGetResponse(_input_proto, _output_proto, _config):
37 """Give an example GetResponse with a minimal coverage set."""
38 _output_proto = artifacts_pb2.GetResponse(
39 artifacts=common_pb2.UploadedArtifactsByService(
40 image=image_controller.ExampleGetResponse(),
41 sysroot=sysroot_controller.ExampleGetResponse(),
42 ))
43 return controller.RETURN_CODE_SUCCESS
44
45
LaMont Jones0f5171b2021-01-29 12:28:56 -070046@faux.empty_error
George Engelbrechtc9a8e812021-06-16 18:14:17 -060047@faux.success(ExampleGetResponse)
LaMont Jones0f5171b2021-01-29 12:28:56 -070048@validate.exists('result_path.path.path')
49@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +000050def Get(
51 input_proto: artifacts_pb2.GetRequest,
52 output_proto: artifacts_pb2.GetResponse,
53 _config: 'api_config.ApiConfig'):
LaMont Jones0f5171b2021-01-29 12:28:56 -070054 """Get all artifacts.
55
56 Get all artifacts for the build.
57
George Engelbrechtc9a8e812021-06-16 18:14:17 -060058 Note: As the individual artifact_type bundlers are added here, they *must*
59 stop uploading it via the individual bundler function.
LaMont Jones0f5171b2021-01-29 12:28:56 -070060
61 Args:
Varun Somani04dccd72021-10-09 01:06:11 +000062 input_proto: The input proto.
63 output_proto: The output proto.
64 _config: The API call config.
LaMont Jones0f5171b2021-01-29 12:28:56 -070065 """
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000066 output_dir = input_proto.result_path.path.path
67
George Engelbrechtc9a8e812021-06-16 18:14:17 -060068 sysroot = controller_util.ParseSysroot(input_proto.sysroot)
LaMont Jones8b88e9d2021-06-28 16:37:34 -060069 # This endpoint does not currently support any artifacts that are built
70 # without a sysroot being present.
71 if not sysroot.path:
72 return controller.RETURN_CODE_SUCCESS
73
George Engelbrechtc9a8e812021-06-16 18:14:17 -060074 chroot = controller_util.ParseChroot(input_proto.chroot)
75 build_target = controller_util.ParseBuildTarget(
76 input_proto.sysroot.build_target)
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000077
George Engelbrechtc9a8e812021-06-16 18:14:17 -060078 # A list of RegisteredGet tuples (input proto, output proto, get results).
79 get_res_list = [
80 RegisteredGet(
81 output_proto.artifacts.image,
82 image_controller.GetArtifacts(
83 input_proto.artifact_info.image, chroot, sysroot, build_target,
84 output_dir)),
85 RegisteredGet(
86 output_proto.artifacts.sysroot,
87 sysroot_controller.GetArtifacts(
88 input_proto.artifact_info.sysroot, chroot, sysroot, build_target,
David Wellingc1433c22021-06-25 16:29:48 +000089 output_dir)),
90 RegisteredGet(
91 output_proto.artifacts.test,
92 test_controller.GetArtifacts(
Jack Neusc9707c32021-07-23 21:48:54 +000093 input_proto.artifact_info.test, chroot, sysroot, build_target,
94 output_dir)),
George Engelbrechtc9a8e812021-06-16 18:14:17 -060095 ]
Jaques Clapauchf616bcd2021-04-09 20:14:40 +000096
George Engelbrechtc9a8e812021-06-16 18:14:17 -060097 for get_res in get_res_list:
98 for artifact_dict in get_res.artifact_dict:
99 get_res.output_proto.artifacts.add(
100 artifact_type=artifact_dict['type'],
101 paths=[
102 common_pb2.Path(
103 path=x, location=common_pb2.Path.Location.OUTSIDE)
104 for x in artifact_dict['paths']
105 ])
LaMont Jones0f5171b2021-01-29 12:28:56 -0700106 return controller.RETURN_CODE_SUCCESS
107
108
LaMont Jones58362a42021-02-04 17:40:08 -0700109def _BuildSetupResponse(_input_proto, output_proto, _config):
110 """Just return POINTLESS for now."""
111 # All of the artifact types we support claim that the build is POINTLESS.
112 output_proto.build_relevance = artifacts_pb2.BuildSetupResponse.POINTLESS
113
114
115@faux.success(_BuildSetupResponse)
116@faux.empty_error
117@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +0000118def BuildSetup(
119 _input_proto: artifacts_pb2.GetRequest,
120 output_proto: artifacts_pb2.GetResponse,
121 _config: 'api_config.ApiConfig'):
122
LaMont Jones58362a42021-02-04 17:40:08 -0700123 """Setup anything needed for building artifacts
124
125 If any artifact types require steps prior to building the package, they go
126 here. For example, see ToolchainService/PrepareForBuild.
127
128 Note: crbug/1034529 introduces this method as a noop. As the individual
129 artifact_type bundlers are added here, they *must* stop uploading it via the
130 individual bundler function.
131
132 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000133 _input_proto: The input proto.
134 output_proto: The output proto.
135 _config: The API call config.
LaMont Jones58362a42021-02-04 17:40:08 -0700136 """
137 # If any artifact_type says "NEEDED", the return is NEEDED.
138 # Otherwise, if any artifact_type says "UNKNOWN", the return is UNKNOWN.
139 # Otherwise, the return is POINTLESS.
140 output_proto.build_relevance = artifacts_pb2.BuildSetupResponse.POINTLESS
141 return controller.RETURN_CODE_SUCCESS
142
143
Varun Somani04dccd72021-10-09 01:06:11 +0000144def _GetImageDir(build_root: str, target: str) -> Optional[str]:
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600145 """Return path containing images for the given build target.
146
Alex Kleine2612a02019-04-18 13:51:06 -0600147 TODO(saklein) Expand image_lib.GetLatestImageLink to support this use case.
148
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600149 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000150 build_root: Path to checkout where build occurs.
151 target: Name of the build target.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600152
153 Returns:
Alex Kleind2bf1462019-10-24 16:37:04 -0600154 Path to the latest directory containing target images or None.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600155 """
156 image_dir = os.path.join(build_root, 'src/build/images', target, 'latest')
157 if not os.path.exists(image_dir):
Alex Kleind2bf1462019-10-24 16:37:04 -0600158 logging.warning('Expected to find image output for target %s at %s, but '
159 'path does not exist', target, image_dir)
160 return None
161
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600162 return image_dir
163
164
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700165def _BundleImageArchivesResponse(input_proto, output_proto, _config):
166 """Add artifact paths to a successful response."""
167 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
168 'path0.tar.xz')
169 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
170 'path1.tar.xz')
171
172
173@faux.success(_BundleImageArchivesResponse)
174@faux.empty_error
Alex Kleind91e95a2019-09-17 10:39:02 -0600175@validate.require('build_target.name')
176@validate.exists('output_dir')
177@validate.validation_complete
178def BundleImageArchives(input_proto, output_proto, _config):
179 """Create a .tar.xz archive for each image that has been created."""
180 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
181 output_dir = input_proto.output_dir
182 image_dir = _GetImageDir(constants.SOURCE_ROOT, build_target.name)
Alex Kleind2bf1462019-10-24 16:37:04 -0600183 if image_dir is None:
184 return
Alex Kleind91e95a2019-09-17 10:39:02 -0600185
186 archives = artifacts.ArchiveImages(image_dir, output_dir)
187
188 for archive in archives:
189 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
190
191
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700192def _BundleImageZipResponse(input_proto, output_proto, _config):
193 """Add artifact zip files to a successful response."""
194 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
195 'image.zip')
196
197
198@faux.success(_BundleImageZipResponse)
199@faux.empty_error
Michael Mortensen01910922019-07-24 14:48:10 -0600200@validate.require('build_target.name', 'output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600201@validate.exists('output_dir')
202@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +0000203def BundleImageZip(
204 input_proto: artifacts_pb2.BundleRequest,
205 output_proto: artifacts_pb2.BundleResponse,
206 _config: 'api_config.ApiConfig'):
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600207 """Bundle image.zip.
208
209 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000210 input_proto: The input proto.
211 output_proto: The output proto.
212 _config: The API call config.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600213 """
214 target = input_proto.build_target.name
215 output_dir = input_proto.output_dir
216 image_dir = _GetImageDir(constants.SOURCE_ROOT, target)
Alex Kleind2bf1462019-10-24 16:37:04 -0600217 if image_dir is None:
218 return None
Alex Klein231d2da2019-07-22 16:44:45 -0600219
Michael Mortensen01910922019-07-24 14:48:10 -0600220 archive = artifacts.BundleImageZip(output_dir, image_dir)
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600221 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
222
223
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700224def _BundleTestUpdatePayloadsResponse(input_proto, output_proto, _config):
225 """Add test payload files to a successful response."""
226 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
227 'payload1.bin')
George Engelbrechtf0239d52022-04-06 13:09:33 -0600228 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
229 'payload1.json')
230 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
231 'payload1.log')
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700232
233
234@faux.success(_BundleTestUpdatePayloadsResponse)
235@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600236@validate.require('build_target.name', 'output_dir')
237@validate.exists('output_dir')
238@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +0000239def BundleTestUpdatePayloads(
240 input_proto: artifacts_pb2.BundleRequest,
241 output_proto: artifacts_pb2.BundleResponse,
242 _config: 'api_config.ApiConfig'):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600243 """Generate minimal update payloads for the build target for testing.
244
245 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000246 input_proto: The input proto.
247 output_proto: The output proto.
248 _config: The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600249 """
250 target = input_proto.build_target.name
251 output_dir = input_proto.output_dir
252 build_root = constants.SOURCE_ROOT
253
254 # Use the first available image to create the update payload.
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600255 img_dir = _GetImageDir(build_root, target)
Alex Kleind2bf1462019-10-24 16:37:04 -0600256 if img_dir is None:
257 return None
258
Alex Kleincb541e82019-06-26 15:06:11 -0600259 img_types = [constants.IMAGE_TYPE_TEST, constants.IMAGE_TYPE_DEV,
260 constants.IMAGE_TYPE_BASE]
261 img_names = [constants.IMAGE_TYPE_TO_NAME[t] for t in img_types]
Mike Frysinger66ce4132019-07-17 22:52:52 -0400262 img_paths = [os.path.join(img_dir, x) for x in img_names]
Mike Frysingera552be42018-08-17 14:39:32 -0400263 valid_images = [x for x in img_paths if os.path.exists(x)]
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600264
Alex Kleincb541e82019-06-26 15:06:11 -0600265 if not valid_images:
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600266 cros_build_lib.Die(
267 'Expected to find an image of type among %r for target "%s" '
Evan Hernandez9f125ac2019-04-08 17:18:47 -0600268 'at path %s.', img_types, target, img_dir)
Alex Kleincb541e82019-06-26 15:06:11 -0600269 image = valid_images[0]
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600270
Alex Kleincb541e82019-06-26 15:06:11 -0600271 payloads = artifacts.BundleTestUpdatePayloads(image, output_dir)
272 for payload in payloads:
273 output_proto.artifacts.add().path = payload
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600274
275
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700276def _BundleAutotestFilesResponse(input_proto, output_proto, _config):
277 """Add test autotest files to a successful response."""
278 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
279 'autotest-a.tar.gz')
280
281
282@faux.success(_BundleAutotestFilesResponse)
283@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600284@validate.require('output_dir')
285@validate.exists('output_dir')
Varun Somani04dccd72021-10-09 01:06:11 +0000286def BundleAutotestFiles(
287 input_proto: artifacts_pb2.BundleRequest,
288 output_proto: artifacts_pb2.BundleResponse,
289 config: 'api_config.ApiConfig'):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600290 """Tar the autotest files for a build target.
291
292 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000293 input_proto: The input proto.
294 output_proto: The output proto.
295 config: The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600296 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600297 output_dir = input_proto.output_dir
Alex Klein238d8862019-05-07 11:32:46 -0600298 target = input_proto.build_target.name
Alex Kleine21a0952019-08-23 16:08:16 -0600299 chroot = controller_util.ParseChroot(input_proto.chroot)
300
Alex Klein238d8862019-05-07 11:32:46 -0600301 if target:
Alex Kleine21a0952019-08-23 16:08:16 -0600302 sysroot_path = os.path.join('/build', target)
Alex Klein238d8862019-05-07 11:32:46 -0600303 else:
304 # New style call, use chroot and sysroot.
Alex Klein238d8862019-05-07 11:32:46 -0600305 sysroot_path = input_proto.sysroot.path
306 if not sysroot_path:
307 cros_build_lib.Die('sysroot.path is required.')
308
Alex Kleine21a0952019-08-23 16:08:16 -0600309 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein238d8862019-05-07 11:32:46 -0600310
Alex Klein231d2da2019-07-22 16:44:45 -0600311 # TODO(saklein): Switch to the validate_only decorator when legacy handling
312 # is removed.
313 if config.validate_only:
314 return controller.RETURN_CODE_VALID_INPUT
315
Alex Kleine21a0952019-08-23 16:08:16 -0600316 if not sysroot.Exists(chroot=chroot):
Alex Klein238d8862019-05-07 11:32:46 -0600317 cros_build_lib.Die('Sysroot path must exist: %s', sysroot.path)
318
319 try:
320 # Note that this returns the full path to *multiple* tarballs.
Alex Kleine21a0952019-08-23 16:08:16 -0600321 archives = artifacts.BundleAutotestFiles(chroot, sysroot, output_dir)
Alex Klein238d8862019-05-07 11:32:46 -0600322 except artifacts.Error as e:
Alex Klein03890312020-06-30 09:59:50 -0600323 logging.warning(e)
324 return
Alex Klein238d8862019-05-07 11:32:46 -0600325
326 for archive in archives.values():
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600327 output_proto.artifacts.add().path = archive
328
329
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700330def _BundleTastFilesResponse(input_proto, output_proto, _config):
331 """Add test tast files to a successful response."""
332 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
333 'tast_bundles.tar.gz')
334
335
336@faux.success(_BundleTastFilesResponse)
337@faux.empty_error
Alex Kleinb9d810b2019-07-01 12:38:02 -0600338@validate.require('output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600339@validate.exists('output_dir')
Varun Somani04dccd72021-10-09 01:06:11 +0000340def BundleTastFiles(
341 input_proto: artifacts_pb2.BundleRequest,
342 output_proto: artifacts_pb2.BundleResponse,
343 config: 'api_config.ApiConfig'):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600344 """Tar the tast files for a build target.
345
346 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000347 input_proto: The input proto.
348 output_proto: The output proto.
349 config: The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600350 """
351 target = input_proto.build_target.name
352 output_dir = input_proto.output_dir
353 build_root = constants.SOURCE_ROOT
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600354
Alex Kleinb9d810b2019-07-01 12:38:02 -0600355 chroot = controller_util.ParseChroot(input_proto.chroot)
356 sysroot_path = input_proto.sysroot.path
357
358 # TODO(saklein) Cleanup legacy handling after it has been switched over.
359 if target:
360 # Legacy handling.
Alex Klein171da612019-08-06 14:00:42 -0600361 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
Alex Kleinb9d810b2019-07-01 12:38:02 -0600362 sysroot_path = os.path.join('/build', target)
363
364 # New handling - chroot & sysroot based.
365 # TODO(saklein) Switch this to the require decorator when legacy is removed.
366 if not sysroot_path:
367 cros_build_lib.Die('sysroot.path is required.')
368
Alex Klein231d2da2019-07-22 16:44:45 -0600369 # TODO(saklein): Switch to the validation_complete decorator when legacy
370 # handling is removed.
371 if config.validate_only:
372 return controller.RETURN_CODE_VALID_INPUT
373
Alex Kleinb9d810b2019-07-01 12:38:02 -0600374 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein231d2da2019-07-22 16:44:45 -0600375 if not sysroot.Exists(chroot=chroot):
Alex Kleinb9d810b2019-07-01 12:38:02 -0600376 cros_build_lib.Die('Sysroot must exist.')
377
378 archive = artifacts.BundleTastFiles(chroot, sysroot, output_dir)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600379
LaMont Jonesb9793cd2020-06-11 08:14:46 -0600380 if archive:
381 output_proto.artifacts.add().path = archive
382 else:
383 logging.warning('Found no tast files for %s.', target)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600384
385
Fergus Dall34d74e12020-09-29 15:52:32 +1000386def BundlePinnedGuestImages(_input_proto, _output_proto, _config):
387 # TODO(crbug/1034529): Remove this endpoint
388 pass
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700389
Fergus Dall34d74e12020-09-29 15:52:32 +1000390def FetchPinnedGuestImageUris(_input_proto, _output_proto, _config):
391 # TODO(crbug/1034529): Remove this endpoint
392 pass
Alex Klein7bf0ecb2019-06-25 10:04:15 -0600393
394
Greg Edelstondc941072021-08-11 12:32:30 -0600395def _FetchMetadataResponse(_input_proto, output_proto, _config):
396 """Populate the output_proto with sample data."""
397 for fp in ('/metadata/foo.txt', '/metadata/bar.jsonproto'):
398 output_proto.filepaths.add(path=common_pb2.Path(
399 path=fp, location=common_pb2.Path.OUTSIDE))
400 return controller.RETURN_CODE_SUCCESS
401
402
403@faux.success(_FetchMetadataResponse)
404@faux.empty_error
405@validate.exists('chroot.path')
406@validate.require('sysroot.path')
407@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +0000408def FetchMetadata(
409 input_proto: artifacts_pb2.FetchMetadataRequest,
410 output_proto: artifacts_pb2.FetchMetadataResponse,
411 _config: 'api_config.ApiConfig'):
Greg Edelstondc941072021-08-11 12:32:30 -0600412 """FetchMetadata returns the paths to all build/test metadata files.
413
414 This implements ArtifactsService.FetchMetadata.
415
416 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000417 input_proto: The input proto.
418 output_proto: The output proto.
419 config: The API call config.
Greg Edelstondc941072021-08-11 12:32:30 -0600420 """
421 chroot = controller_util.ParseChroot(input_proto.chroot)
422 sysroot = controller_util.ParseSysroot(input_proto.sysroot)
423 for path in test.FindAllMetadataFiles(chroot, sysroot):
424 output_proto.filepaths.add(
425 path=common_pb2.Path(path=path, location=common_pb2.Path.OUTSIDE))
426 return controller.RETURN_CODE_SUCCESS
427
428
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700429def _BundleFirmwareResponse(input_proto, output_proto, _config):
430 """Add test firmware image files to a successful response."""
431 output_proto.artifacts.add().path = os.path.join(
432 input_proto.output_dir, 'firmware.tar.gz')
433
434
435@faux.success(_BundleFirmwareResponse)
436@faux.empty_error
Michael Mortensen38675192019-06-28 16:52:55 +0000437@validate.require('output_dir', 'sysroot.path')
Alex Klein231d2da2019-07-22 16:44:45 -0600438@validate.exists('output_dir')
439@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +0000440def BundleFirmware(
441 input_proto: artifacts_pb2.BundleRequest,
442 output_proto: artifacts_pb2.BundleResponse,
443 _config: 'api_config.ApiConfig'):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600444 """Tar the firmware images for a build target.
445
446 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000447 input_proto: The input proto.
448 output_proto: The output proto.
449 _config: The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600450 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600451 output_dir = input_proto.output_dir
Michael Mortensen38675192019-06-28 16:52:55 +0000452 chroot = controller_util.ParseChroot(input_proto.chroot)
453 sysroot_path = input_proto.sysroot.path
454 sysroot = sysroot_lib.Sysroot(sysroot_path)
Alex Klein231d2da2019-07-22 16:44:45 -0600455
456 if not chroot.exists():
457 cros_build_lib.Die('Chroot does not exist: %s', chroot.path)
458 elif not sysroot.Exists(chroot=chroot):
459 cros_build_lib.Die('Sysroot does not exist: %s',
460 chroot.full_path(sysroot.path))
461
Michael Mortensen38675192019-06-28 16:52:55 +0000462 archive = artifacts.BuildFirmwareArchive(chroot, sysroot, output_dir)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600463
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600464 if archive is None:
George Engelbrecht9e41e172021-11-18 17:04:22 -0700465 logging.warning(
Michael Mortensen38675192019-06-28 16:52:55 +0000466 'Could not create firmware archive. No firmware found for %s.',
467 sysroot_path)
George Engelbrecht9e41e172021-11-18 17:04:22 -0700468 return
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600469
Alex Klein231d2da2019-07-22 16:44:45 -0600470 output_proto.artifacts.add().path = archive
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600471
472
Yicheng Liea1181f2020-09-22 11:51:10 -0700473def _BundleFpmcuUnittestsResponse(input_proto, output_proto, _config):
474 """Add fingerprint MCU unittest binaries to a successful response."""
475 output_proto.artifacts.add().path = os.path.join(
476 input_proto.output_dir, 'fpmcu_unittests.tar.gz')
477
478
479@faux.success(_BundleFpmcuUnittestsResponse)
480@faux.empty_error
481@validate.require('output_dir', 'sysroot.path')
482@validate.exists('output_dir')
483@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +0000484def BundleFpmcuUnittests(
485 input_proto: artifacts_pb2.BundleRequest,
486 output_proto: artifacts_pb2.BundleResponse,
487 _config: 'api_config.ApiConfig'):
Yicheng Liea1181f2020-09-22 11:51:10 -0700488 """Tar the fingerprint MCU unittest binaries for a build target.
489
490 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000491 input_proto: The input proto.
492 output_proto: The output proto.
493 _config: The API call config.
Yicheng Liea1181f2020-09-22 11:51:10 -0700494 """
495 output_dir = input_proto.output_dir
496 chroot = controller_util.ParseChroot(input_proto.chroot)
497 sysroot_path = input_proto.sysroot.path
498 sysroot = sysroot_lib.Sysroot(sysroot_path)
499
500 if not chroot.exists():
501 cros_build_lib.Die('Chroot does not exist: %s', chroot.path)
502 elif not sysroot.Exists(chroot=chroot):
503 cros_build_lib.Die('Sysroot does not exist: %s',
504 chroot.full_path(sysroot.path))
505
506 archive = artifacts.BundleFpmcuUnittests(chroot, sysroot, output_dir)
507
508 if archive is None:
509 logging.warning(
510 'No fpmcu unittests found for %s.', sysroot_path)
511 return
512
513 output_proto.artifacts.add().path = archive
514
515
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700516def _BundleEbuildLogsResponse(input_proto, output_proto, _config):
517 """Add test log files to a successful response."""
518 output_proto.artifacts.add().path = os.path.join(
519 input_proto.output_dir, 'ebuild-logs.tar.gz')
520
521
522@faux.success(_BundleEbuildLogsResponse)
523@faux.empty_error
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600524@validate.exists('output_dir')
Varun Somani04dccd72021-10-09 01:06:11 +0000525def BundleEbuildLogs(
526 input_proto: artifacts_pb2.BundleRequest,
527 output_proto: artifacts_pb2.BundleResponse,
528 config: 'api_config.ApiConfig'):
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600529 """Tar the ebuild logs for a build target.
530
531 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000532 input_proto: The input proto.
533 output_proto: The output proto.
534 config: The API call config.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600535 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600536 output_dir = input_proto.output_dir
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600537 sysroot_path = input_proto.sysroot.path
538 chroot = controller_util.ParseChroot(input_proto.chroot)
Evan Hernandeza478d802019-04-08 15:08:24 -0600539
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600540 # TODO(mmortensen) Cleanup legacy handling after it has been switched over.
541 target = input_proto.build_target.name
542 if target:
543 # Legacy handling.
544 build_root = constants.SOURCE_ROOT
Alex Klein171da612019-08-06 14:00:42 -0600545 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600546 sysroot_path = os.path.join('/build', target)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600547
Alex Klein231d2da2019-07-22 16:44:45 -0600548 # TODO(saklein): Switch to validation_complete decorator after legacy
549 # handling has been cleaned up.
550 if config.validate_only:
551 return controller.RETURN_CODE_VALID_INPUT
552
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600553 sysroot = sysroot_lib.Sysroot(sysroot_path)
554 archive = artifacts.BundleEBuildLogsTarball(chroot, sysroot, output_dir)
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600555 if archive is None:
556 cros_build_lib.Die(
Michael Mortensen3f382cb2019-07-29 13:21:49 -0600557 'Could not create ebuild logs archive. No logs found for %s.',
558 sysroot.path)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600559 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
Alex Klein6504eca2019-04-18 15:37:56 -0600560
561
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700562def _BundleChromeOSConfigResponse(input_proto, output_proto, _config):
563 """Add test config files to a successful response."""
564 output_proto.artifacts.add().path = os.path.join(
565 input_proto.output_dir, 'config.yaml')
566
567
568@faux.success(_BundleChromeOSConfigResponse)
569@faux.empty_error
Andrew Lamb811aead2019-08-12 10:25:05 -0600570@validate.exists('output_dir')
571@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +0000572def BundleChromeOSConfig(
573 input_proto: artifacts_pb2.BundleRequest,
574 output_proto: artifacts_pb2.BundleResponse,
575 _config: 'api_config.ApiConfig'):
Andrew Lamb811aead2019-08-12 10:25:05 -0600576 """Output the ChromeOS Config payload for a build target.
577
578 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000579 input_proto: The input proto.
580 output_proto: The output proto.
581 _config: The API call config.
Andrew Lamb811aead2019-08-12 10:25:05 -0600582 """
583 output_dir = input_proto.output_dir
Andrew Lamb67bd68f2019-08-15 09:09:15 -0600584 sysroot_path = input_proto.sysroot.path
Andrew Lamb811aead2019-08-12 10:25:05 -0600585 chroot = controller_util.ParseChroot(input_proto.chroot)
586
Andrew Lamb67bd68f2019-08-15 09:09:15 -0600587 # TODO(mmortensen) Cleanup legacy handling after it has been switched over.
588 target = input_proto.build_target.name
589 if target:
590 # Legacy handling.
591 build_root = constants.SOURCE_ROOT
592 chroot = chroot_lib.Chroot(path=os.path.join(build_root, 'chroot'))
593 sysroot_path = os.path.join('/build', target)
594
595 sysroot = sysroot_lib.Sysroot(sysroot_path)
Andrew Lamb811aead2019-08-12 10:25:05 -0600596 chromeos_config = artifacts.BundleChromeOSConfig(chroot, sysroot, output_dir)
Alex Klein383a7a32021-12-07 16:01:19 -0700597 if not chromeos_config:
598 return
599
Andrew Lamb811aead2019-08-12 10:25:05 -0600600 output_proto.artifacts.add().path = os.path.join(output_dir, chromeos_config)
601
602
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700603def _BundleSimpleChromeArtifactsResponse(input_proto, output_proto, _config):
604 """Add test simple chrome files to a successful response."""
605 output_proto.artifacts.add().path = os.path.join(
606 input_proto.output_dir, 'simple_chrome.txt')
607
608
609@faux.success(_BundleSimpleChromeArtifactsResponse)
610@faux.empty_error
Alex Klein231d2da2019-07-22 16:44:45 -0600611@validate.require('output_dir', 'sysroot.build_target.name', 'sysroot.path')
612@validate.exists('output_dir')
613@validate.validation_complete
614def BundleSimpleChromeArtifacts(input_proto, output_proto, _config):
Alex Klein2275d692019-04-23 16:04:12 -0600615 """Create the simple chrome artifacts."""
Alex Klein2275d692019-04-23 16:04:12 -0600616 sysroot_path = input_proto.sysroot.path
Alex Klein2275d692019-04-23 16:04:12 -0600617 output_dir = input_proto.output_dir
618
Alex Klein2275d692019-04-23 16:04:12 -0600619 # Build out the argument instances.
Alex Klein26e472b2020-03-10 14:35:01 -0600620 build_target = controller_util.ParseBuildTarget(
621 input_proto.sysroot.build_target)
622 chroot = controller_util.ParseChroot(input_proto.chroot)
Alex Klein2275d692019-04-23 16:04:12 -0600623 # Sysroot.path needs to be the fully qualified path, including the chroot.
624 full_sysroot_path = os.path.join(chroot.path, sysroot_path.lstrip(os.sep))
625 sysroot = sysroot_lib.Sysroot(full_sysroot_path)
626
627 # Quick sanity check that the sysroot exists before we go on.
628 if not sysroot.Exists():
629 cros_build_lib.Die('The sysroot does not exist.')
630
631 try:
632 results = artifacts.BundleSimpleChromeArtifacts(chroot, sysroot,
633 build_target, output_dir)
634 except artifacts.Error as e:
635 cros_build_lib.Die('Error %s raised in BundleSimpleChromeArtifacts: %s',
636 type(e), e)
637
638 for file_name in results:
639 output_proto.artifacts.add().path = file_name
640
641
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700642def _BundleVmFilesResponse(input_proto, output_proto, _config):
643 """Add test vm files to a successful response."""
644 output_proto.artifacts.add().path = os.path.join(
645 input_proto.output_dir, 'f1.tar')
646
647
648@faux.success(_BundleVmFilesResponse)
649@faux.empty_error
Michael Mortensen51f06722019-07-18 09:55:50 -0600650@validate.require('chroot.path', 'test_results_dir', 'output_dir')
Alex Klein231d2da2019-07-22 16:44:45 -0600651@validate.exists('output_dir')
652@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +0000653def BundleVmFiles(
654 input_proto: artifacts_pb2.BundleVmFilesRequest,
655 output_proto: artifacts_pb2.BundleResponse,
656 _config: 'api_config.ApiConfig'):
Alex Klein6504eca2019-04-18 15:37:56 -0600657 """Tar VM disk and memory files.
658
659 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000660 input_proto: The input proto.
661 output_proto: The output proto.
662 _config: The API call config.
Alex Klein6504eca2019-04-18 15:37:56 -0600663 """
Michael Mortensen51f06722019-07-18 09:55:50 -0600664 chroot = controller_util.ParseChroot(input_proto.chroot)
665 test_results_dir = input_proto.test_results_dir
Alex Klein6504eca2019-04-18 15:37:56 -0600666 output_dir = input_proto.output_dir
667
Michael Mortensen51f06722019-07-18 09:55:50 -0600668 archives = artifacts.BundleVmFiles(
669 chroot, test_results_dir, output_dir)
Alex Klein6504eca2019-04-18 15:37:56 -0600670 for archive in archives:
671 output_proto.artifacts.add().path = archive
Tiancong Wangc4805b72019-06-11 12:12:03 -0700672
Michael Mortensen2d6a2402019-11-26 13:40:40 -0700673def _ExportCpeReportResponse(input_proto, output_proto, _config):
674 """Add test cpe results to a successful response."""
675 output_proto.artifacts.add().path = os.path.join(
676 input_proto.output_dir, 'cpe_report.txt')
677 output_proto.artifacts.add().path = os.path.join(
678 input_proto.output_dir, 'cpe_warnings.txt')
679
680
681@faux.success(_ExportCpeReportResponse)
682@faux.empty_error
Alex Klein0b1cbfc2019-08-14 10:09:58 -0600683@validate.exists('output_dir')
Varun Somani04dccd72021-10-09 01:06:11 +0000684def ExportCpeReport(
685 input_proto: artifacts_pb2.BundleRequest,
686 output_proto: artifacts_pb2.BundleResponse,
687 config: 'api_config.ApiConfig'):
Alex Klein0b1cbfc2019-08-14 10:09:58 -0600688 """Export a CPE report.
689
690 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000691 input_proto: The input proto.
692 output_proto: The output proto.
693 config: The API call config.
Alex Klein0b1cbfc2019-08-14 10:09:58 -0600694 """
695 chroot = controller_util.ParseChroot(input_proto.chroot)
696 output_dir = input_proto.output_dir
697
698 if input_proto.build_target.name:
699 # Legacy handling - use the default sysroot path for the build target.
700 build_target = controller_util.ParseBuildTarget(input_proto.build_target)
701 sysroot = sysroot_lib.Sysroot(build_target.root)
702 elif input_proto.sysroot.path:
703 sysroot = sysroot_lib.Sysroot(input_proto.sysroot.path)
704 else:
705 # TODO(saklein): Switch to validate decorators once legacy handling can be
706 # cleaned up.
707 cros_build_lib.Die('sysroot.path is required.')
708
709 if config.validate_only:
710 return controller.RETURN_CODE_VALID_INPUT
711
712 cpe_result = artifacts.GenerateCpeReport(chroot, sysroot, output_dir)
713
714 output_proto.artifacts.add().path = cpe_result.report
715 output_proto.artifacts.add().path = cpe_result.warnings
Shao-Chuan Leea44dddc2020-10-30 17:16:55 +0900716
717
718def _BundleGceTarballResponse(input_proto, output_proto, _config):
719 """Add artifact tarball to a successful response."""
720 output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
721 constants.TEST_IMAGE_GCE_TAR)
722
723
724@faux.success(_BundleGceTarballResponse)
725@faux.empty_error
726@validate.require('build_target.name', 'output_dir')
727@validate.exists('output_dir')
728@validate.validation_complete
Varun Somani04dccd72021-10-09 01:06:11 +0000729def BundleGceTarball(
730 input_proto: artifacts_pb2.BundleRequest,
731 output_proto: artifacts_pb2.BundleResponse,
732 _config: 'api_config.ApiConfig'):
Shao-Chuan Leea44dddc2020-10-30 17:16:55 +0900733 """Bundle the test image into a tarball suitable for importing into GCE.
734
735 Args:
Varun Somani04dccd72021-10-09 01:06:11 +0000736 input_proto: The input proto.
737 output_proto: The output proto.
738 _config: The API call config.
Shao-Chuan Leea44dddc2020-10-30 17:16:55 +0900739 """
740 target = input_proto.build_target.name
741 output_dir = input_proto.output_dir
742 image_dir = _GetImageDir(constants.SOURCE_ROOT, target)
743 if image_dir is None:
744 return None
745
746 tarball = artifacts.BundleGceTarball(output_dir, image_dir)
747 output_proto.artifacts.add().path = tarball