blob: ddee4f6de81b51d59f38183147762be66f93e81b [file] [log] [blame]
Evan Hernandezf388cbf2019-04-01 11:15:23 -06001# -*- coding: utf-8 -*-
2# Copyright 2019 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Implements ArtifactService."""
7
8from __future__ import print_function
9
10import os
11
Alex Klein2b236722019-06-19 15:44:26 -060012from chromite.api import validate
Alex Klein238d8862019-05-07 11:32:46 -060013from chromite.api.controller import controller_util
Evan Hernandezf388cbf2019-04-01 11:15:23 -060014from chromite.cbuildbot import commands
Alex Klein6504eca2019-04-18 15:37:56 -060015from chromite.cbuildbot.stages import vm_test_stages
Alex Klein2275d692019-04-23 16:04:12 -060016from chromite.lib import build_target_util
17from chromite.lib import chroot_lib
Evan Hernandezf388cbf2019-04-01 11:15:23 -060018from chromite.lib import constants
19from chromite.lib import cros_build_lib
Evan Hernandezde445982019-04-22 13:42:34 -060020from chromite.lib import cros_logging as logging
Evan Hernandezf388cbf2019-04-01 11:15:23 -060021from chromite.lib import osutils
Alex Klein2275d692019-04-23 16:04:12 -060022from chromite.lib import sysroot_lib
23from chromite.service import artifacts
Evan Hernandezf388cbf2019-04-01 11:15:23 -060024
25
Evan Hernandez9f125ac2019-04-08 17:18:47 -060026def _GetImageDir(build_root, target):
27 """Return path containing images for the given build target.
28
Alex Kleine2612a02019-04-18 13:51:06 -060029 TODO(saklein) Expand image_lib.GetLatestImageLink to support this use case.
30
Evan Hernandez9f125ac2019-04-08 17:18:47 -060031 Args:
32 build_root (str): Path to checkout where build occurs.
33 target (str): Name of the build target.
34
35 Returns:
36 Path to the directory containing target images.
37
38 Raises:
39 DieSystemExit: If the image dir does not exist.
40 """
41 image_dir = os.path.join(build_root, 'src/build/images', target, 'latest')
42 if not os.path.exists(image_dir):
43 cros_build_lib.Die('Expected to find image output for target %s at %s, '
44 'but path does not exist' % (target, image_dir))
45 return image_dir
46
47
48def BundleImageZip(input_proto, output_proto):
49 """Bundle image.zip.
50
51 Args:
52 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -060053 output_proto (BundleResponse): The output proto.
Evan Hernandez9f125ac2019-04-08 17:18:47 -060054 """
55 target = input_proto.build_target.name
56 output_dir = input_proto.output_dir
57 image_dir = _GetImageDir(constants.SOURCE_ROOT, target)
58 archive = commands.BuildImageZip(output_dir, image_dir)
59 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
60
61
Evan Hernandezf388cbf2019-04-01 11:15:23 -060062def BundleTestUpdatePayloads(input_proto, output_proto):
63 """Generate minimal update payloads for the build target for testing.
64
65 Args:
66 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -060067 output_proto (BundleResponse): The output proto.
Evan Hernandezf388cbf2019-04-01 11:15:23 -060068 """
69 target = input_proto.build_target.name
70 output_dir = input_proto.output_dir
71 build_root = constants.SOURCE_ROOT
72
73 # Use the first available image to create the update payload.
Evan Hernandez9f125ac2019-04-08 17:18:47 -060074 img_dir = _GetImageDir(build_root, target)
Evan Hernandezf388cbf2019-04-01 11:15:23 -060075 img_types = [
76 constants.IMAGE_TYPE_TEST, constants.IMAGE_TYPE_DEV,
77 constants.IMAGE_TYPE_BASE
78 ]
79 img_paths = []
80 for img_type in img_types:
Evan Hernandez9f125ac2019-04-08 17:18:47 -060081 img_path = os.path.join(img_dir, constants.IMAGE_TYPE_TO_NAME[img_type])
Evan Hernandezf388cbf2019-04-01 11:15:23 -060082 if os.path.exists(img_path):
83 img_paths.append(img_path)
84
85 if not img_paths:
86 cros_build_lib.Die(
87 'Expected to find an image of type among %r for target "%s" '
Evan Hernandez9f125ac2019-04-08 17:18:47 -060088 'at path %s.', img_types, target, img_dir)
Evan Hernandezf388cbf2019-04-01 11:15:23 -060089 img = img_paths[0]
90
91 # Unfortunately, the relevant commands.py functions do not return
92 # a list of generated files. As a workaround, we have commands.py
93 # put the files in a separate temporary directory so we can catalog them,
94 # then move them to the output dir.
Alex Kleine2612a02019-04-18 13:51:06 -060095 # TODO(crbug.com/954283): Replace with a chromite/service implementation.
Evan Hernandezf388cbf2019-04-01 11:15:23 -060096 with osutils.TempDir() as temp:
97 commands.GeneratePayloads(img, temp, full=True, stateful=True, delta=True)
98 commands.GenerateQuickProvisionPayloads(img, temp)
99 for path in osutils.DirectoryIterator(temp):
100 if os.path.isfile(path):
101 rel_path = os.path.relpath(path, temp)
102 output_proto.artifacts.add().path = os.path.join(output_dir, rel_path)
Evan Hernandezb04e2aa2019-04-08 16:55:54 -0600103 osutils.CopyDirContents(temp, output_dir, allow_nonempty=True)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600104
105
106def BundleAutotestFiles(input_proto, output_proto):
107 """Tar the autotest files for a build target.
108
109 Args:
110 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600111 output_proto (BundleResponse): The output proto.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600112 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600113 output_dir = input_proto.output_dir
Alex Klein238d8862019-05-07 11:32:46 -0600114 if not output_dir:
115 cros_build_lib.Die('output_dir is required.')
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600116
Alex Klein238d8862019-05-07 11:32:46 -0600117 target = input_proto.build_target.name
118 if target:
119 # Legacy call, build out sysroot path from default source root and the
120 # build target.
121 target = input_proto.build_target.name
122 build_root = constants.SOURCE_ROOT
123 sysroot_path = os.path.join(build_root, constants.DEFAULT_CHROOT_DIR,
124 'build', target)
125 sysroot = sysroot_lib.Sysroot(sysroot_path)
126 else:
127 # New style call, use chroot and sysroot.
128 chroot = controller_util.ParseChroot(input_proto.chroot)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600129
Alex Klein238d8862019-05-07 11:32:46 -0600130 sysroot_path = input_proto.sysroot.path
131 if not sysroot_path:
132 cros_build_lib.Die('sysroot.path is required.')
133
134 # Since we're staying outside the chroot, prepend the chroot path to the
135 # sysroot path so we have a valid full path to the sysroot.
136 sysroot = sysroot_lib.Sysroot(os.path.join(chroot.path,
137 sysroot_path.lstrip(os.sep)))
138
139 if not sysroot.Exists():
140 cros_build_lib.Die('Sysroot path must exist: %s', sysroot.path)
141
142 try:
143 # Note that this returns the full path to *multiple* tarballs.
144 archives = artifacts.BundleAutotestFiles(sysroot, output_dir)
145 except artifacts.Error as e:
146 cros_build_lib.Die(e.message)
147
148 for archive in archives.values():
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600149 output_proto.artifacts.add().path = archive
150
151
152def BundleTastFiles(input_proto, output_proto):
153 """Tar the tast files for a build target.
154
155 Args:
156 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600157 output_proto (BundleResponse): The output proto.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600158 """
159 target = input_proto.build_target.name
160 output_dir = input_proto.output_dir
161 build_root = constants.SOURCE_ROOT
Evan Hernandez36589c62019-04-05 18:14:42 -0600162 cwd = os.path.join(build_root, 'chroot/build', target, 'build')
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600163
164 # Note that unlike the functions below, this returns the full path
165 # to the tarball.
Alex Kleine2612a02019-04-18 13:51:06 -0600166 # TODO(crbug.com/954294): Replace with a chromite/service implementation.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600167 archive = commands.BuildTastBundleTarball(build_root, cwd, output_dir)
168
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600169 if archive is None:
170 cros_build_lib.Die(
171 'Could not bundle Tast files. '
172 'No Tast directories found for %s.', target)
173
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600174 output_proto.artifacts.add().path = archive
175
176
177def BundlePinnedGuestImages(input_proto, output_proto):
178 """Tar the pinned guest images for a build target.
179
180 Args:
181 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600182 output_proto (BundleResponse): The output proto.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600183 """
184 target = input_proto.build_target.name
185 output_dir = input_proto.output_dir
186 build_root = constants.SOURCE_ROOT
187
Alex Kleine2612a02019-04-18 13:51:06 -0600188 # TODO(crbug.com/954299): Replace with a chromite/service implementation.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600189 archive = commands.BuildPinnedGuestImagesTarball(build_root, target,
190 output_dir)
191
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600192 if archive is None:
Evan Hernandezde445982019-04-22 13:42:34 -0600193 logging.warning('Found no pinned guest images for %s.', target)
194 return
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600195
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600196 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
197
198
Alex Klein7bf0ecb2019-06-25 10:04:15 -0600199def FetchPinnedGuestImages(input_proto, output_proto):
200 """Get the pinned guest image information."""
201 sysroot_path = input_proto.sysroot.path
202 if not sysroot_path:
203 cros_build_lib.Die('sysroot.path is required.')
204
205 chroot = controller_util.ParseChroot(input_proto.chroot)
206 sysroot = sysroot_lib.Sysroot(sysroot_path)
207
208 if not chroot.exists():
209 cros_build_lib.Die('Chroot does not exist: %s', chroot.path)
210
211 if not sysroot.Exists(chroot=chroot):
212 cros_build_lib.Die('Sysroot does not exist: %s',
213 chroot.full_path(sysroot.path))
214
215 pins = artifacts.FetchPinnedGuestImages(chroot, sysroot)
216
217 for pin in pins:
218 pinned_image = output_proto.pinned_images.add()
219 pinned_image.filename = pin.filename
220 pinned_image.uri = pin.uri
221
222
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600223def BundleFirmware(input_proto, output_proto):
224 """Tar the firmware images for a build target.
225
226 Args:
227 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600228 output_proto (BundleResponse): The output proto.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600229 """
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600230 output_dir = input_proto.output_dir
Michael Mortensen3df9c762019-06-19 07:50:09 -0600231 if not output_dir:
232 cros_build_lib.Die('output_dir is required.')
233 chroot = controller_util.ParseChroot(input_proto.chroot)
234 sysroot_path = input_proto.sysroot.path
235 if not sysroot_path:
236 cros_build_lib.Die('sysroot.path is required.')
237 sysroot = sysroot_lib.Sysroot(sysroot_path)
238 archive = artifacts.BuildFirmwareArchive(chroot, sysroot, output_dir)
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600239
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600240 if archive is None:
241 cros_build_lib.Die(
Michael Mortensen3df9c762019-06-19 07:50:09 -0600242 'Could not create firmware archive. No firmware found for %s.',
243 sysroot_path)
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600244
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600245 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
246
247
248def BundleEbuildLogs(input_proto, output_proto):
249 """Tar the ebuild logs for a build target.
250
251 Args:
252 input_proto (BundleRequest): The input proto.
Alex Klein6504eca2019-04-18 15:37:56 -0600253 output_proto (BundleResponse): The output proto.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600254 """
255 target = input_proto.build_target.name
256 output_dir = input_proto.output_dir
Evan Hernandeza478d802019-04-08 15:08:24 -0600257
258 # commands.BuildEbuildLogsTarball conflates "buildroot" with sysroot.
259 # Adjust accordingly...
260 build_root = os.path.join(constants.SOURCE_ROOT, 'chroot', 'build')
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600261
Alex Kleine2612a02019-04-18 13:51:06 -0600262 # TODO(crbug.com/954303): Replace with a chromite/service implementation.
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600263 archive = commands.BuildEbuildLogsTarball(build_root, target, output_dir)
264
Evan Hernandez9a5d3122019-04-09 10:51:23 -0600265 if archive is None:
266 cros_build_lib.Die(
267 'Could not create ebuild logs archive. No logs found for %s.', target)
268
Evan Hernandezf388cbf2019-04-01 11:15:23 -0600269 output_proto.artifacts.add().path = os.path.join(output_dir, archive)
Alex Klein6504eca2019-04-18 15:37:56 -0600270
271
Alex Klein2275d692019-04-23 16:04:12 -0600272def BundleSimpleChromeArtifacts(input_proto, output_proto):
273 """Create the simple chrome artifacts."""
274 # Required args.
275 sysroot_path = input_proto.sysroot.path
276 build_target_name = input_proto.sysroot.build_target.name
277 output_dir = input_proto.output_dir
278
279 if not build_target_name:
280 cros_build_lib.Die('build_target.name is required')
281 if not output_dir:
282 cros_build_lib.Die('output_dir is required.')
283 if not os.path.exists(output_dir):
284 cros_build_lib.Die('output_dir (%s) does not exist.', output_dir)
285 if not sysroot_path:
286 cros_build_lib.Die('sysroot.path is required.')
287
288 # Optional args.
289 chroot_path = input_proto.chroot.path or constants.DEFAULT_CHROOT_PATH
290 cache_dir = input_proto.chroot.cache_dir
291
292 # Build out the argument instances.
293 build_target = build_target_util.BuildTarget(build_target_name)
294 chroot = chroot_lib.Chroot(path=chroot_path, cache_dir=cache_dir)
295 # Sysroot.path needs to be the fully qualified path, including the chroot.
296 full_sysroot_path = os.path.join(chroot.path, sysroot_path.lstrip(os.sep))
297 sysroot = sysroot_lib.Sysroot(full_sysroot_path)
298
299 # Quick sanity check that the sysroot exists before we go on.
300 if not sysroot.Exists():
301 cros_build_lib.Die('The sysroot does not exist.')
302
303 try:
304 results = artifacts.BundleSimpleChromeArtifacts(chroot, sysroot,
305 build_target, output_dir)
306 except artifacts.Error as e:
307 cros_build_lib.Die('Error %s raised in BundleSimpleChromeArtifacts: %s',
308 type(e), e)
309
310 for file_name in results:
311 output_proto.artifacts.add().path = file_name
312
313
Alex Klein2b236722019-06-19 15:44:26 -0600314@validate.require('chroot.path', 'sysroot.path', 'test_results_dir',
315 'output_dir')
Alex Klein6504eca2019-04-18 15:37:56 -0600316def BundleVmFiles(input_proto, output_proto):
317 """Tar VM disk and memory files.
318
319 Args:
320 input_proto (SysrootBundleRequest): The input proto.
321 output_proto (BundleResponse): The output proto.
322 """
323 chroot = input_proto.chroot.path
Alex Klein2b236722019-06-19 15:44:26 -0600324 sysroot = input_proto.sysroot.path.lstrip(os.sep)
325 test_results_dir = input_proto.test_results_dir.lstrip(os.sep)
Alex Klein6504eca2019-04-18 15:37:56 -0600326 output_dir = input_proto.output_dir
327
Alex Klein6504eca2019-04-18 15:37:56 -0600328 # TODO(crbug.com/954344): Replace with a chromite/service implementation.
Alex Klein2b236722019-06-19 15:44:26 -0600329 image_dir = os.path.join(chroot, sysroot, test_results_dir)
Alex Klein6504eca2019-04-18 15:37:56 -0600330 archives = vm_test_stages.ArchiveVMFilesFromImageDir(image_dir, output_dir)
331 for archive in archives:
332 output_proto.artifacts.add().path = archive
Tiancong Wangc4805b72019-06-11 12:12:03 -0700333
334def BundleOrderfileGenerationArtifacts(input_proto, output_proto):
335 """Create tarballs of all the artifacts of orderfile_generate builder.
336
337 Args:
338 input_proto (BundleRequest): The input proto.
339 output_proto (BundleResponse): The output proto.
340 """
341 # Required args.
342 build_target_name = input_proto.build_target.name
343 output_dir = input_proto.output_dir
344 chrome_version = input_proto.chrome_version
345
346 if not build_target_name:
347 cros_build_lib.Die('build_target.name is required.')
348 if not chrome_version:
349 cros_build_lib.Die('chrome_version is required.')
350 if not output_dir:
351 cros_build_lib.Die('output_dir is required.')
352 elif not os.path.isdir(output_dir):
353 cros_build_lib.Die('output_dir does not exist.')
354
355 chroot = controller_util.ParseChroot(input_proto.chroot)
356
357 try:
358 results = artifacts.BundleOrderfileGenerationArtifacts(
359 chroot, input_proto.build_target, chrome_version, output_dir)
360 except artifacts.Error as e:
361 cros_build_lib.Die('Error %s raised in BundleSimpleChromeArtifacts: %s',
362 type(e), e)
363
364 for file_name in results:
365 output_proto.artifacts.add().path = file_name