blob: b8a3369306a9c22fa47de0d08647470a9de54db3 [file] [log] [blame]
Mike Frysingerf1ba7ad2022-09-12 05:42:57 -04001# Copyright 2019 The ChromiumOS Authors
Alex Kleinda35fcf2019-03-07 16:01:15 -07002# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Sysroot controller."""
6
Chris McDonald1672ddb2021-07-21 11:48:23 -06007import logging
Michael Mortensen4ccfb082020-01-22 16:24:03 -07008import os
Jack Neus26b94672022-10-27 17:33:21 +00009import traceback
Michael Mortensen4ccfb082020-01-22 16:24:03 -070010
Alex Klein8cb365a2019-05-15 16:24:53 -060011from chromite.api import controller
Alex Klein076841b2019-08-29 15:19:39 -060012from chromite.api import faux
Alex Klein2b236722019-06-19 15:44:26 -060013from chromite.api import validate
Alex Kleina9d500b2019-04-22 15:37:51 -060014from chromite.api.controller import controller_util
Chris McDonald1672ddb2021-07-21 11:48:23 -060015from chromite.api.gen.chromiumos import common_pb2
Will Bradley7e5b8c12019-07-30 12:44:15 -060016from chromite.api.metrics import deserialize_metrics_log
LaMont Jonesfeffd1b2020-08-05 18:24:59 -060017from chromite.lib import binpkg
George Engelbrechtc9a8e812021-06-16 18:14:17 -060018from chromite.lib import build_target_lib
19from chromite.lib import chroot_lib
Alex Kleinda35fcf2019-03-07 16:01:15 -070020from chromite.lib import cros_build_lib
Michael Mortensen798ee192020-01-17 13:04:43 -070021from chromite.lib import goma_lib
Alex Kleinaef41942022-04-19 14:13:17 -060022from chromite.lib import metrics_lib
Michael Mortensen3f6b4bd2020-02-07 14:16:43 -070023from chromite.lib import osutils
Alex Kleina9d64602019-05-17 14:55:37 -060024from chromite.lib import portage_util
Alex Kleinda35fcf2019-03-07 16:01:15 -070025from chromite.lib import sysroot_lib
26from chromite.service import sysroot
27
Chris McDonald1672ddb2021-07-21 11:48:23 -060028
Alex Klein1699fab2022-09-08 08:46:06 -060029_ACCEPTED_LICENSES = "@CHROMEOS"
Alex Kleinda35fcf2019-03-07 16:01:15 -070030
Alex Kleineaa48532022-07-28 08:51:32 -060031DEFAULT_BACKTRACK = 30
32
Alex Kleind4e1e422019-03-18 16:00:41 -060033
Yoshiki Iguchia43704b2021-12-16 13:31:48 +090034def _GetGomaLogDirectory():
Alex Klein1699fab2022-09-08 08:46:06 -060035 """Get goma's log directory based on the env variables.
Yoshiki Iguchia43704b2021-12-16 13:31:48 +090036
Alex Klein1699fab2022-09-08 08:46:06 -060037 Returns:
Alex Klein611dddd2022-10-11 17:02:01 -060038 a string of a directory name where goma's log may exist, or None if no
39 potential directories exist.
Alex Klein1699fab2022-09-08 08:46:06 -060040 """
41 # TODO(crbug.com/1045001): Replace environment variable with query to
42 # goma object after goma refactoring allows this.
43 candidates = [
44 "GLOG_log_dir",
45 "GOOGLE_LOG_DIR",
46 "TEST_TMPDIR",
47 "TMPDIR",
48 "TMP",
49 ]
50 for candidate in candidates:
51 value = os.environ.get(candidate)
52 if value and os.path.isdir(value):
53 return value
Yoshiki Iguchia43704b2021-12-16 13:31:48 +090054
Alex Klein1699fab2022-09-08 08:46:06 -060055 # "/tmp" will always exist.
56 return "/tmp"
Yoshiki Iguchia43704b2021-12-16 13:31:48 +090057
58
George Engelbrechtc9a8e812021-06-16 18:14:17 -060059def ExampleGetResponse():
Alex Klein1699fab2022-09-08 08:46:06 -060060 """Give an example response to assemble upstream in caller artifacts."""
61 uabs = common_pb2.UploadedArtifactsByService
62 cabs = common_pb2.ArtifactsByService
63 return uabs.Sysroot(
64 artifacts=[
65 uabs.Sysroot.ArtifactPaths(
66 artifact_type=cabs.Sysroot.ArtifactType.SIMPLE_CHROME_SYSROOT,
67 paths=[
68 common_pb2.Path(
69 path="/tmp/sysroot_chromeos-base_chromeos-chrome.tar.xz",
70 location=common_pb2.Path.OUTSIDE,
71 )
72 ],
73 ),
74 uabs.Sysroot.ArtifactPaths(
75 artifact_type=cabs.Sysroot.ArtifactType.DEBUG_SYMBOLS,
76 paths=[
77 common_pb2.Path(
78 path="/tmp/debug.tgz", location=common_pb2.Path.OUTSIDE
79 )
80 ],
81 ),
82 uabs.Sysroot.ArtifactPaths(
83 artifact_type=cabs.Sysroot.ArtifactType.BREAKPAD_DEBUG_SYMBOLS,
84 paths=[
85 common_pb2.Path(
86 path="/tmp/debug_breakpad.tar.xz",
87 location=common_pb2.Path.OUTSIDE,
88 )
89 ],
90 ),
91 ]
92 )
George Engelbrechtc9a8e812021-06-16 18:14:17 -060093
94
Alex Klein1699fab2022-09-08 08:46:06 -060095def GetArtifacts(
96 in_proto: common_pb2.ArtifactsByService.Sysroot,
97 chroot: chroot_lib.Chroot,
98 sysroot_class: sysroot_lib.Sysroot,
99 build_target: build_target_lib.BuildTarget,
100 output_dir: str,
101) -> list:
102 """Builds and copies sysroot artifacts to specified output_dir.
George Engelbrechtc9a8e812021-06-16 18:14:17 -0600103
Alex Klein1699fab2022-09-08 08:46:06 -0600104 Copies sysroot artifacts to output_dir, returning a list of (output_dir: str)
105 paths to the desired files.
George Engelbrechtc9a8e812021-06-16 18:14:17 -0600106
Alex Klein1699fab2022-09-08 08:46:06 -0600107 Args:
Alex Klein611dddd2022-10-11 17:02:01 -0600108 in_proto: Proto request defining reqs.
109 chroot: The chroot class used for these artifacts.
110 sysroot_class: The sysroot class used for these artifacts.
111 build_target: The build target used for these artifacts.
112 output_dir: The path to write artifacts to.
George Engelbrechtc9a8e812021-06-16 18:14:17 -0600113
Alex Klein1699fab2022-09-08 08:46:06 -0600114 Returns:
Alex Klein611dddd2022-10-11 17:02:01 -0600115 A list of dictionary mappings of ArtifactType to list of paths.
Alex Klein1699fab2022-09-08 08:46:06 -0600116 """
117 generated = []
118 artifact_types = {
119 in_proto.ArtifactType.SIMPLE_CHROME_SYSROOT: sysroot.CreateSimpleChromeSysroot,
120 in_proto.ArtifactType.CHROME_EBUILD_ENV: sysroot.CreateChromeEbuildEnv,
121 in_proto.ArtifactType.BREAKPAD_DEBUG_SYMBOLS: sysroot.BundleBreakpadSymbols,
122 in_proto.ArtifactType.DEBUG_SYMBOLS: sysroot.BundleDebugSymbols,
Jack Neus11b6ebd2022-10-21 17:54:36 +0000123 in_proto.ArtifactType.FUZZER_SYSROOT: sysroot.CreateFuzzerSysroot,
Alex Klein1699fab2022-09-08 08:46:06 -0600124 }
Jack Neus5e56fef2021-06-18 16:57:28 +0000125
Alex Klein1699fab2022-09-08 08:46:06 -0600126 for output_artifact in in_proto.output_artifacts:
127 for artifact_type, func in artifact_types.items():
128 if artifact_type in output_artifact.artifact_types:
Jack Neus26b94672022-10-27 17:33:21 +0000129 try:
130 result = func(
131 chroot, sysroot_class, build_target, output_dir
132 )
133 except Exception as e:
134 generated.append(
135 {
136 "type": artifact_type,
137 "failed": True,
138 "failure_reason": str(e),
139 }
140 )
141 artifact_name = (
142 common_pb2.ArtifactsByService.Sysroot.ArtifactType.Name(
143 artifact_type
144 )
145 )
146 logging.warning(
147 "%s artifact generation failed with exception %s",
148 artifact_name,
149 e,
150 )
151 logging.warning("traceback:\n%s", traceback.format_exc())
152 continue
Alex Klein1699fab2022-09-08 08:46:06 -0600153 if result:
154 generated.append(
155 {
156 "paths": [result]
157 if isinstance(result, str)
158 else result,
159 "type": artifact_type,
160 }
161 )
Jack Neus5e56fef2021-06-18 16:57:28 +0000162
Alex Klein1699fab2022-09-08 08:46:06 -0600163 return generated
George Engelbrechtc9a8e812021-06-16 18:14:17 -0600164
165
Alex Klein076841b2019-08-29 15:19:39 -0600166@faux.all_empty
Alex Klein1699fab2022-09-08 08:46:06 -0600167@validate.require("build_target.name")
Alex Klein231d2da2019-07-22 16:44:45 -0600168@validate.validation_complete
169def Create(input_proto, output_proto, _config):
Alex Klein1699fab2022-09-08 08:46:06 -0600170 """Create or replace a sysroot."""
171 update_chroot = not input_proto.flags.chroot_current
172 replace_sysroot = input_proto.flags.replace
Alex Kleinda35fcf2019-03-07 16:01:15 -0700173
Alex Klein1699fab2022-09-08 08:46:06 -0600174 build_target = controller_util.ParseBuildTarget(
175 input_proto.build_target, input_proto.profile
176 )
177 package_indexes = [
178 binpkg.PackageIndexInfo.from_protobuf(x)
179 for x in input_proto.package_indexes
180 ]
181 run_configs = sysroot.SetupBoardRunConfig(
182 force=replace_sysroot,
183 upgrade_chroot=update_chroot,
184 package_indexes=package_indexes,
185 backtrack=DEFAULT_BACKTRACK,
186 )
Alex Kleinda35fcf2019-03-07 16:01:15 -0700187
Alex Klein1699fab2022-09-08 08:46:06 -0600188 try:
189 created = sysroot.Create(
190 build_target, run_configs, accept_licenses=_ACCEPTED_LICENSES
191 )
192 except sysroot.Error as e:
193 cros_build_lib.Die(e)
Alex Kleinda35fcf2019-03-07 16:01:15 -0700194
Alex Klein1699fab2022-09-08 08:46:06 -0600195 output_proto.sysroot.path = created.path
196 output_proto.sysroot.build_target.name = build_target.name
Alex Kleinda35fcf2019-03-07 16:01:15 -0700197
Alex Klein1699fab2022-09-08 08:46:06 -0600198 return controller.RETURN_CODE_SUCCESS
Alex Kleinda35fcf2019-03-07 16:01:15 -0700199
Alex Klein076841b2019-08-29 15:19:39 -0600200
Michael Mortensen98592f62019-09-27 13:34:18 -0600201@faux.all_empty
Alex Klein1699fab2022-09-08 08:46:06 -0600202@validate.require("build_target.name", "packages")
203@validate.require_each("packages", ["category", "package_name"])
Michael Mortensen3f6b4bd2020-02-07 14:16:43 -0700204@validate.validation_complete
205def GenerateArchive(input_proto, output_proto, _config):
Alex Klein1699fab2022-09-08 08:46:06 -0600206 """Generate a sysroot. Typically used by informational builders."""
207 build_target_name = input_proto.build_target.name
208 pkg_list = []
209 for package in input_proto.packages:
210 pkg_list.append("%s/%s" % (package.category, package.package_name))
Michael Mortensen3f6b4bd2020-02-07 14:16:43 -0700211
Alex Klein1699fab2022-09-08 08:46:06 -0600212 with osutils.TempDir(delete=False) as temp_output_dir:
213 sysroot_tar_path = sysroot.GenerateArchive(
214 temp_output_dir, build_target_name, pkg_list
215 )
Michael Mortensen3f6b4bd2020-02-07 14:16:43 -0700216
Alex Klein1699fab2022-09-08 08:46:06 -0600217 # By assigning this Path variable to the tar path, the tar file will be
218 # copied out to the input_proto's ResultPath location.
219 output_proto.sysroot_archive.path = sysroot_tar_path
220 output_proto.sysroot_archive.location = common_pb2.Path.INSIDE
Michael Mortensen3f6b4bd2020-02-07 14:16:43 -0700221
222
Alex Klein076841b2019-08-29 15:19:39 -0600223def _MockFailedPackagesResponse(_input_proto, output_proto, _config):
Alex Klein1699fab2022-09-08 08:46:06 -0600224 """Mock error response that populates failed packages."""
225 fail = output_proto.failed_package_data.add()
226 fail.name.package_name = "package"
227 fail.name.category = "category"
228 fail.name.version = "1.0.0_rc-r1"
229 fail.log_path.path = (
230 "/path/to/package:category-1.0.0_rc-r1:20210609-1337.log"
231 )
232 fail.log_path.location = common_pb2.Path.INSIDE
Lizzy Presland7e23a612021-11-09 21:49:42 +0000233
Alex Klein1699fab2022-09-08 08:46:06 -0600234 fail2 = output_proto.failed_package_data.add()
235 fail2.name.package_name = "bar"
236 fail2.name.category = "foo"
237 fail2.name.version = "3.7-r99"
238 fail2.log_path.path = "/path/to/foo:bar-3.7-r99:20210609-1620.log"
239 fail2.log_path.location = common_pb2.Path.INSIDE
Lizzy Presland7e23a612021-11-09 21:49:42 +0000240
Alex Klein076841b2019-08-29 15:19:39 -0600241
242@faux.empty_success
243@faux.error(_MockFailedPackagesResponse)
Alex Klein1699fab2022-09-08 08:46:06 -0600244@validate.require("sysroot.path", "sysroot.build_target.name")
245@validate.exists("sysroot.path")
Alex Klein231d2da2019-07-22 16:44:45 -0600246@validate.validation_complete
247def InstallToolchain(input_proto, output_proto, _config):
Alex Klein1699fab2022-09-08 08:46:06 -0600248 """Install the toolchain into a sysroot."""
249 compile_source = (
250 input_proto.flags.compile_source or input_proto.flags.toolchain_changed
251 )
Alex Kleinda35fcf2019-03-07 16:01:15 -0700252
Alex Klein1699fab2022-09-08 08:46:06 -0600253 sysroot_path = input_proto.sysroot.path
Alex Kleinda35fcf2019-03-07 16:01:15 -0700254
Alex Klein1699fab2022-09-08 08:46:06 -0600255 build_target = controller_util.ParseBuildTarget(
256 input_proto.sysroot.build_target
257 )
258 target_sysroot = sysroot_lib.Sysroot(sysroot_path)
259 run_configs = sysroot.SetupBoardRunConfig(usepkg=not compile_source)
Alex Kleinda35fcf2019-03-07 16:01:15 -0700260
Alex Klein1699fab2022-09-08 08:46:06 -0600261 _LogBinhost(build_target.name)
Alex Kleina9d64602019-05-17 14:55:37 -0600262
Alex Klein1699fab2022-09-08 08:46:06 -0600263 try:
264 sysroot.InstallToolchain(build_target, target_sysroot, run_configs)
265 except sysroot_lib.ToolchainInstallError as e:
266 controller_util.retrieve_package_log_paths(
267 e.failed_toolchain_info, output_proto, target_sysroot
268 )
Alex Kleinda35fcf2019-03-07 16:01:15 -0700269
Alex Klein1699fab2022-09-08 08:46:06 -0600270 return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE
Alex Kleind4e1e422019-03-18 16:00:41 -0600271
Alex Klein1699fab2022-09-08 08:46:06 -0600272 return controller.RETURN_CODE_SUCCESS
Alex Kleind4e1e422019-03-18 16:00:41 -0600273
Alex Klein076841b2019-08-29 15:19:39 -0600274
275@faux.empty_success
276@faux.error(_MockFailedPackagesResponse)
Alex Klein1699fab2022-09-08 08:46:06 -0600277@validate.require("sysroot.build_target.name")
278@validate.exists("sysroot.path")
279@validate.require_each("packages", ["category", "package_name"])
280@validate.require_each("use_flags", ["flag"])
Alex Klein231d2da2019-07-22 16:44:45 -0600281@validate.validation_complete
Alex Kleinaef41942022-04-19 14:13:17 -0600282@metrics_lib.collect_metrics
Alex Klein231d2da2019-07-22 16:44:45 -0600283def InstallPackages(input_proto, output_proto, _config):
Alex Klein1699fab2022-09-08 08:46:06 -0600284 """Install packages into a sysroot, building as necessary and permitted."""
285 compile_source = (
286 input_proto.flags.compile_source or input_proto.flags.toolchain_changed
287 )
Joanna Wang1ec0c812021-11-17 17:41:27 -0800288
Alex Klein1699fab2022-09-08 08:46:06 -0600289 use_remoteexec = bool(
290 input_proto.remoteexec_config.reproxy_cfg_file
291 and input_proto.remoteexec_config.reclient_dir
292 )
Joanna Wang1ec0c812021-11-17 17:41:27 -0800293
Alex Klein1699fab2022-09-08 08:46:06 -0600294 # Testing if Goma will support unknown compilers now.
295 use_goma = input_proto.flags.use_goma and not use_remoteexec
Alex Kleind4e1e422019-03-18 16:00:41 -0600296
Alex Klein1699fab2022-09-08 08:46:06 -0600297 target_sysroot = sysroot_lib.Sysroot(input_proto.sysroot.path)
298 build_target = controller_util.ParseBuildTarget(
299 input_proto.sysroot.build_target
300 )
Alex Kleinca572ee2020-09-03 10:47:14 -0600301
Alex Klein1699fab2022-09-08 08:46:06 -0600302 # Get the package atom for each specified package. The field is optional, so
303 # error only when we cannot parse an atom for each of the given packages.
304 packages = [
305 controller_util.PackageInfoToCPV(x).cp for x in input_proto.packages
306 ]
Alex Kleinca572ee2020-09-03 10:47:14 -0600307
Alex Klein1699fab2022-09-08 08:46:06 -0600308 package_indexes = [
309 binpkg.PackageIndexInfo.from_protobuf(x)
310 for x in input_proto.package_indexes
311 ]
Alex Kleind4e1e422019-03-18 16:00:41 -0600312
Alex Klein1699fab2022-09-08 08:46:06 -0600313 # Calculate which packages would have been merged, but don't install anything.
314 dryrun = input_proto.flags.dryrun
Navil Perez5766d1b2021-05-26 17:38:15 +0000315
Alex Klein1699fab2022-09-08 08:46:06 -0600316 if not target_sysroot.IsToolchainInstalled():
317 cros_build_lib.Die("Toolchain must first be installed.")
Alex Kleind4e1e422019-03-18 16:00:41 -0600318
Alex Klein1699fab2022-09-08 08:46:06 -0600319 _LogBinhost(build_target.name)
Alex Kleina9d64602019-05-17 14:55:37 -0600320
Alex Klein1699fab2022-09-08 08:46:06 -0600321 use_flags = [u.flag for u in input_proto.use_flags]
322 build_packages_config = sysroot.BuildPackagesRunConfig(
323 use_any_chrome=False,
324 usepkg=not compile_source,
325 install_debug_symbols=True,
326 packages=packages,
327 package_indexes=package_indexes,
328 use_flags=use_flags,
329 use_goma=use_goma,
330 use_remoteexec=use_remoteexec,
331 incremental_build=False,
332 dryrun=dryrun,
333 backtrack=DEFAULT_BACKTRACK,
334 )
Alex Kleind4e1e422019-03-18 16:00:41 -0600335
Alex Klein1699fab2022-09-08 08:46:06 -0600336 try:
337 sysroot.BuildPackages(
338 build_target, target_sysroot, build_packages_config
339 )
340 except sysroot_lib.PackageInstallError as e:
341 if not e.failed_packages:
342 # No packages to report, so just exit with an error code.
343 return controller.RETURN_CODE_COMPLETED_UNSUCCESSFULLY
Alex Klein2557b4f2019-07-11 14:34:00 -0600344
Alex Klein1699fab2022-09-08 08:46:06 -0600345 controller_util.retrieve_package_log_paths(
346 e.failed_packages, output_proto, target_sysroot
347 )
Alex Kleind4e1e422019-03-18 16:00:41 -0600348
Alex Klein1699fab2022-09-08 08:46:06 -0600349 return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE
350 finally:
351 # Copy goma logs to specified directory if there is a goma_config and
352 # it contains a log_dir to store artifacts.
353 if input_proto.goma_config.log_dir.dir:
354 log_source_dir = _GetGomaLogDirectory()
355 archiver = goma_lib.LogsArchiver(
356 log_source_dir,
357 dest_dir=input_proto.goma_config.log_dir.dir,
358 stats_file=input_proto.goma_config.stats_file,
359 counterz_file=input_proto.goma_config.counterz_file,
360 )
361 archiver_tuple = archiver.Archive()
362 if archiver_tuple.stats_file:
363 output_proto.goma_artifacts.stats_file = (
364 archiver_tuple.stats_file
365 )
366 if archiver_tuple.counterz_file:
367 output_proto.goma_artifacts.counterz_file = (
368 archiver_tuple.counterz_file
369 )
370 output_proto.goma_artifacts.log_files[:] = archiver_tuple.log_files
Alex Kleina9d64602019-05-17 14:55:37 -0600371
Alex Klein1699fab2022-09-08 08:46:06 -0600372 # Return without populating the response if it is a dryrun.
373 if dryrun:
374 return controller.RETURN_CODE_SUCCESS
Navil Perez5766d1b2021-05-26 17:38:15 +0000375
Alex Klein1699fab2022-09-08 08:46:06 -0600376 # Read metric events log and pipe them into output_proto.events.
377 deserialize_metrics_log(output_proto.events, prefix=build_target.name)
Will Bradley7e5b8c12019-07-30 12:44:15 -0600378
Alex Kleina9d64602019-05-17 14:55:37 -0600379
380def _LogBinhost(board):
Alex Klein1699fab2022-09-08 08:46:06 -0600381 """Log the portage binhost for the given board."""
382 binhost = portage_util.PortageqEnvvar(
383 "PORTAGE_BINHOST", board=board, allow_undefined=True
384 )
385 if not binhost:
386 logging.warning("Portage Binhost not found.")
387 else:
388 logging.info("Portage Binhost: %s", binhost)