blob: 679c467dd6c2385db349b36dff671843c51061e2 [file] [log] [blame]
Evan Hernandezd437b4e2019-03-25 13:48:30 -06001# -*- coding: utf-8 -*-
2# Copyright 2019 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Portage Binhost operations."""
7
8from __future__ import print_function
9
10import urlparse
11
Alex Klein2b236722019-06-19 15:44:26 -060012from chromite.api import validate
Alex Kleinaf0e0452019-06-03 18:10:01 -060013from chromite.api.controller import controller_util
Evan Hernandezd437b4e2019-03-25 13:48:30 -060014from chromite.api.gen.chromite.api import binhost_pb2
Alex Kleina471f682019-05-31 11:23:45 -060015from chromite.lib import build_target_util
LaMont Jonesc64ae212019-04-15 15:41:28 -060016from chromite.lib import constants
17from chromite.lib import cros_build_lib
Evan Hernandezd437b4e2019-03-25 13:48:30 -060018from chromite.lib import gs
Alex Kleinaf0e0452019-06-03 18:10:01 -060019from chromite.lib import sysroot_lib
Evan Hernandezd437b4e2019-03-25 13:48:30 -060020from chromite.service import binhost
21
Alex Kleinaf0e0452019-06-03 18:10:01 -060022_OVERLAY_TYPE_TO_NAME = {
23 binhost_pb2.OVERLAYTYPE_PUBLIC: constants.PUBLIC_OVERLAYS,
24 binhost_pb2.OVERLAYTYPE_PRIVATE: constants.PRIVATE_OVERLAYS,
25 binhost_pb2.OVERLAYTYPE_BOTH: constants.BOTH_OVERLAYS,
26 binhost_pb2.OVERLAYTYPE_NONE: None
27}
28
Evan Hernandezd437b4e2019-03-25 13:48:30 -060029
Alex Klein2b236722019-06-19 15:44:26 -060030@validate.require('build_target.name')
Alex Klein7e40d252019-06-10 09:01:32 -060031def GetBinhosts(input_proto, output_proto):
32 """Get a list of binhosts."""
Alex Klein2b236722019-06-19 15:44:26 -060033 build_target = build_target_util.BuildTarget(input_proto.build_target.name)
Alex Klein7e40d252019-06-10 09:01:32 -060034
35 binhosts = binhost.GetBinhosts(build_target)
36
37 for current in binhosts:
38 new_binhost = output_proto.binhosts.add()
39 new_binhost.uri = current
Alex Kleind3ac0bd2019-06-10 15:17:51 -060040 new_binhost.package_index = 'Packages'
Alex Klein7e40d252019-06-10 09:01:32 -060041
42
Alex Klein2b236722019-06-19 15:44:26 -060043@validate.require('build_target.name')
Alex Kleina471f682019-05-31 11:23:45 -060044def GetPrivatePrebuiltAclArgs(input_proto, output_proto):
45 """Get the ACL args from the files in the private overlays."""
Alex Klein2b236722019-06-19 15:44:26 -060046 build_target = build_target_util.BuildTarget(input_proto.build_target.name)
Alex Kleina471f682019-05-31 11:23:45 -060047
48 try:
49 args = binhost.GetPrebuiltAclArgs(build_target)
50 except binhost.Error as e:
51 cros_build_lib.Die(e.message)
52
53 for arg, value in args:
54 new_arg = output_proto.args.add()
55 new_arg.arg = arg
56 new_arg.value = value
57
58
Evan Hernandezd437b4e2019-03-25 13:48:30 -060059def PrepareBinhostUploads(input_proto, output_proto):
60 """Return a list of files to uplooad to the binhost.
61
62 See BinhostService documentation in api/proto/binhost.proto.
63
64 Args:
65 input_proto (PrepareBinhostUploadsRequest): The input proto.
66 output_proto (PrepareBinhostUploadsResponse): The output proto.
67 """
Alex Kleinaf0e0452019-06-03 18:10:01 -060068 target_name = (input_proto.sysroot.build_target.name
69 or input_proto.build_target.name)
70 sysroot_path = input_proto.sysroot.path
Evan Hernandezd437b4e2019-03-25 13:48:30 -060071
Alex Kleinaf0e0452019-06-03 18:10:01 -060072 if not sysroot_path and not target_name:
73 cros_build_lib.Die('Sysroot.path is required.')
74
75 build_target = build_target_util.BuildTarget(target_name)
76 chroot = controller_util.ParseChroot(input_proto.chroot)
77
78 if not sysroot_path:
79 # Very temporary, so not worried about this not calling the lib function.
80 sysroot_path = build_target.root
81 sysroot = sysroot_lib.Sysroot(sysroot_path)
82
83 uri = input_proto.uri
Evan Hernandezd437b4e2019-03-25 13:48:30 -060084 # For now, we enforce that all input URIs are Google Storage buckets.
85 if not gs.PathIsGs(uri):
86 raise ValueError('Upload URI %s must be Google Storage.' % uri)
87
88 parsed_uri = urlparse.urlparse(uri)
Alex Klein8dffea42019-06-14 14:01:36 -060089 upload_uri = gs.GetGsURL(parsed_uri.netloc, for_gsutil=True).rstrip('/')
Alex Klein3e728442019-05-15 11:46:57 -060090 upload_path = parsed_uri.path.lstrip('/')
Evan Hernandezd437b4e2019-03-25 13:48:30 -060091
92 # Read all packages and update the index. The index must be uploaded to the
93 # binhost for Portage to use it, so include it in upload_targets.
Alex Kleinaf0e0452019-06-03 18:10:01 -060094 uploads_dir = binhost.GetPrebuiltsRoot(chroot, sysroot, build_target)
Alex Klein3e728442019-05-15 11:46:57 -060095 index_path = binhost.UpdatePackageIndex(uploads_dir, upload_uri, upload_path,
96 sudo=True)
Alex Kleinb5d57a62019-06-20 12:04:53 -060097 upload_targets = binhost.GetPrebuiltsFiles(uploads_dir)
Evan Hernandezd437b4e2019-03-25 13:48:30 -060098 assert index_path.startswith(uploads_dir), (
99 'expected index_path to start with uploads_dir')
100 upload_targets.append(index_path[len(uploads_dir):])
101
102 output_proto.uploads_dir = uploads_dir
103 for upload_target in upload_targets:
104 output_proto.upload_targets.add().path = upload_target.strip('/')
105
106
Alex Klein2b236722019-06-19 15:44:26 -0600107@validate.require('build_target.name', 'uri')
Evan Hernandezd437b4e2019-03-25 13:48:30 -0600108def SetBinhost(input_proto, output_proto):
109 """Set the URI for a given binhost key and build target.
110
111 See BinhostService documentation in api/proto/binhost.proto.
112
113 Args:
114 input_proto (SetBinhostRequest): The input proto.
115 output_proto (SetBinhostResponse): The output proto.
116 """
117 target = input_proto.build_target.name
118 key = binhost_pb2.BinhostKey.Name(input_proto.key)
119 uri = input_proto.uri
120 private = input_proto.private
Alex Klein6fb0eb82019-05-20 16:16:14 -0600121
122 # Temporary measure to force the new parallel cq post submit builders to write
123 # to a different file than the old ones. Writing to the same file was causing
124 # them to fight over the new one's value and the old logic of clearing out
125 # the values for files it didn't update. Once we've done a full switch over,
126 # we can dump this logic and delete all of the PARALLEL_POSTSUBMIT_BINHOST
127 # configs.
128 # TODO(crbug.com/965244) remove this.
129 if key == 'POSTSUBMIT_BINHOST':
130 key = 'PARALLEL_POSTSUBMIT_BINHOST'
131
Evan Hernandezd437b4e2019-03-25 13:48:30 -0600132 output_proto.output_file = binhost.SetBinhost(target, key, uri,
133 private=private)
LaMont Jonesc64ae212019-04-15 15:41:28 -0600134
LaMont Jonesc64ae212019-04-15 15:41:28 -0600135
Alex Kleinaf0e0452019-06-03 18:10:01 -0600136def RegenBuildCache(input_proto, _output_proto):
LaMont Jonesc64ae212019-04-15 15:41:28 -0600137 """Regenerate the Build Cache for a build target.
138
139 See BinhostService documentation in api/proto/binhost.proto.
140
141 Args:
142 input_proto (RegenBuildCacheRequest): The input proto.
Alex Kleinaf0e0452019-06-03 18:10:01 -0600143 _output_proto (RegenBuildCacheResponse): The output proto.
LaMont Jonesc64ae212019-04-15 15:41:28 -0600144 """
145 overlay_type = input_proto.overlay_type
Evan Hernandez763f58b2019-05-15 10:57:49 -0600146 if overlay_type in _OVERLAY_TYPE_TO_NAME:
Alex Kleinaf0e0452019-06-03 18:10:01 -0600147 binhost.RegenBuildCache(_OVERLAY_TYPE_TO_NAME[overlay_type])
LaMont Jonesc64ae212019-04-15 15:41:28 -0600148 else:
149 cros_build_lib.Die('Overlay_type must be specified.')