blob: 788876abbf2240d55696e8d8f4e0099cb5280979 [file] [log] [blame]
Mike Frysingerf1ba7ad2022-09-12 05:42:57 -04001# Copyright 2018 The ChromiumOS Authors
Don Garrettbe3608b2018-06-05 17:10:09 -07002# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
Don Garrettff45f4b2018-10-04 09:08:01 -07005# pylint: disable=line-too-long
Mike Frysingera2527f82023-02-03 04:32:15 -05006
Don Garrettbe3608b2018-06-05 17:10:09 -07007"""Generate LUCI Scheduler config file.
8
9This generates the LUCI Scheduler configuration file for ChromeOS builds based
10on the chromeos_config contents.
11
Don Garrettff45f4b2018-10-04 09:08:01 -070012Changes to chromite/config/luci-scheduler.cfg will be autodeployed:
Mike Nicholsb575c612021-04-09 10:04:43 -060013 https://data.corp.google.com/sites/chromeos_ci_cros_ci_builds/utility/?f=board_name:in:luci-scheduler-updater
Don Garrettbe3608b2018-06-05 17:10:09 -070014
Don Garrettc8323dc2018-08-22 15:58:26 -070015Notes:
16 Normal builds are scheduled based on the builder values for
17 'schedule' and 'triggered_gitiles' in config/chromeos_config.py.
18
19 Branched builds are scheduled based on the function
20 chromeos_config.BranchScheduleConfig()
Don Garrettbe3608b2018-06-05 17:10:09 -070021"""
Don Garrettff45f4b2018-10-04 09:08:01 -070022# pylint: enable=line-too-long
Don Garrettbe3608b2018-06-05 17:10:09 -070023
Don Garrettbe3608b2018-06-05 17:10:09 -070024import sys
25
Don Garrett8dace272018-07-19 14:07:42 -070026from chromite.config import chromeos_config
Don Garrettbe3608b2018-06-05 17:10:09 -070027from chromite.lib import commandline
28from chromite.lib import config_lib
29
30
31_CONFIG_HEADER = """# Defines buckets on luci-scheduler.appspot.com.
32#
33# For schema of this file and documentation see ProjectConfig message in
Mike Frysinger92dc6492021-02-17 16:01:09 -050034# https://github.com/luci/luci-go/blob/HEAD/scheduler/appengine/messages/config.proto
Don Garrettbe3608b2018-06-05 17:10:09 -070035
36# Generated with chromite/scripts/gen_luci_scheduler
37
Don Garrettff45f4b2018-10-04 09:08:01 -070038# Autodeployed with:
Mike Nicholsb575c612021-04-09 10:04:43 -060039# https://data.corp.google.com/sites/chromeos_ci_cros_ci_builds/utility/?f=board_name:in:luci-scheduler-updater
Don Garrettff45f4b2018-10-04 09:08:01 -070040
Don Garrettbe3608b2018-06-05 17:10:09 -070041acl_sets {
42 name: "default"
43 acls {
44 role: READER
45 granted_to: "group:googlers"
46 }
47 acls {
48 role: OWNER
49 granted_to: "group:project-chromeos-admins"
50 }
Jason D. Clintona9e374a2018-10-18 18:41:00 -060051 acls {
52 role: TRIGGERER
53 granted_to: "group:mdb/chromeos-build-access"
LaMont Jones48ddfd92020-09-16 13:46:41 -060054 }
55 acls {
56 role: TRIGGERER
LaMont Jonesd1bb0002020-09-16 09:35:10 -060057 granted_to: "group:project-chromeos-buildbucket-schedulers"
Jason D. Clintona9e374a2018-10-18 18:41:00 -060058 }
Don Garrettbe3608b2018-06-05 17:10:09 -070059}
60"""
61
Alex Klein1699fab2022-09-08 08:46:06 -060062
Don Garrett9a0d90c2018-10-30 12:47:14 -070063def buildJobName(build_config):
Alex Klein1699fab2022-09-08 08:46:06 -060064 if "schedule_branch" in build_config:
65 return "%s-%s" % (build_config.schedule_branch, build_config.name)
66 else:
67 return build_config.name
Don Garrett9a0d90c2018-10-30 12:47:14 -070068
Don Garrettbe3608b2018-06-05 17:10:09 -070069
70def genSchedulerJob(build_config):
Alex Klein1699fab2022-09-08 08:46:06 -060071 """Generate the luci scheduler job for a given build config.
Don Garrettbe3608b2018-06-05 17:10:09 -070072
Alex Klein1699fab2022-09-08 08:46:06 -060073 Args:
74 build_config: config_lib.BuildConfig.
Don Garrettbe3608b2018-06-05 17:10:09 -070075
Alex Klein1699fab2022-09-08 08:46:06 -060076 Returns:
77 Multiline string to include in the luci scheduler configuration.
78 """
79 job_name = buildJobName(build_config)
80 if "schedule_branch" in build_config:
81 branch = build_config.schedule_branch
82 else:
83 branch = "main"
Don Garrett73148e52018-08-17 16:54:46 -070084
Alex Klein1699fab2022-09-08 08:46:06 -060085 tags = {
86 "cbb_branch": branch,
87 "cbb_config": build_config.name,
88 "cbb_display_label": build_config.display_label,
89 "cbb_workspace_branch": build_config.workspace_branch,
90 "cbb_goma_client_type": build_config.goma_client_type,
91 }
Don Garrett73148e52018-08-17 16:54:46 -070092
Alex Klein1699fab2022-09-08 08:46:06 -060093 # Filter out tags with no value set.
94 tags = {k: v for k, v in tags.items() if v}
Don Garrett73148e52018-08-17 16:54:46 -070095
Alex Klein1699fab2022-09-08 08:46:06 -060096 tag_lines = [
97 ' tags: "%s:%s"' % (k, tags[k]) for k in sorted(tags.keys())
98 ]
99 prop_lines = [
100 ' properties: "%s:%s"' % (k, tags[k]) for k in sorted(tags.keys())
101 ]
Don Garrett73148e52018-08-17 16:54:46 -0700102
Alex Klein1699fab2022-09-08 08:46:06 -0600103 # TODO: Move --buildbot arg into recipe, and remove from here.
104 template = """
Don Garrettbe3608b2018-06-05 17:10:09 -0700105job {
Don Garrett8dace272018-07-19 14:07:42 -0700106 id: "%(job_name)s"
Vadim Shtayura6398e002021-01-11 12:02:11 -0800107 realm: "cbb-jobs"
Don Garrettbe3608b2018-06-05 17:10:09 -0700108 acl_sets: "default"
109 schedule: "%(schedule)s"
110 buildbucket: {
111 server: "cr-buildbucket.appspot.com"
Vadim Shtayura959cb0f2022-02-03 12:45:58 -0800112 bucket: "general"
Don Garrettbe3608b2018-06-05 17:10:09 -0700113 builder: "%(builder)s"
Don Garrett73148e52018-08-17 16:54:46 -0700114%(tag_lines)s
115%(prop_lines)s
Don Garrettbe3608b2018-06-05 17:10:09 -0700116 properties: "cbb_extra_args:[\\"--buildbot\\"]"
117 }
118}
119"""
Don Garrett8dace272018-07-19 14:07:42 -0700120
Alex Klein1699fab2022-09-08 08:46:06 -0600121 return template % {
122 "job_name": job_name,
123 "builder": build_config.luci_builder,
124 "schedule": build_config.schedule,
125 "tag_lines": "\n".join(tag_lines),
126 "prop_lines": "\n".join(prop_lines),
127 }
Don Garrettbe3608b2018-06-05 17:10:09 -0700128
129
David Burgercba0d272020-06-24 16:21:26 -0600130def genSchedulerTrigger(trigger_name, repo, refs, path_regexps, builds):
Alex Klein1699fab2022-09-08 08:46:06 -0600131 """Generate the luci scheduler job for a given build config.
Don Garrettbe3608b2018-06-05 17:10:09 -0700132
Alex Klein1699fab2022-09-08 08:46:06 -0600133 Args:
134 trigger_name: Name of the trigger as a string.
135 repo: Gitiles URL git git repository.
136 refs: Iterable of git refs to check. May use regular expressions.
137 path_regexps: Iterable of path regular expressions of files to trigger on
138 or falsy to trigger on everything.
139 builds: Iterable of build config names to trigger.
Don Garrettbe3608b2018-06-05 17:10:09 -0700140
Alex Klein1699fab2022-09-08 08:46:06 -0600141 Returns:
142 Multiline string to include in the luci scheduler configuration.
143 """
144 template = """
Don Garrettbe3608b2018-06-05 17:10:09 -0700145trigger {
146 id: "%(trigger_name)s"
Vadim Shtayura6398e002021-01-11 12:02:11 -0800147 realm: "cbb-jobs"
Don Garrettbe3608b2018-06-05 17:10:09 -0700148 acl_sets: "default"
149 schedule: "with 5m interval"
150 gitiles: {
151 repo: "%(repo)s"
David Burgercba0d272020-06-24 16:21:26 -0600152%(refs)s%(path_regexps)s
Don Garrettbe3608b2018-06-05 17:10:09 -0700153 }
154%(triggers)s
155}
156"""
Alex Klein1699fab2022-09-08 08:46:06 -0600157 if path_regexps:
158 path_regexps = "\n" + "\n".join(
159 ' path_regexps: "%s"' % r for r in path_regexps
160 )
161 else:
162 path_regexps = ""
163 return template % {
164 "trigger_name": trigger_name,
165 "repo": repo,
166 "refs": "\n".join(' refs: "%s"' % r for r in refs),
167 "path_regexps": path_regexps,
168 "triggers": "\n".join(' triggers: "%s"' % b for b in builds),
169 }
Don Garrettbe3608b2018-06-05 17:10:09 -0700170
171
Don Garrett8dace272018-07-19 14:07:42 -0700172def genLuciSchedulerConfig(site_config, branch_config):
Alex Klein1699fab2022-09-08 08:46:06 -0600173 """Generate a luciSchedulerConfig as a string.
Don Garrettbe3608b2018-06-05 17:10:09 -0700174
Alex Klein1699fab2022-09-08 08:46:06 -0600175 Args:
176 site_config: A config_lib.SiteConfig instance.
177 branch_config: A list of BuildConfig instances to schedule.
Don Garrettbe3608b2018-06-05 17:10:09 -0700178
Alex Klein1699fab2022-09-08 08:46:06 -0600179 Returns:
180 The complete scheduler configuration contents as a string.
181 """
182 # Trigger collection is used to collect together trigger information, so
183 # we can reuse the same trigger for multiple builds as needed.
184 # It maps gitiles_key to a set of build_names.
185 # A gitiles_key = (gitiles_url, tuple(ref_list))
186 trigger_collection = {}
Don Garrettbe3608b2018-06-05 17:10:09 -0700187
Alex Klein1699fab2022-09-08 08:46:06 -0600188 jobs = []
Don Garrettbe3608b2018-06-05 17:10:09 -0700189
Alex Klein1699fab2022-09-08 08:46:06 -0600190 # Order the configs consistently.
191 configs = [
192 site_config[name] for name in sorted(site_config)
193 ] + branch_config
Don Garrett8dace272018-07-19 14:07:42 -0700194
Alex Klein1699fab2022-09-08 08:46:06 -0600195 for config in configs:
196 # Populate jobs.
197 if config.schedule:
198 jobs.append(genSchedulerJob(config))
Don Garrettbe3608b2018-06-05 17:10:09 -0700199
Alex Klein1699fab2022-09-08 08:46:06 -0600200 # Populate trigger_collection.
201 if config.triggered_gitiles:
202 for trigger in config.triggered_gitiles:
203 try:
204 gitiles_url, ref_list, path_regexps = trigger
205 except ValueError:
206 gitiles_url, ref_list = trigger
207 path_regexps = []
208 gitiles_key = (
209 gitiles_url,
210 tuple(ref_list),
211 tuple(path_regexps),
212 )
213 trigger_collection.setdefault(gitiles_key, set())
214 trigger_collection[gitiles_key].add(buildJobName(config))
Don Garrettbe3608b2018-06-05 17:10:09 -0700215
Alex Klein1699fab2022-09-08 08:46:06 -0600216 # Populate triggers.
217 triggers = []
218 trigger_counter = 0
219 for gitiles_key in sorted(trigger_collection):
220 builds = sorted(trigger_collection[gitiles_key])
Don Garrettbe3608b2018-06-05 17:10:09 -0700221
Alex Klein1699fab2022-09-08 08:46:06 -0600222 trigger_name = "trigger_%s" % trigger_counter
223 gitiles_url, refs, path_regexps = gitiles_key
224 triggers.append(
225 genSchedulerTrigger(
226 trigger_name, gitiles_url, refs, path_regexps, builds
227 )
228 )
229 trigger_counter += 1
Don Garrettbe3608b2018-06-05 17:10:09 -0700230
Alex Klein1699fab2022-09-08 08:46:06 -0600231 return "".join([_CONFIG_HEADER] + triggers + jobs)
Don Garrettbe3608b2018-06-05 17:10:09 -0700232
233
234def GetParser():
Alex Klein1699fab2022-09-08 08:46:06 -0600235 """Creates the argparse parser."""
236 parser = commandline.ArgumentParser(description=__doc__)
Don Garrettbe3608b2018-06-05 17:10:09 -0700237
Alex Klein1699fab2022-09-08 08:46:06 -0600238 parser.add_argument(
239 "-o", "--file_out", type="path", help="Write output to specified file."
240 )
Don Garrettbe3608b2018-06-05 17:10:09 -0700241
Alex Klein1699fab2022-09-08 08:46:06 -0600242 return parser
Don Garrettbe3608b2018-06-05 17:10:09 -0700243
244
245def main(argv):
Alex Klein1699fab2022-09-08 08:46:06 -0600246 parser = GetParser()
247 options = parser.parse_args(argv)
248 options.Freeze()
Don Garrettbe3608b2018-06-05 17:10:09 -0700249
Alex Klein1699fab2022-09-08 08:46:06 -0600250 site_config = config_lib.GetConfig()
251 branch_config = chromeos_config.BranchScheduleConfig()
Don Garrettbe3608b2018-06-05 17:10:09 -0700252
Alex Klein1699fab2022-09-08 08:46:06 -0600253 with (
254 open(options.file_out, "w") if options.file_out else sys.stdout
255 ) as fh:
256 fh.write(genLuciSchedulerConfig(site_config, branch_config))