blob: d1c570f8ae97f334c84011ade6a42e0f7f2f36fe [file] [log] [blame]
Oleh Prypin1a0593f2019-03-11 09:43:28 +01001#!/usr/bin/env lucicfg
2
3# Copyright (c) 2019 The WebRTC project authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7# https://chromium.googlesource.com/infra/luci/luci-go/+/master/lucicfg/doc/
8
Jeremy Leconteea2016b2020-07-01 09:47:22 +02009"""LUCI project configuration for WebRTC CQ and CI."""
10
11lucicfg.check_version("1.15.0")
Mirko Bonadei9189d9c2020-06-09 21:54:18 +020012
Oleh Prypin1a0593f2019-03-11 09:43:28 +010013WEBRTC_GIT = "https://webrtc.googlesource.com/src"
14WEBRTC_GERRIT = "https://webrtc-review.googlesource.com/src"
15
Mirko Bonadei32ca4862020-03-11 10:39:45 +010016GOMA_BACKEND_WEBRTC_RBE_PROD = {
Mirko Bonadei1e905342020-03-04 09:39:42 +010017 "$build/goma": {
18 "server_host": "goma.chromium.org",
Jeremy Leconteea2016b2020-07-01 09:47:22 +020019 "use_luci_auth": True,
20 },
Mirko Bonadei1e905342020-03-04 09:39:42 +010021}
22
Mirko Bonadei84360ca2020-03-12 08:35:48 +010023GOMA_BACKEND_WEBRTC_IOS_RBE_PROD = {
24 "$build/goma": {
25 "server_host": "goma.chromium.org",
Jeremy Leconteea2016b2020-07-01 09:47:22 +020026 "use_luci_auth": True,
Mirko Bonadei84360ca2020-03-12 08:35:48 +010027 },
Artem Titov1d47d312020-11-03 14:45:52 +010028 "$depot_tools/osx_sdk": {"sdk_version": "12a7209"},
Mirko Bonadei84360ca2020-03-12 08:35:48 +010029}
30
Patrik Höglund3916a972019-12-04 13:19:38 +010031GOMA_BACKEND_RBE_ATS_PROD = {
32 "$build/goma": {
33 "server_host": "goma.chromium.org",
Mirko Bonadei2688eca2020-03-25 09:15:54 +010034 "use_luci_auth": True,
Jeremy Leconteea2016b2020-07-01 09:47:22 +020035 "enable_ats": True,
36 },
Patrik Höglund3916a972019-12-04 13:19:38 +010037}
38
Mirko Bonadeidbe40522021-04-21 10:42:00 +020039# Disable ATS on CQ/try.
40GOMA_BACKEND_RBE_NO_ATS_PROD = {
41 "$build/goma": {
42 "server_host": "goma.chromium.org",
43 "use_luci_auth": True,
44 "enable_ats": False,
45 },
46}
47
Oleh Prypin1a0593f2019-03-11 09:43:28 +010048# Top-level configs:
49
Owen Rodley6b721ba2020-05-26 10:04:20 +100050# Enable luci.tree_closer.
51lucicfg.enable_experiment("crbug.com/1054172")
52
Oleh Prypin1a0593f2019-03-11 09:43:28 +010053lucicfg.config(
54 config_dir = ".",
55 tracked_files = [
56 "commit-queue.cfg",
57 "cr-buildbucket.cfg",
58 "luci-logdog.cfg",
59 "luci-milo.cfg",
Oleh Prypin71d17422019-03-28 08:43:16 +010060 "luci-notify.cfg",
Oleh Prypin705b6a62019-04-03 23:10:51 +020061 "luci-notify/**/*",
Oleh Prypin1a0593f2019-03-11 09:43:28 +010062 "luci-scheduler.cfg",
63 "project.cfg",
64 ],
Jeremy Leconteea2016b2020-07-01 09:47:22 +020065 lint_checks = ["default"],
Oleh Prypin1a0593f2019-03-11 09:43:28 +010066)
67
68luci.project(
69 name = "webrtc",
70 buildbucket = "cr-buildbucket.appspot.com",
71 logdog = "luci-logdog.appspot.com",
72 milo = "luci-milo.appspot.com",
Oleh Prypin71d17422019-03-28 08:43:16 +010073 notify = "luci-notify.appspot.com",
Oleh Prypin1a0593f2019-03-11 09:43:28 +010074 scheduler = "luci-scheduler.appspot.com",
75 swarming = "chromium-swarm.appspot.com",
76 acls = [
77 acl.entry(
78 [acl.BUILDBUCKET_READER, acl.LOGDOG_READER, acl.PROJECT_CONFIGS_READER, acl.SCHEDULER_READER],
79 groups = ["all"],
80 ),
81 acl.entry(acl.LOGDOG_WRITER, groups = ["luci-logdog-chromium-writers"]),
82 acl.entry(acl.SCHEDULER_OWNER, groups = ["project-webrtc-admins"]),
83 ],
84)
85
86luci.logdog(
87 gs_bucket = "chromium-luci-logdog",
88)
89
90luci.milo(
91 logo = "https://storage.googleapis.com/chrome-infra/webrtc-logo-vert-retro-255x305.png",
92)
93
Owen Rodleya55c38e2020-06-15 11:16:10 +100094luci.notify(tree_closing_enabled = True)
95
Oleh Prypin1a0593f2019-03-11 09:43:28 +010096luci.cq(
97 status_host = "chromium-cq-status.appspot.com",
98 submit_max_burst = 1,
99 submit_burst_delay = 1 * time.minute,
100)
101
102luci.gitiles_poller(
103 name = "webrtc-gitiles-trigger-master",
104 bucket = "ci",
105 repo = WEBRTC_GIT,
106)
107
108# Bucket definitions:
109
110luci.bucket(
111 name = "try",
112 acls = [
113 acl.entry(acl.BUILDBUCKET_TRIGGERER, groups = [
114 "service-account-cq",
115 "project-webrtc-tryjob-access",
116 ]),
117 ],
118)
119
120luci.bucket(
121 name = "ci",
122 acls = [
Oleh Prypinf35939d2019-05-03 20:42:38 +0200123 acl.entry(acl.BUILDBUCKET_TRIGGERER, groups = [
124 "project-webrtc-ci-schedulers",
125 ], users = [
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100126 "luci-scheduler@appspot.gserviceaccount.com",
127 ]),
Jeremy Leconte2e25bb52020-07-16 09:38:56 +0200128 acl.entry(acl.BUILDBUCKET_TRIGGERER, groups = [
129 # Allow Pinpoint to trigger builds for bisection
130 "service-account-chromeperf",
131 ]),
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100132 ],
133)
134
135luci.bucket(
136 name = "perf",
137 acls = [
138 acl.entry(acl.BUILDBUCKET_TRIGGERER, users = [
139 "luci-scheduler@appspot.gserviceaccount.com",
140 "webrtc-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
141 ]),
142 ],
143)
144
145luci.bucket(
146 name = "cron",
147 acls = [
148 acl.entry(acl.BUILDBUCKET_TRIGGERER, users = [
149 "luci-scheduler@appspot.gserviceaccount.com",
150 ]),
151 ],
152)
153
154# Commit queue definitions:
155
156luci.cq_group(
157 name = "cq",
158 tree_status_host = "webrtc-status.appspot.com",
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100159 watch = [cq.refset(repo = WEBRTC_GERRIT, refs = ["refs/heads/master"])],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100160 acls = [
161 acl.entry(acl.CQ_COMMITTER, groups = ["project-webrtc-committers"]),
162 acl.entry(acl.CQ_DRY_RUNNER, groups = ["project-webrtc-tryjob-access"]),
163 ],
164 retry_config = cq.RETRY_ALL_FAILURES,
Oleh Prypin5416f2d2019-08-02 09:43:15 +0200165 cancel_stale_tryjobs = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100166)
167
168luci.cq_group(
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100169 name = "cq_branch",
170 watch = [cq.refset(repo = WEBRTC_GERRIT, refs = ["refs/branch-heads/.+"])],
171 acls = [
172 acl.entry(acl.CQ_COMMITTER, groups = ["project-webrtc-committers"]),
173 acl.entry(acl.CQ_DRY_RUNNER, groups = ["project-webrtc-tryjob-access"]),
174 ],
175 retry_config = cq.RETRY_ALL_FAILURES,
Oleh Prypin5416f2d2019-08-02 09:43:15 +0200176 cancel_stale_tryjobs = True,
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100177)
178
179luci.cq_group(
180 name = "cq_infra",
181 watch = [cq.refset(repo = WEBRTC_GERRIT, refs = ["refs/heads/infra/config"])],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100182 acls = [
183 acl.entry(acl.CQ_COMMITTER, groups = ["project-webrtc-admins"]),
184 acl.entry(acl.CQ_DRY_RUNNER, groups = ["project-webrtc-tryjob-access"]),
185 ],
186 retry_config = cq.RETRY_ALL_FAILURES,
Oleh Prypin5416f2d2019-08-02 09:43:15 +0200187 cancel_stale_tryjobs = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100188)
189
190luci.cq_tryjob_verifier(
191 builder = "presubmit",
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100192 cq_group = "cq_infra",
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100193)
194
Oleh Prypine56cd9b2019-05-29 21:14:31 +0200195luci.cq_tryjob_verifier(
196 builder = "webrtc-internal:g3.webrtc-internal.try/internal_compile_lite",
197 owner_whitelist = ["project-webrtc-internal-tryjob-access"],
198 cq_group = "cq",
199)
200
Oleh Prypin71d17422019-03-28 08:43:16 +0100201# Notifier definitions:
202
203luci.notifier(
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200204 name = "post_submit_failure_notifier",
205 on_new_status = ["FAILURE"],
Oleh Prypin71d17422019-03-28 08:43:16 +0100206 notify_emails = ["webrtc-sheriffs-robots@google.com"],
Oleh Prypin705b6a62019-04-03 23:10:51 +0200207 template = luci.notifier_template(
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200208 name = "build_failure",
209 body = io.read_file("luci-notify/email-templates/build_failure.template"),
Oleh Prypin705b6a62019-04-03 23:10:51 +0200210 ),
Oleh Prypin71d17422019-03-28 08:43:16 +0100211)
212
213luci.notifier(
214 name = "cron_notifier",
Jeremy Leconte67a904e2020-07-08 11:48:18 +0200215 on_new_status = ["FAILURE", "INFRA_FAILURE"],
Mirko Bonadei450e9b72020-07-10 19:19:22 +0000216 notify_emails = ["webrtc-troopers-robots@google.com"],
Oleh Prypin705b6a62019-04-03 23:10:51 +0200217 template = luci.notifier_template(
218 name = "cron",
219 body = io.read_file("luci-notify/email-templates/cron.template"),
220 ),
Oleh Prypin71d17422019-03-28 08:43:16 +0100221)
222
Jeremy Leconte67a904e2020-07-08 11:48:18 +0200223luci.notifier(
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200224 name = "infra_failure_notifier",
Jeremy Leconte67a904e2020-07-08 11:48:18 +0200225 on_new_status = ["INFRA_FAILURE"],
Mirko Bonadei450e9b72020-07-10 19:19:22 +0000226 notify_emails = ["webrtc-troopers-robots@google.com"],
Jeremy Leconte67a904e2020-07-08 11:48:18 +0200227 template = luci.notifier_template(
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200228 name = "infra_failure",
229 body = io.read_file("luci-notify/email-templates/infra_failure.template"),
Jeremy Leconte67a904e2020-07-08 11:48:18 +0200230 ),
231)
232
Owen Rodley6b721ba2020-05-26 10:04:20 +1000233# Tree closer definitions:
234
235luci.tree_closer(
236 name = "webrtc_tree_closer",
237 tree_status_host = "webrtc-status.appspot.com",
Owen Rodley6b721ba2020-05-26 10:04:20 +1000238 # TODO: These step filters are copied verbatim from Gatekeeper, for testing
239 # that LUCI-Notify would take the exact same actions. Once we've switched
240 # over, this should be updated - several of these steps don't exist in
241 # WebRTC recipes.
242 failed_step_regexp = [
243 "bot_update",
244 "compile",
245 "gclient runhooks",
246 "runhooks",
247 "update",
248 "extract build",
249 "cleanup_temp",
250 "taskkill",
251 "compile",
252 "gn",
253 ],
254 failed_step_regexp_exclude = ".*\\(experimental\\).*",
255)
256
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100257# Recipe definitions:
258
259def recipe(recipe, pkg = "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"):
260 return luci.recipe(
261 name = recipe.split("/")[-1],
262 cipd_package = pkg,
263 cipd_version = "refs/heads/master",
264 recipe = recipe,
265 )
266
267recipe("chromium_trybot")
268recipe("run_presubmit")
269recipe("webrtc/auto_roll_webrtc_deps")
270recipe("webrtc/ios")
271recipe("webrtc/ios_api_framework")
272recipe("webrtc/libfuzzer")
273recipe("webrtc/more_configs")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100274recipe("webrtc/standalone")
Mirko Bonadei54bbd1c2020-12-14 15:54:46 +0100275recipe("webrtc/update_webrtc_binary_version")
Jeff Yoonf421c682020-12-17 00:16:22 -0800276recipe("lkgr_finder", pkg = "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100277
278# Console definitions:
279
280luci.console_view(name = "ci", title = "Main", repo = WEBRTC_GIT, header = "console-header.textpb")
281luci.console_view(name = "perf", title = "Perf", repo = WEBRTC_GIT, header = "console-header.textpb")
282luci.list_view(name = "cron", title = "Cron")
283luci.console_view(name = "fyi", title = "FYI", repo = WEBRTC_GIT, header = "console-header.textpb")
284luci.list_view(name = "try", title = "Tryserver")
285
286def add_milo(builder, views):
287 """Add Milo console entries for the builder.
288
289 Args:
290 builder: builder name (str).
291 views: dict where keys are names of consoles and values are either a
292 category for the console (str, pipe-separated) or True, which means
293 adding to a list view rather than a console.
294 """
295 for view_name, category in views.items():
296 if category == None:
297 continue
298 elif type(category) == "string":
299 category, _, short_name = category.rpartition("|")
300 luci.console_view_entry(
301 console_view = view_name,
302 builder = builder,
303 category = category or None,
304 short_name = short_name or None,
305 )
306 elif category == True:
307 luci.list_view_entry(
308 list_view = view_name,
309 builder = builder,
310 )
311 else:
312 fail("Unexpected value for category: %r" % category)
313
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200314lkgr_builders = {}
315
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100316# Builder-defining functions:
317
Artem Titarenkof6588b72019-11-14 14:28:11 +0100318def webrtc_builder(
319 name,
320 recipe = "standalone",
321 dimensions = {},
Mirko Bonadei825f9c22020-07-01 11:52:39 +0200322 properties = {},
Artem Titarenkof6588b72019-11-14 14:28:11 +0100323 priority = 30,
324 execution_timeout = 2 * time.hour,
Mirko Bonadeib7ef4602020-07-01 13:28:26 +0200325 goma_jobs = None,
Artem Titarenkof6588b72019-11-14 14:28:11 +0100326 **kwargs):
Mirko Bonadeib7ef4602020-07-01 13:28:26 +0200327 """WebRTC specific wrapper around luci.builder.
328
329 Args:
330 name: builder name (str).
331 recipe: string with the name of the recipe to run.
332 dimensions: dict of Swarming dimensions (strings) to search machines by.
333 properties: dict of properties to pass to the recipe (on top of the default ones).
334 priority: int [1-255] or None, indicating swarming task priority, lower is
335 more important. If None, defer the decision to Buildbucket service.
336 execution_timeout: int or None, how long to wait for a running build to finish before
337 forcefully aborting it and marking the build as timed out. If None,
338 defer the decision to Buildbucket service.
339 goma_jobs: int or None, number of jobs to be used by the builder. If None, defer the
340 decition to the goma module.
341 **kwargs: Pass on to webrtc_builder / luci.builder.
342 Returns:
343 A luci.builder.
344 """
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100345 dimensions = merge_dicts({"cpu": "x86-64"}, dimensions)
346
Mirko Bonadeib7ef4602020-07-01 13:28:26 +0200347 if goma_jobs != None:
348 goma_additional_params = {"$build/goma": {"jobs": goma_jobs}}
349 properties = merge_dicts(properties, goma_additional_params)
Ye Kuang8df20492020-10-15 14:54:17 +0900350 properties = merge_dicts(properties, {
351 "$recipe_engine/isolated": {
352 "server": "https://isolateserver.appspot.com",
353 },
354 })
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100355 return luci.builder(
356 name = name,
Oleh Prypin705b6a62019-04-03 23:10:51 +0200357 executable = recipe,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100358 dimensions = {k: v for k, v in dimensions.items() if v != None},
Mirko Bonadeib7ef4602020-07-01 13:28:26 +0200359 properties = properties,
Artem Titarenkof6588b72019-11-14 14:28:11 +0100360 execution_timeout = execution_timeout,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100361 priority = priority,
362 build_numbers = True,
363 swarming_tags = ["vpython:native-python-wrapper"],
364 **kwargs
365 )
366
367def ci_builder(
368 name,
369 ci_cat,
370 perf_cat = None,
371 fyi_cat = None,
372 properties = {},
373 dimensions = {},
374 prioritized = False,
Oleh Prypinf5432c52019-03-19 15:10:37 +0100375 enabled = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100376 **kwargs):
Oleh Prypine5677672019-04-25 12:21:35 +0200377 """Add a post-submit builder.
378
379 Args:
380 name: builder name (str).
381 ci_cat: the category + name for the /ci/ console, or None to omit from the console.
382 perf_cat: the category + name for the /perf/ console, or None to omit from the console.
383 fyi_cat: the category + name for the /fyi/ console, or None to omit from the console.
384 properties: dict of properties to pass to the recipe (on top of the default ones).
385 dimensions: dict of Swarming dimensions (strings) to search machines by.
386 prioritized: True to make this builder have a higher priority and never batch builds.
387 enabled: False to exclude this builder from consoles and failure notifications.
388 **kwargs: Pass on to webrtc_builder / luci.builder.
389 Returns:
390 A luci.builder.
391
392 Notifications are also disabled if a builder is not on either of /ci/ or /perf/ consoles.
393 """
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100394 if prioritized:
395 kwargs["triggering_policy"] = scheduler.greedy_batching(
396 max_batch_size = 1,
397 max_concurrent_invocations = 3,
398 )
399 kwargs["priority"] = 29
400
Oleh Prypinf5432c52019-03-19 15:10:37 +0100401 if enabled:
402 add_milo(name, {"ci": ci_cat, "perf": perf_cat, "fyi": fyi_cat})
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200403 if ci_cat:
404 lkgr_builders[name] = True
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100405 return webrtc_builder(
406 name = name,
Garrett Beaty4a106ce2020-09-09 14:51:23 -0700407 properties = merge_dicts({"builder_group": "client.webrtc"}, properties),
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100408 dimensions = merge_dicts({"pool": "luci.webrtc.ci"}, dimensions),
409 bucket = "ci",
410 service_account = "webrtc-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
Oleh Prypinf5432c52019-03-19 15:10:37 +0100411 triggered_by = ["webrtc-gitiles-trigger-master"] if enabled else None,
Oleh Prypin71d17422019-03-28 08:43:16 +0100412 repo = WEBRTC_GIT,
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200413 notifies = ["post_submit_failure_notifier", "webrtc_tree_closer", "infra_failure_notifier"] if enabled and (ci_cat or perf_cat) else None,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100414 **kwargs
415 )
416
417def try_builder(
418 name,
419 try_cat = True,
420 fyi_cat = None,
421 properties = {},
422 dimensions = {},
423 cq = {},
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100424 branch_cq = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100425 **kwargs):
Oleh Prypine5677672019-04-25 12:21:35 +0200426 """Add a pre-submit builder.
427
428 Args:
429 name: builder name (str).
430 try_cat: boolean, whether to include this builder in the /try/ console. See also: `add_milo`.
431 fyi_cat: the category + name for the /fyi/ console, or None to omit from the console.
432 properties: dict of properties to pass to the recipe (on top of the default ones).
433 dimensions: dict of Swarming dimensions (strings) to search machines by.
434 cq: None to exclude this from all commit queues, or a dict of kwargs for cq_tryjob_verifier.
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100435 branch_cq: False to exclude this builder just from the release-branch CQ.
Oleh Prypine5677672019-04-25 12:21:35 +0200436 **kwargs: Pass on to webrtc_builder / luci.builder.
437 Returns:
438 A luci.builder.
439 """
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100440 add_milo(name, {"try": try_cat, "fyi": fyi_cat})
441 if cq != None:
442 luci.cq_tryjob_verifier(
443 builder = name,
444 cq_group = "cq",
445 **cq
446 )
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100447 if branch_cq:
448 luci.cq_tryjob_verifier(
449 builder = name,
450 cq_group = "cq_branch",
451 **cq
452 )
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100453
454 return webrtc_builder(
455 name = name,
Garrett Beaty4a106ce2020-09-09 14:51:23 -0700456 properties = merge_dicts({"builder_group": "tryserver.webrtc"}, properties),
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100457 dimensions = merge_dicts({"pool": "luci.webrtc.try"}, dimensions),
458 bucket = "try",
459 service_account = "webrtc-try-builder@chops-service-accounts.iam.gserviceaccount.com",
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200460 notifies = ["infra_failure_notifier"],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100461 **kwargs
462 )
463
464def perf_builder(
465 name,
466 perf_cat,
467 recipe = "standalone",
468 properties = {},
469 dimensions = {},
470 **kwargs):
471 add_milo(name, {"perf": perf_cat})
472 return webrtc_builder(
473 name = name,
474 recipe = recipe,
Garrett Beaty4a106ce2020-09-09 14:51:23 -0700475 properties = merge_dicts({"builder_group": "client.webrtc.perf"}, properties),
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100476 dimensions = merge_dicts({"pool": "luci.webrtc.perf", "os": "Linux", "cpu": None}, dimensions),
477 bucket = "perf",
478 service_account = "webrtc-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
479 # log_base of 1.7 means:
480 # when there are P pending builds, LUCI will batch the first B builds.
481 # P: 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 ...
482 # B: 1 1 2 2 3 3 3 3 4 4 4 4 4 4 5 ...
Jeremy Leconteea2016b2020-07-01 09:47:22 +0200483 triggering_policy = scheduler.logarithmic_batching(log_base = 1.7),
Artem Titarenkof6588b72019-11-14 14:28:11 +0100484 execution_timeout = 3 * time.hour,
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200485 notifies = ["post_submit_failure_notifier", "infra_failure_notifier"],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100486 **kwargs
487 )
488
Mirko Bonadei12643772020-03-27 13:16:35 +0100489def android_perf_builder(
490 name,
491 perf_cat,
492 recipe = "standalone",
493 properties = {},
494 dimensions = {},
495 **kwargs):
496 return perf_builder(
497 name = name,
498 perf_cat = perf_cat,
499 recipe = recipe,
500 properties = merge_dicts(GOMA_BACKEND_WEBRTC_RBE_PROD, properties),
501 dimensions = dimensions,
502 **kwargs
503 )
504
505def linux_perf_builder(
506 name,
507 perf_cat,
508 recipe = "standalone",
509 properties = {},
510 dimensions = {},
511 **kwargs):
512 return perf_builder(
513 name = name,
514 perf_cat = perf_cat,
515 recipe = recipe,
516 properties = merge_dicts(GOMA_BACKEND_WEBRTC_RBE_PROD, properties),
517 dimensions = dimensions,
518 **kwargs
519 )
520
521def mac_perf_builder(
522 name,
523 perf_cat,
524 recipe = "standalone",
525 properties = {},
526 dimensions = {},
527 **kwargs):
528 return perf_builder(
529 name = name,
530 perf_cat = perf_cat,
531 recipe = recipe,
532 properties = merge_dicts(GOMA_BACKEND_WEBRTC_RBE_PROD, properties),
533 dimensions = dimensions,
534 **kwargs
535 )
536
537def win_perf_builder(
538 name,
539 perf_cat,
540 recipe = "standalone",
541 properties = {},
542 dimensions = {},
543 **kwargs):
544 return perf_builder(
545 name = name,
546 perf_cat = perf_cat,
547 recipe = recipe,
548 properties = merge_dicts(GOMA_BACKEND_RBE_ATS_PROD, properties),
549 dimensions = dimensions,
550 **kwargs
551 )
552
Mirko Bonadei54bbd1c2020-12-14 15:54:46 +0100553def cron_builder(name, dimensions = {}, service_account = None, **kwargs):
554 if service_account == None:
555 service_account = "chromium-webrtc-autoroll@webrtc-ci.iam.gserviceaccount.com"
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100556 add_milo(name, {"cron": True})
557 return webrtc_builder(
558 name = name,
559 dimensions = merge_dicts({"pool": "luci.webrtc.cron", "os": "Linux"}, dimensions),
560 bucket = "cron",
Mirko Bonadei54bbd1c2020-12-14 15:54:46 +0100561 service_account = service_account,
Oleh Prypin71d17422019-03-28 08:43:16 +0100562 notifies = ["cron_notifier"],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100563 **kwargs
564 )
565
566# Helpers:
567
568def merge_dicts(a, b):
569 """Return the result of merging two dicts.
570
571 If matching values are both dicts or both lists, they will be merged (non-recursively).
572
573 Args:
574 a: first dict.
575 b: second dict (takes priority).
576 Returns:
577 Merged dict.
578 """
579 a = dict(a)
580 for k, bv in b.items():
581 av = a.get(k)
582 if type(av) == "dict" and type(bv) == "dict":
583 a[k] = dict(av)
584 a[k].update(bv)
585 elif type(av) == "list" and type(bv) == "list":
586 a[k] = av + bv
587 else:
588 a[k] = bv
589 return a
590
591def normal_builder_factory(**common_kwargs):
592 def builder(*args, **kwargs):
593 return ci_builder(*args, **merge_dicts(common_kwargs, kwargs))
594
595 def try_job(*args, **kwargs):
596 return try_builder(*args, **merge_dicts(common_kwargs, kwargs))
597
598 return builder, try_job
599
600# Mixins:
601
602linux_builder, linux_try_job = normal_builder_factory(
603 dimensions = {"os": "Linux", "inside_docker": "0"},
Mirko Bonadei32ca4862020-03-11 10:39:45 +0100604 properties = GOMA_BACKEND_WEBRTC_RBE_PROD,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100605)
606
607android_builder, android_try_job = normal_builder_factory(
608 dimensions = {"os": "Linux"},
Mirko Bonadei32ca4862020-03-11 10:39:45 +0100609 properties = GOMA_BACKEND_WEBRTC_RBE_PROD,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100610)
611
Mirko Bonadeidbe40522021-04-21 10:42:00 +0200612def win_builder(*args, **kwargs):
613 return ci_builder(
614 *args,
615 **merge_dicts({
616 "dimensions": {"os": "Windows"},
617 "properties": GOMA_BACKEND_RBE_ATS_PROD,
618 }, kwargs)
619 )
620
621def win_try_job(*args, **kwargs):
622 return try_builder(
623 *args,
624 **merge_dicts({
625 "dimensions": {"os": "Windows"},
626 "properties": GOMA_BACKEND_RBE_NO_ATS_PROD,
627 }, kwargs)
628 )
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100629
630mac_builder, mac_try_job = normal_builder_factory(
631 dimensions = {"os": "Mac"},
Mirko Bonadei32ca4862020-03-11 10:39:45 +0100632 properties = GOMA_BACKEND_WEBRTC_RBE_PROD,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100633)
634
635ios_builder, ios_try_job = normal_builder_factory(
Mirko Bonadei8096c232020-08-26 21:44:42 +0200636 dimensions = {"os": "Mac-10.15"},
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100637 recipe = "ios",
Mirko Bonadei84360ca2020-03-12 08:35:48 +0100638 properties = GOMA_BACKEND_WEBRTC_IOS_RBE_PROD,
639 caches = [swarming.cache("osx_sdk")],
640)
641
642# Some iOS builders don't use goma because they build using
643# `use_xcode_clang=true` which is not supported by goma.
644# https://ci.chromium.org/p/webrtc/builders/ci/iOS%20API%20Framework%20Builder
645# https://ci.chromium.org/p/webrtc/builders/try/ios_api_framework
646ios_builder_no_goma, ios_try_job_no_goma = normal_builder_factory(
Mirko Bonadei8096c232020-08-26 21:44:42 +0200647 dimensions = {"os": "Mac-10.15"},
Mirko Bonadei84360ca2020-03-12 08:35:48 +0100648 recipe = "ios",
Artem Titov1d47d312020-11-03 14:45:52 +0100649 properties = {"$depot_tools/osx_sdk": {"sdk_version": "12a7209"}},
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100650 caches = [swarming.cache("osx_sdk")],
651)
652
653# Actual builder configuration:
654
655android_builder("Android32 (M Nexus5X)(dbg)", "Android|arm|dbg")
656android_try_job("android_compile_arm_dbg", cq = None)
657android_try_job("android_arm_dbg")
658android_builder("Android32 (M Nexus5X)", "Android|arm|rel")
659android_try_job("android_arm_rel")
660android_builder("Android32 Builder arm", "Android|arm|size", perf_cat = "Android|arm|Builder|", prioritized = True)
661android_try_job("android_compile_arm_rel")
Mirko Bonadei86b4f2d2020-07-08 14:24:48 +0200662android_perf_builder("Perf Android32 (M Nexus5)", "Android|arm|Tester|M Nexus5", triggered_by = ["Android32 Builder arm"])
Mirko Bonadei12643772020-03-27 13:16:35 +0100663android_perf_builder("Perf Android32 (M AOSP Nexus6)", "Android|arm|Tester|M AOSP Nexus6", triggered_by = ["Android32 Builder arm"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100664android_try_job("android_compile_arm64_dbg", cq = None)
665android_try_job("android_arm64_dbg", cq = None)
666android_builder("Android64 (M Nexus5X)", "Android|arm64|rel")
667android_try_job("android_arm64_rel")
668android_builder("Android64 Builder arm64", "Android|arm64|size", perf_cat = "Android|arm64|Builder|", prioritized = True)
Mirko Bonadei12643772020-03-27 13:16:35 +0100669android_perf_builder("Perf Android64 (M Nexus5X)", "Android|arm64|Tester|M Nexus5X", triggered_by = ["Android64 Builder arm64"])
670android_perf_builder("Perf Android64 (O Pixel2)", "Android|arm64|Tester|O Pixel2", triggered_by = ["Android64 Builder arm64"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100671android_try_job("android_compile_arm64_rel")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100672android_builder("Android64 Builder x64 (dbg)", "Android|x64|dbg")
673android_try_job("android_compile_x64_dbg")
674android_try_job("android_compile_x64_rel", cq = None)
675android_builder("Android32 Builder x86 (dbg)", "Android|x86|dbg")
676android_try_job("android_compile_x86_dbg")
677android_builder("Android32 Builder x86", "Android|x86|rel")
678android_try_job("android_compile_x86_rel")
679android_builder("Android32 (more configs)", "Android|arm|more", recipe = "more_configs")
680android_try_job("android_arm_more_configs", recipe = "more_configs")
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100681android_try_job("android_chromium_compile", recipe = "chromium_trybot", branch_cq = False)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100682
683ios_builder("iOS32 Debug", "iOS|arm|dbg")
684ios_try_job("ios_compile_arm_dbg")
685ios_builder("iOS32 Release", "iOS|arm|rel")
686ios_try_job("ios_compile_arm_rel")
687ios_builder("iOS64 Debug", "iOS|arm64|dbg")
688ios_try_job("ios_compile_arm64_dbg")
689ios_builder("iOS64 Release", "iOS|arm64|rel")
690ios_try_job("ios_compile_arm64_rel")
Artem Titov1d47d312020-11-03 14:45:52 +0100691ios_builder("iOS64 Sim Debug (iOS 14.0)", "iOS|x64|14")
Mirko Bonadeif1e0c0b2021-01-12 13:05:52 +0100692ios_try_job("ios_sim_x64_dbg_ios14", cq = None)
Artem Titov1d47d312020-11-03 14:45:52 +0100693ios_builder("iOS64 Sim Debug (iOS 13)", "iOS|x64|13")
Artem Titove973dd62020-11-03 18:05:21 +0100694ios_try_job("ios_sim_x64_dbg_ios13", cq = None)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100695ios_builder("iOS64 Sim Debug (iOS 12)", "iOS|x64|12")
Andrey Logvin34ac6592021-02-04 13:03:49 +0000696
697# TODO(crbug.com/1174112): return ios_sim_x64_dbg_ios12 to cq.
698ios_try_job("ios_sim_x64_dbg_ios12", cq = None)
Mirko Bonadei84360ca2020-03-12 08:35:48 +0100699ios_builder_no_goma("iOS API Framework Builder", "iOS|fat|size", recipe = "ios_api_framework", prioritized = True)
Jeremy Leconteea2016b2020-07-01 09:47:22 +0200700ios_try_job_no_goma("ios_api_framework", recipe = "ios_api_framework", cq = None)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100701
702linux_builder("Linux32 Debug", "Linux|x86|dbg")
703linux_try_job("linux_x86_dbg")
704linux_builder("Linux32 Release", "Linux|x86|rel")
705linux_try_job("linux_x86_rel")
706linux_builder("Linux64 Debug", "Linux|x64|dbg")
707linux_try_job("linux_dbg", cq = None)
708linux_try_job("linux_compile_dbg")
709linux_builder("Linux64 Release", "Linux|x64|rel")
710linux_try_job("linux_rel")
711linux_builder("Linux64 Builder", "Linux|x64|size", perf_cat = "Linux|x64|Builder|", prioritized = True)
712linux_try_job("linux_compile_rel")
Mirko Bonadei12643772020-03-27 13:16:35 +0100713linux_perf_builder("Perf Linux Trusty", "Linux|x64|Tester|Trusty", triggered_by = ["Linux64 Builder"])
714linux_perf_builder("Perf Linux Xenial", None, triggered_by = ["Linux64 Builder"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100715linux_builder("Linux32 Debug (ARM)", "Linux|arm|dbg")
716linux_try_job("linux_compile_arm_dbg")
717linux_builder("Linux32 Release (ARM)", "Linux|arm|rel")
718linux_try_job("linux_compile_arm_rel")
719linux_builder("Linux64 Debug (ARM)", "Linux|arm64|dbg")
720linux_try_job("linux_compile_arm64_dbg")
721linux_builder("Linux64 Release (ARM)", "Linux|arm64|rel")
722linux_try_job("linux_compile_arm64_rel")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100723linux_builder("Linux Asan", "Linux|x64|asan")
724linux_try_job("linux_asan")
725linux_builder("Linux MSan", "Linux|x64|msan")
726linux_try_job("linux_msan")
727linux_builder("Linux Tsan v2", "Linux|x64|tsan")
728linux_try_job("linux_tsan2")
729linux_builder("Linux UBSan", "Linux|x64|ubsan")
730linux_try_job("linux_ubsan")
731linux_builder("Linux UBSan vptr", "Linux|x64|ubsan")
732linux_try_job("linux_ubsan_vptr")
733linux_builder("Linux64 Release (Libfuzzer)", "Linux|x64|fuzz", recipe = "libfuzzer")
734linux_try_job("linux_libfuzzer_rel", recipe = "libfuzzer")
735linux_builder("Linux (more configs)", "Linux|x64|more", recipe = "more_configs")
736linux_try_job("linux_more_configs", recipe = "more_configs")
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100737linux_try_job("linux_chromium_compile", recipe = "chromium_trybot", branch_cq = False)
Mirko Bonadei40de50f2019-11-07 11:23:47 +0100738linux_try_job("linux_chromium_compile_dbg", recipe = "chromium_trybot", branch_cq = False)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100739
740mac_builder("Mac64 Debug", "Mac|x64|dbg")
741mac_try_job("mac_dbg", cq = None)
742mac_try_job("mac_compile_dbg")
743mac_builder("Mac64 Release", "Mac|x64|rel")
744mac_try_job("mac_rel")
745mac_try_job("mac_compile_rel", cq = None)
746mac_builder("Mac64 Builder", ci_cat = None, perf_cat = "Mac|x64|Builder|")
Mirko Bonadei12643772020-03-27 13:16:35 +0100747mac_perf_builder("Perf Mac 10.11", "Mac|x64|Tester|10.11", triggered_by = ["Mac64 Builder"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100748mac_builder("Mac Asan", "Mac|x64|asan")
749mac_try_job("mac_asan")
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100750mac_try_job("mac_chromium_compile", recipe = "chromium_trybot", dimensions = {"cores": "8"}, branch_cq = False)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100751
752win_builder("Win32 Debug", "Win MSVC|x86|dbg")
753win_try_job("win_x86_msvc_dbg")
754win_try_job("win_compile_x86_msvc_dbg", cq = None)
Oleh Prypin63041b82019-03-19 22:56:01 +0000755win_builder("Win32 Release", "Win MSVC|x86|rel")
756win_try_job("win_x86_msvc_rel")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100757win_try_job("win_compile_x86_msvc_rel", cq = None)
758win_builder("Win64 Debug", "Win MSVC|x64|dbg")
759win_try_job("win_x64_msvc_dbg")
760win_try_job("win_compile_x64_msvc_dbg", cq = None)
761win_builder("Win64 Release", "Win MSVC|x64|rel")
762win_try_job("win_x64_msvc_rel")
763win_try_job("win_compile_x64_msvc_rel", cq = None)
764win_builder("Win32 Debug (Clang)", "Win Clang|x86|dbg")
765win_try_job("win_x86_clang_dbg", cq = None)
766win_try_job("win_compile_x86_clang_dbg")
767win_builder("Win32 Release (Clang)", "Win Clang|x86|rel")
768win_try_job("win_x86_clang_rel")
769win_try_job("win_compile_x86_clang_rel", cq = None)
770win_builder("Win32 Builder (Clang)", ci_cat = None, perf_cat = "Win|x86|Builder|")
Mirko Bonadei12643772020-03-27 13:16:35 +0100771win_perf_builder("Perf Win7", "Win|x86|Tester|7", triggered_by = ["Win32 Builder (Clang)"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100772win_builder("Win64 Debug (Clang)", "Win Clang|x64|dbg")
773win_try_job("win_x64_clang_dbg", cq = None)
774win_try_job("win_x64_clang_dbg_win8", cq = None)
775win_try_job("win_x64_clang_dbg_win10", cq = None)
776win_try_job("win_compile_x64_clang_dbg")
777win_builder("Win64 Release (Clang)", "Win Clang|x64|rel")
778win_try_job("win_x64_clang_rel", cq = None)
779win_try_job("win_compile_x64_clang_rel")
Mirko Bonadeiafbe33d2019-05-31 09:27:54 +0200780win_builder("Win64 ASan", "Win Clang|x64|asan")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100781win_try_job("win_asan")
782win_builder("Win64 UWP", ci_cat = None, fyi_cat = "")
783win_try_job("win_x64_uwp", cq = None, try_cat = None, fyi_cat = "")
784win_builder("Win (more configs)", "Win Clang|x86|more", recipe = "more_configs")
785win_try_job("win_x86_more_configs", recipe = "more_configs")
Mirko Bonadei825f9c22020-07-01 11:52:39 +0200786win_try_job("win_chromium_compile", recipe = "chromium_trybot", branch_cq = False, goma_jobs = 150)
787win_try_job("win_chromium_compile_dbg", recipe = "chromium_trybot", branch_cq = False, goma_jobs = 150)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100788
789linux_try_job(
790 "presubmit",
791 recipe = "run_presubmit",
792 properties = {"repo_name": "webrtc", "runhooks": True},
793 priority = 28,
794 cq = {"disable_reuse": True},
795)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100796
797cron_builder(
798 "Auto-roll - WebRTC DEPS",
799 recipe = "auto_roll_webrtc_deps",
Oleh Prypindc68a722019-06-25 10:43:13 +0200800 schedule = "0 */2 * * *", # Every 2 hours.
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100801)
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200802
Mirko Bonadei54bbd1c2020-12-14 15:54:46 +0100803cron_builder(
804 "WebRTC version update",
805 recipe = "update_webrtc_binary_version",
806 schedule = "0 4 * * *", # Every day at 4am.
807 service_account = "webrtc-version-updater@webrtc-ci.iam.gserviceaccount.com",
808)
809
Mirko Bonadeiab3ff262020-12-18 10:17:43 +0100810skipped_lkgr_bots = [
811 "iOS64 Sim Debug (iOS 14.0)",
812 "iOS64 Sim Debug (iOS 13)",
813 "iOS64 Sim Debug (iOS 12)",
814]
815
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200816lkgr_config = {
817 "project": "webrtc",
818 "source_url": WEBRTC_GIT,
819 "status_url": "https://webrtc-status.appspot.com",
820 "allowed_lag": 300, # hours
821 "allowed_gap": 12, # commits behind
Mirko Bonadei450e9b72020-07-10 19:19:22 +0000822 "error_recipients": "webrtc-troopers-robots@google.com",
Michael Achenbach09dd7dc2019-07-26 15:26:11 +0200823 "buckets": {
824 "webrtc/ci": {
825 # bucket alias: luci.webrtc.ci
Mirko Bonadeiab3ff262020-12-18 10:17:43 +0100826 "builders": [
827 b
828 for b in sorted(lkgr_builders)
829 if b not in skipped_lkgr_bots
830 ],
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200831 },
Michael Achenbach09dd7dc2019-07-26 15:26:11 +0200832 "chromium/webrtc.fyi": {
833 # bucket alias: luci.chromium.webrtc.fyi
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200834 "builders": [
835 "WebRTC Chromium FYI Android Builder (dbg)",
836 "WebRTC Chromium FYI Android Builder ARM64 (dbg)",
837 "WebRTC Chromium FYI Android Builder",
Mirko Bonadei04fe32f2020-05-01 00:45:58 +0200838 "WebRTC Chromium FYI Android Tests (dbg) (L Nexus5)",
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200839 "WebRTC Chromium FYI Android Tests (dbg) (M Nexus5X)",
840 "WebRTC Chromium FYI Linux Builder (dbg)",
841 "WebRTC Chromium FYI Linux Builder",
842 "WebRTC Chromium FYI Linux Tester",
843 "WebRTC Chromium FYI Mac Builder (dbg)",
844 "WebRTC Chromium FYI Mac Builder",
845 "WebRTC Chromium FYI Mac Tester",
846 "WebRTC Chromium FYI Win Builder (dbg)",
847 "WebRTC Chromium FYI Win Builder",
848 "WebRTC Chromium FYI Win10 Tester",
Mirko Bonadeif1ef9292021-03-25 18:07:48 +0100849 # TODO(bugs.webrtc.org/12616): Re-enable when the gclient
850 # runhooks step is fixed or skipped.
851 # "WebRTC Chromium FYI Win7 Tester",
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200852 "WebRTC Chromium FYI Win8 Tester",
Mirko Bonadei2f05cfe2020-11-25 10:41:58 +0100853 "WebRTC Chromium FYI ios-device",
854 "WebRTC Chromium FYI ios-simulator",
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200855 ],
856 },
857 },
858}
859
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100860cron_builder(
861 "WebRTC lkgr finder",
862 recipe = "lkgr_finder",
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200863 properties = {
864 "project": "webrtc",
865 "repo": WEBRTC_GIT,
866 "ref": "refs/heads/lkgr",
867 "lkgr_status_gs_path": "chromium-webrtc/lkgr-status",
868 "config": lkgr_config,
869 },
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100870 schedule = "*/10 * * * *", # Every 10 minutes.
871)