blob: 3f6277a621f8e05e81debe84b52e63f36cc9c30a [file] [log] [blame]
Oleh Prypin1a0593f2019-03-11 09:43:28 +01001#!/usr/bin/env lucicfg
2
3# Copyright (c) 2019 The WebRTC project authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7# https://chromium.googlesource.com/infra/luci/luci-go/+/master/lucicfg/doc/
8
Jeremy Leconteea2016b2020-07-01 09:47:22 +02009"""LUCI project configuration for WebRTC CQ and CI."""
10
11lucicfg.check_version("1.15.0")
Mirko Bonadei9189d9c2020-06-09 21:54:18 +020012
Oleh Prypin1a0593f2019-03-11 09:43:28 +010013WEBRTC_GIT = "https://webrtc.googlesource.com/src"
14WEBRTC_GERRIT = "https://webrtc-review.googlesource.com/src"
15
Mirko Bonadei32ca4862020-03-11 10:39:45 +010016GOMA_BACKEND_WEBRTC_RBE_PROD = {
Mirko Bonadei1e905342020-03-04 09:39:42 +010017 "$build/goma": {
18 "server_host": "goma.chromium.org",
Jeremy Leconteea2016b2020-07-01 09:47:22 +020019 "use_luci_auth": True,
20 },
Mirko Bonadei1e905342020-03-04 09:39:42 +010021}
22
Mirko Bonadei84360ca2020-03-12 08:35:48 +010023GOMA_BACKEND_WEBRTC_IOS_RBE_PROD = {
24 "$build/goma": {
25 "server_host": "goma.chromium.org",
Jeremy Leconteea2016b2020-07-01 09:47:22 +020026 "use_luci_auth": True,
Mirko Bonadei84360ca2020-03-12 08:35:48 +010027 },
Artem Titov1d47d312020-11-03 14:45:52 +010028 "$depot_tools/osx_sdk": {"sdk_version": "12a7209"},
Mirko Bonadei84360ca2020-03-12 08:35:48 +010029}
30
Patrik Höglund3916a972019-12-04 13:19:38 +010031GOMA_BACKEND_RBE_ATS_PROD = {
32 "$build/goma": {
33 "server_host": "goma.chromium.org",
Mirko Bonadei2688eca2020-03-25 09:15:54 +010034 "use_luci_auth": True,
Jeremy Leconteea2016b2020-07-01 09:47:22 +020035 "enable_ats": True,
36 },
Patrik Höglund3916a972019-12-04 13:19:38 +010037}
38
Mirko Bonadei5aafd472021-04-21 10:42:00 +020039# Disable ATS on CQ/try.
40GOMA_BACKEND_RBE_NO_ATS_PROD = {
41 "$build/goma": {
42 "server_host": "goma.chromium.org",
43 "use_luci_auth": True,
44 "enable_ats": False,
45 },
46}
47
Oleh Prypin1a0593f2019-03-11 09:43:28 +010048# Top-level configs:
49
50lucicfg.config(
51 config_dir = ".",
52 tracked_files = [
53 "commit-queue.cfg",
54 "cr-buildbucket.cfg",
55 "luci-logdog.cfg",
56 "luci-milo.cfg",
Oleh Prypin71d17422019-03-28 08:43:16 +010057 "luci-notify.cfg",
Oleh Prypin705b6a62019-04-03 23:10:51 +020058 "luci-notify/**/*",
Oleh Prypin1a0593f2019-03-11 09:43:28 +010059 "luci-scheduler.cfg",
60 "project.cfg",
61 ],
Jeremy Leconteea2016b2020-07-01 09:47:22 +020062 lint_checks = ["default"],
Oleh Prypin1a0593f2019-03-11 09:43:28 +010063)
64
65luci.project(
66 name = "webrtc",
67 buildbucket = "cr-buildbucket.appspot.com",
68 logdog = "luci-logdog.appspot.com",
69 milo = "luci-milo.appspot.com",
Oleh Prypin71d17422019-03-28 08:43:16 +010070 notify = "luci-notify.appspot.com",
Oleh Prypin1a0593f2019-03-11 09:43:28 +010071 scheduler = "luci-scheduler.appspot.com",
72 swarming = "chromium-swarm.appspot.com",
73 acls = [
74 acl.entry(
75 [acl.BUILDBUCKET_READER, acl.LOGDOG_READER, acl.PROJECT_CONFIGS_READER, acl.SCHEDULER_READER],
76 groups = ["all"],
77 ),
78 acl.entry(acl.LOGDOG_WRITER, groups = ["luci-logdog-chromium-writers"]),
79 acl.entry(acl.SCHEDULER_OWNER, groups = ["project-webrtc-admins"]),
80 ],
81)
82
83luci.logdog(
84 gs_bucket = "chromium-luci-logdog",
85)
86
87luci.milo(
88 logo = "https://storage.googleapis.com/chrome-infra/webrtc-logo-vert-retro-255x305.png",
89)
90
Owen Rodleya55c38e2020-06-15 11:16:10 +100091luci.notify(tree_closing_enabled = True)
92
Oleh Prypin1a0593f2019-03-11 09:43:28 +010093luci.cq(
94 status_host = "chromium-cq-status.appspot.com",
95 submit_max_burst = 1,
96 submit_burst_delay = 1 * time.minute,
97)
98
99luci.gitiles_poller(
100 name = "webrtc-gitiles-trigger-master",
101 bucket = "ci",
102 repo = WEBRTC_GIT,
103)
104
105# Bucket definitions:
106
107luci.bucket(
108 name = "try",
109 acls = [
110 acl.entry(acl.BUILDBUCKET_TRIGGERER, groups = [
111 "service-account-cq",
112 "project-webrtc-tryjob-access",
113 ]),
114 ],
115)
116
117luci.bucket(
118 name = "ci",
119 acls = [
Oleh Prypinf35939d2019-05-03 20:42:38 +0200120 acl.entry(acl.BUILDBUCKET_TRIGGERER, groups = [
121 "project-webrtc-ci-schedulers",
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100122 ]),
Jeremy Leconte2e25bb52020-07-16 09:38:56 +0200123 acl.entry(acl.BUILDBUCKET_TRIGGERER, groups = [
124 # Allow Pinpoint to trigger builds for bisection
125 "service-account-chromeperf",
126 ]),
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100127 ],
128)
129
130luci.bucket(
131 name = "perf",
132 acls = [
133 acl.entry(acl.BUILDBUCKET_TRIGGERER, users = [
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100134 "webrtc-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
135 ]),
136 ],
137)
138
139luci.bucket(
140 name = "cron",
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100141)
142
143# Commit queue definitions:
144
145luci.cq_group(
146 name = "cq",
147 tree_status_host = "webrtc-status.appspot.com",
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100148 watch = [cq.refset(repo = WEBRTC_GERRIT, refs = ["refs/heads/master"])],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100149 acls = [
150 acl.entry(acl.CQ_COMMITTER, groups = ["project-webrtc-committers"]),
151 acl.entry(acl.CQ_DRY_RUNNER, groups = ["project-webrtc-tryjob-access"]),
152 ],
153 retry_config = cq.RETRY_ALL_FAILURES,
Oleh Prypin5416f2d2019-08-02 09:43:15 +0200154 cancel_stale_tryjobs = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100155)
156
157luci.cq_group(
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100158 name = "cq_branch",
159 watch = [cq.refset(repo = WEBRTC_GERRIT, refs = ["refs/branch-heads/.+"])],
160 acls = [
161 acl.entry(acl.CQ_COMMITTER, groups = ["project-webrtc-committers"]),
162 acl.entry(acl.CQ_DRY_RUNNER, groups = ["project-webrtc-tryjob-access"]),
163 ],
164 retry_config = cq.RETRY_ALL_FAILURES,
Oleh Prypin5416f2d2019-08-02 09:43:15 +0200165 cancel_stale_tryjobs = True,
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100166)
167
168luci.cq_group(
169 name = "cq_infra",
170 watch = [cq.refset(repo = WEBRTC_GERRIT, refs = ["refs/heads/infra/config"])],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100171 acls = [
172 acl.entry(acl.CQ_COMMITTER, groups = ["project-webrtc-admins"]),
173 acl.entry(acl.CQ_DRY_RUNNER, groups = ["project-webrtc-tryjob-access"]),
174 ],
175 retry_config = cq.RETRY_ALL_FAILURES,
Oleh Prypin5416f2d2019-08-02 09:43:15 +0200176 cancel_stale_tryjobs = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100177)
178
179luci.cq_tryjob_verifier(
180 builder = "presubmit",
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100181 cq_group = "cq_infra",
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100182)
183
Oleh Prypine56cd9b2019-05-29 21:14:31 +0200184luci.cq_tryjob_verifier(
185 builder = "webrtc-internal:g3.webrtc-internal.try/internal_compile_lite",
186 owner_whitelist = ["project-webrtc-internal-tryjob-access"],
187 cq_group = "cq",
188)
189
Oleh Prypin71d17422019-03-28 08:43:16 +0100190# Notifier definitions:
191
192luci.notifier(
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200193 name = "post_submit_failure_notifier",
194 on_new_status = ["FAILURE"],
Oleh Prypin71d17422019-03-28 08:43:16 +0100195 notify_emails = ["webrtc-sheriffs-robots@google.com"],
Oleh Prypin705b6a62019-04-03 23:10:51 +0200196 template = luci.notifier_template(
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200197 name = "build_failure",
198 body = io.read_file("luci-notify/email-templates/build_failure.template"),
Oleh Prypin705b6a62019-04-03 23:10:51 +0200199 ),
Oleh Prypin71d17422019-03-28 08:43:16 +0100200)
201
202luci.notifier(
203 name = "cron_notifier",
Jeremy Leconte67a904e2020-07-08 11:48:18 +0200204 on_new_status = ["FAILURE", "INFRA_FAILURE"],
Mirko Bonadei450e9b72020-07-10 19:19:22 +0000205 notify_emails = ["webrtc-troopers-robots@google.com"],
Oleh Prypin705b6a62019-04-03 23:10:51 +0200206 template = luci.notifier_template(
207 name = "cron",
208 body = io.read_file("luci-notify/email-templates/cron.template"),
209 ),
Oleh Prypin71d17422019-03-28 08:43:16 +0100210)
211
Jeremy Leconte67a904e2020-07-08 11:48:18 +0200212luci.notifier(
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200213 name = "infra_failure_notifier",
Jeremy Leconte67a904e2020-07-08 11:48:18 +0200214 on_new_status = ["INFRA_FAILURE"],
Mirko Bonadei450e9b72020-07-10 19:19:22 +0000215 notify_emails = ["webrtc-troopers-robots@google.com"],
Jeremy Leconte67a904e2020-07-08 11:48:18 +0200216 template = luci.notifier_template(
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200217 name = "infra_failure",
218 body = io.read_file("luci-notify/email-templates/infra_failure.template"),
Jeremy Leconte67a904e2020-07-08 11:48:18 +0200219 ),
220)
221
Owen Rodley6b721ba2020-05-26 10:04:20 +1000222# Tree closer definitions:
223
224luci.tree_closer(
225 name = "webrtc_tree_closer",
226 tree_status_host = "webrtc-status.appspot.com",
Owen Rodley6b721ba2020-05-26 10:04:20 +1000227 # TODO: These step filters are copied verbatim from Gatekeeper, for testing
228 # that LUCI-Notify would take the exact same actions. Once we've switched
229 # over, this should be updated - several of these steps don't exist in
230 # WebRTC recipes.
231 failed_step_regexp = [
232 "bot_update",
233 "compile",
234 "gclient runhooks",
235 "runhooks",
236 "update",
237 "extract build",
238 "cleanup_temp",
239 "taskkill",
240 "compile",
241 "gn",
242 ],
243 failed_step_regexp_exclude = ".*\\(experimental\\).*",
244)
245
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100246# Recipe definitions:
247
248def recipe(recipe, pkg = "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"):
249 return luci.recipe(
250 name = recipe.split("/")[-1],
251 cipd_package = pkg,
252 cipd_version = "refs/heads/master",
253 recipe = recipe,
254 )
255
256recipe("chromium_trybot")
257recipe("run_presubmit")
258recipe("webrtc/auto_roll_webrtc_deps")
259recipe("webrtc/ios")
260recipe("webrtc/ios_api_framework")
261recipe("webrtc/libfuzzer")
262recipe("webrtc/more_configs")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100263recipe("webrtc/standalone")
Mirko Bonadei54bbd1c2020-12-14 15:54:46 +0100264recipe("webrtc/update_webrtc_binary_version")
Jeff Yoonf421c682020-12-17 00:16:22 -0800265recipe("lkgr_finder", pkg = "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100266
267# Console definitions:
268
269luci.console_view(name = "ci", title = "Main", repo = WEBRTC_GIT, header = "console-header.textpb")
270luci.console_view(name = "perf", title = "Perf", repo = WEBRTC_GIT, header = "console-header.textpb")
271luci.list_view(name = "cron", title = "Cron")
272luci.console_view(name = "fyi", title = "FYI", repo = WEBRTC_GIT, header = "console-header.textpb")
273luci.list_view(name = "try", title = "Tryserver")
274
275def add_milo(builder, views):
276 """Add Milo console entries for the builder.
277
278 Args:
279 builder: builder name (str).
280 views: dict where keys are names of consoles and values are either a
281 category for the console (str, pipe-separated) or True, which means
282 adding to a list view rather than a console.
283 """
284 for view_name, category in views.items():
285 if category == None:
286 continue
287 elif type(category) == "string":
288 category, _, short_name = category.rpartition("|")
289 luci.console_view_entry(
290 console_view = view_name,
291 builder = builder,
292 category = category or None,
293 short_name = short_name or None,
294 )
295 elif category == True:
296 luci.list_view_entry(
297 list_view = view_name,
298 builder = builder,
299 )
300 else:
301 fail("Unexpected value for category: %r" % category)
302
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200303lkgr_builders = {}
304
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100305# Builder-defining functions:
306
Artem Titarenkof6588b72019-11-14 14:28:11 +0100307def webrtc_builder(
308 name,
309 recipe = "standalone",
310 dimensions = {},
Mirko Bonadei825f9c22020-07-01 11:52:39 +0200311 properties = {},
Artem Titarenkof6588b72019-11-14 14:28:11 +0100312 priority = 30,
313 execution_timeout = 2 * time.hour,
Mirko Bonadeib7ef4602020-07-01 13:28:26 +0200314 goma_jobs = None,
Artem Titarenkof6588b72019-11-14 14:28:11 +0100315 **kwargs):
Mirko Bonadeib7ef4602020-07-01 13:28:26 +0200316 """WebRTC specific wrapper around luci.builder.
317
318 Args:
319 name: builder name (str).
320 recipe: string with the name of the recipe to run.
321 dimensions: dict of Swarming dimensions (strings) to search machines by.
322 properties: dict of properties to pass to the recipe (on top of the default ones).
323 priority: int [1-255] or None, indicating swarming task priority, lower is
324 more important. If None, defer the decision to Buildbucket service.
325 execution_timeout: int or None, how long to wait for a running build to finish before
326 forcefully aborting it and marking the build as timed out. If None,
327 defer the decision to Buildbucket service.
328 goma_jobs: int or None, number of jobs to be used by the builder. If None, defer the
329 decition to the goma module.
330 **kwargs: Pass on to webrtc_builder / luci.builder.
331 Returns:
332 A luci.builder.
333 """
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100334 dimensions = merge_dicts({"cpu": "x86-64"}, dimensions)
335
Mirko Bonadeib7ef4602020-07-01 13:28:26 +0200336 if goma_jobs != None:
337 goma_additional_params = {"$build/goma": {"jobs": goma_jobs}}
338 properties = merge_dicts(properties, goma_additional_params)
Ye Kuang8df20492020-10-15 14:54:17 +0900339 properties = merge_dicts(properties, {
340 "$recipe_engine/isolated": {
341 "server": "https://isolateserver.appspot.com",
342 },
343 })
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100344 return luci.builder(
345 name = name,
Oleh Prypin705b6a62019-04-03 23:10:51 +0200346 executable = recipe,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100347 dimensions = {k: v for k, v in dimensions.items() if v != None},
Mirko Bonadeib7ef4602020-07-01 13:28:26 +0200348 properties = properties,
Artem Titarenkof6588b72019-11-14 14:28:11 +0100349 execution_timeout = execution_timeout,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100350 priority = priority,
351 build_numbers = True,
352 swarming_tags = ["vpython:native-python-wrapper"],
353 **kwargs
354 )
355
356def ci_builder(
357 name,
358 ci_cat,
359 perf_cat = None,
360 fyi_cat = None,
361 properties = {},
362 dimensions = {},
363 prioritized = False,
Oleh Prypinf5432c52019-03-19 15:10:37 +0100364 enabled = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100365 **kwargs):
Oleh Prypine5677672019-04-25 12:21:35 +0200366 """Add a post-submit builder.
367
368 Args:
369 name: builder name (str).
370 ci_cat: the category + name for the /ci/ console, or None to omit from the console.
371 perf_cat: the category + name for the /perf/ console, or None to omit from the console.
372 fyi_cat: the category + name for the /fyi/ console, or None to omit from the console.
373 properties: dict of properties to pass to the recipe (on top of the default ones).
374 dimensions: dict of Swarming dimensions (strings) to search machines by.
375 prioritized: True to make this builder have a higher priority and never batch builds.
376 enabled: False to exclude this builder from consoles and failure notifications.
377 **kwargs: Pass on to webrtc_builder / luci.builder.
378 Returns:
379 A luci.builder.
380
381 Notifications are also disabled if a builder is not on either of /ci/ or /perf/ consoles.
382 """
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100383 if prioritized:
384 kwargs["triggering_policy"] = scheduler.greedy_batching(
385 max_batch_size = 1,
386 max_concurrent_invocations = 3,
387 )
388 kwargs["priority"] = 29
389
Oleh Prypinf5432c52019-03-19 15:10:37 +0100390 if enabled:
391 add_milo(name, {"ci": ci_cat, "perf": perf_cat, "fyi": fyi_cat})
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200392 if ci_cat:
393 lkgr_builders[name] = True
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100394 return webrtc_builder(
395 name = name,
Garrett Beaty4a106ce2020-09-09 14:51:23 -0700396 properties = merge_dicts({"builder_group": "client.webrtc"}, properties),
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100397 dimensions = merge_dicts({"pool": "luci.webrtc.ci"}, dimensions),
398 bucket = "ci",
399 service_account = "webrtc-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
Oleh Prypinf5432c52019-03-19 15:10:37 +0100400 triggered_by = ["webrtc-gitiles-trigger-master"] if enabled else None,
Oleh Prypin71d17422019-03-28 08:43:16 +0100401 repo = WEBRTC_GIT,
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200402 notifies = ["post_submit_failure_notifier", "webrtc_tree_closer", "infra_failure_notifier"] if enabled and (ci_cat or perf_cat) else None,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100403 **kwargs
404 )
405
406def try_builder(
407 name,
408 try_cat = True,
409 fyi_cat = None,
410 properties = {},
411 dimensions = {},
412 cq = {},
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100413 branch_cq = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100414 **kwargs):
Oleh Prypine5677672019-04-25 12:21:35 +0200415 """Add a pre-submit builder.
416
417 Args:
418 name: builder name (str).
419 try_cat: boolean, whether to include this builder in the /try/ console. See also: `add_milo`.
420 fyi_cat: the category + name for the /fyi/ console, or None to omit from the console.
421 properties: dict of properties to pass to the recipe (on top of the default ones).
422 dimensions: dict of Swarming dimensions (strings) to search machines by.
423 cq: None to exclude this from all commit queues, or a dict of kwargs for cq_tryjob_verifier.
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100424 branch_cq: False to exclude this builder just from the release-branch CQ.
Oleh Prypine5677672019-04-25 12:21:35 +0200425 **kwargs: Pass on to webrtc_builder / luci.builder.
426 Returns:
427 A luci.builder.
428 """
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100429 add_milo(name, {"try": try_cat, "fyi": fyi_cat})
430 if cq != None:
431 luci.cq_tryjob_verifier(
432 builder = name,
433 cq_group = "cq",
434 **cq
435 )
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100436 if branch_cq:
437 luci.cq_tryjob_verifier(
438 builder = name,
439 cq_group = "cq_branch",
440 **cq
441 )
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100442
443 return webrtc_builder(
444 name = name,
Garrett Beaty4a106ce2020-09-09 14:51:23 -0700445 properties = merge_dicts({"builder_group": "tryserver.webrtc"}, properties),
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100446 dimensions = merge_dicts({"pool": "luci.webrtc.try"}, dimensions),
447 bucket = "try",
448 service_account = "webrtc-try-builder@chops-service-accounts.iam.gserviceaccount.com",
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200449 notifies = ["infra_failure_notifier"],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100450 **kwargs
451 )
452
453def perf_builder(
454 name,
455 perf_cat,
456 recipe = "standalone",
457 properties = {},
458 dimensions = {},
459 **kwargs):
460 add_milo(name, {"perf": perf_cat})
461 return webrtc_builder(
462 name = name,
463 recipe = recipe,
Garrett Beaty4a106ce2020-09-09 14:51:23 -0700464 properties = merge_dicts({"builder_group": "client.webrtc.perf"}, properties),
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100465 dimensions = merge_dicts({"pool": "luci.webrtc.perf", "os": "Linux", "cpu": None}, dimensions),
466 bucket = "perf",
467 service_account = "webrtc-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
468 # log_base of 1.7 means:
469 # when there are P pending builds, LUCI will batch the first B builds.
470 # P: 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 ...
471 # B: 1 1 2 2 3 3 3 3 4 4 4 4 4 4 5 ...
Jeremy Leconteea2016b2020-07-01 09:47:22 +0200472 triggering_policy = scheduler.logarithmic_batching(log_base = 1.7),
Artem Titarenkof6588b72019-11-14 14:28:11 +0100473 execution_timeout = 3 * time.hour,
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200474 notifies = ["post_submit_failure_notifier", "infra_failure_notifier"],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100475 **kwargs
476 )
477
Mirko Bonadei12643772020-03-27 13:16:35 +0100478def android_perf_builder(
479 name,
480 perf_cat,
481 recipe = "standalone",
482 properties = {},
483 dimensions = {},
484 **kwargs):
485 return perf_builder(
486 name = name,
487 perf_cat = perf_cat,
488 recipe = recipe,
489 properties = merge_dicts(GOMA_BACKEND_WEBRTC_RBE_PROD, properties),
490 dimensions = dimensions,
491 **kwargs
492 )
493
494def linux_perf_builder(
495 name,
496 perf_cat,
497 recipe = "standalone",
498 properties = {},
499 dimensions = {},
500 **kwargs):
501 return perf_builder(
502 name = name,
503 perf_cat = perf_cat,
504 recipe = recipe,
505 properties = merge_dicts(GOMA_BACKEND_WEBRTC_RBE_PROD, properties),
506 dimensions = dimensions,
507 **kwargs
508 )
509
510def mac_perf_builder(
511 name,
512 perf_cat,
513 recipe = "standalone",
514 properties = {},
515 dimensions = {},
516 **kwargs):
517 return perf_builder(
518 name = name,
519 perf_cat = perf_cat,
520 recipe = recipe,
521 properties = merge_dicts(GOMA_BACKEND_WEBRTC_RBE_PROD, properties),
522 dimensions = dimensions,
523 **kwargs
524 )
525
526def win_perf_builder(
527 name,
528 perf_cat,
529 recipe = "standalone",
530 properties = {},
531 dimensions = {},
532 **kwargs):
533 return perf_builder(
534 name = name,
535 perf_cat = perf_cat,
536 recipe = recipe,
537 properties = merge_dicts(GOMA_BACKEND_RBE_ATS_PROD, properties),
538 dimensions = dimensions,
539 **kwargs
540 )
541
Mirko Bonadei54bbd1c2020-12-14 15:54:46 +0100542def cron_builder(name, dimensions = {}, service_account = None, **kwargs):
543 if service_account == None:
544 service_account = "chromium-webrtc-autoroll@webrtc-ci.iam.gserviceaccount.com"
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100545 add_milo(name, {"cron": True})
546 return webrtc_builder(
547 name = name,
548 dimensions = merge_dicts({"pool": "luci.webrtc.cron", "os": "Linux"}, dimensions),
549 bucket = "cron",
Mirko Bonadei54bbd1c2020-12-14 15:54:46 +0100550 service_account = service_account,
Oleh Prypin71d17422019-03-28 08:43:16 +0100551 notifies = ["cron_notifier"],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100552 **kwargs
553 )
554
555# Helpers:
556
557def merge_dicts(a, b):
558 """Return the result of merging two dicts.
559
560 If matching values are both dicts or both lists, they will be merged (non-recursively).
561
562 Args:
563 a: first dict.
564 b: second dict (takes priority).
565 Returns:
566 Merged dict.
567 """
568 a = dict(a)
569 for k, bv in b.items():
570 av = a.get(k)
571 if type(av) == "dict" and type(bv) == "dict":
572 a[k] = dict(av)
573 a[k].update(bv)
574 elif type(av) == "list" and type(bv) == "list":
575 a[k] = av + bv
576 else:
577 a[k] = bv
578 return a
579
580def normal_builder_factory(**common_kwargs):
581 def builder(*args, **kwargs):
582 return ci_builder(*args, **merge_dicts(common_kwargs, kwargs))
583
584 def try_job(*args, **kwargs):
585 return try_builder(*args, **merge_dicts(common_kwargs, kwargs))
586
587 return builder, try_job
588
589# Mixins:
590
591linux_builder, linux_try_job = normal_builder_factory(
592 dimensions = {"os": "Linux", "inside_docker": "0"},
Mirko Bonadei32ca4862020-03-11 10:39:45 +0100593 properties = GOMA_BACKEND_WEBRTC_RBE_PROD,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100594)
595
596android_builder, android_try_job = normal_builder_factory(
597 dimensions = {"os": "Linux"},
Mirko Bonadei32ca4862020-03-11 10:39:45 +0100598 properties = GOMA_BACKEND_WEBRTC_RBE_PROD,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100599)
600
Mirko Bonadei5aafd472021-04-21 10:42:00 +0200601def win_builder(*args, **kwargs):
602 return ci_builder(
603 *args,
604 **merge_dicts({
605 "dimensions": {"os": "Windows"},
606 "properties": GOMA_BACKEND_RBE_ATS_PROD,
607 }, kwargs)
608 )
609
610def win_try_job(*args, **kwargs):
611 return try_builder(
612 *args,
613 **merge_dicts({
614 "dimensions": {"os": "Windows"},
615 "properties": GOMA_BACKEND_RBE_NO_ATS_PROD,
616 }, kwargs)
617 )
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100618
619mac_builder, mac_try_job = normal_builder_factory(
620 dimensions = {"os": "Mac"},
Mirko Bonadei32ca4862020-03-11 10:39:45 +0100621 properties = GOMA_BACKEND_WEBRTC_RBE_PROD,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100622)
623
624ios_builder, ios_try_job = normal_builder_factory(
Mirko Bonadei8096c232020-08-26 21:44:42 +0200625 dimensions = {"os": "Mac-10.15"},
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100626 recipe = "ios",
Mirko Bonadei84360ca2020-03-12 08:35:48 +0100627 properties = GOMA_BACKEND_WEBRTC_IOS_RBE_PROD,
628 caches = [swarming.cache("osx_sdk")],
629)
630
631# Some iOS builders don't use goma because they build using
632# `use_xcode_clang=true` which is not supported by goma.
633# https://ci.chromium.org/p/webrtc/builders/ci/iOS%20API%20Framework%20Builder
634# https://ci.chromium.org/p/webrtc/builders/try/ios_api_framework
635ios_builder_no_goma, ios_try_job_no_goma = normal_builder_factory(
Mirko Bonadei8096c232020-08-26 21:44:42 +0200636 dimensions = {"os": "Mac-10.15"},
Mirko Bonadei84360ca2020-03-12 08:35:48 +0100637 recipe = "ios",
Artem Titov1d47d312020-11-03 14:45:52 +0100638 properties = {"$depot_tools/osx_sdk": {"sdk_version": "12a7209"}},
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100639 caches = [swarming.cache("osx_sdk")],
640)
641
642# Actual builder configuration:
643
644android_builder("Android32 (M Nexus5X)(dbg)", "Android|arm|dbg")
645android_try_job("android_compile_arm_dbg", cq = None)
646android_try_job("android_arm_dbg")
647android_builder("Android32 (M Nexus5X)", "Android|arm|rel")
648android_try_job("android_arm_rel")
649android_builder("Android32 Builder arm", "Android|arm|size", perf_cat = "Android|arm|Builder|", prioritized = True)
650android_try_job("android_compile_arm_rel")
Mirko Bonadei86b4f2d2020-07-08 14:24:48 +0200651android_perf_builder("Perf Android32 (M Nexus5)", "Android|arm|Tester|M Nexus5", triggered_by = ["Android32 Builder arm"])
Mirko Bonadei12643772020-03-27 13:16:35 +0100652android_perf_builder("Perf Android32 (M AOSP Nexus6)", "Android|arm|Tester|M AOSP Nexus6", triggered_by = ["Android32 Builder arm"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100653android_try_job("android_compile_arm64_dbg", cq = None)
654android_try_job("android_arm64_dbg", cq = None)
655android_builder("Android64 (M Nexus5X)", "Android|arm64|rel")
656android_try_job("android_arm64_rel")
657android_builder("Android64 Builder arm64", "Android|arm64|size", perf_cat = "Android|arm64|Builder|", prioritized = True)
Mirko Bonadei12643772020-03-27 13:16:35 +0100658android_perf_builder("Perf Android64 (M Nexus5X)", "Android|arm64|Tester|M Nexus5X", triggered_by = ["Android64 Builder arm64"])
659android_perf_builder("Perf Android64 (O Pixel2)", "Android|arm64|Tester|O Pixel2", triggered_by = ["Android64 Builder arm64"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100660android_try_job("android_compile_arm64_rel")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100661android_builder("Android64 Builder x64 (dbg)", "Android|x64|dbg")
662android_try_job("android_compile_x64_dbg")
663android_try_job("android_compile_x64_rel", cq = None)
664android_builder("Android32 Builder x86 (dbg)", "Android|x86|dbg")
665android_try_job("android_compile_x86_dbg")
666android_builder("Android32 Builder x86", "Android|x86|rel")
667android_try_job("android_compile_x86_rel")
668android_builder("Android32 (more configs)", "Android|arm|more", recipe = "more_configs")
669android_try_job("android_arm_more_configs", recipe = "more_configs")
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100670android_try_job("android_chromium_compile", recipe = "chromium_trybot", branch_cq = False)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100671
672ios_builder("iOS32 Debug", "iOS|arm|dbg")
673ios_try_job("ios_compile_arm_dbg")
674ios_builder("iOS32 Release", "iOS|arm|rel")
675ios_try_job("ios_compile_arm_rel")
676ios_builder("iOS64 Debug", "iOS|arm64|dbg")
677ios_try_job("ios_compile_arm64_dbg")
678ios_builder("iOS64 Release", "iOS|arm64|rel")
679ios_try_job("ios_compile_arm64_rel")
Artem Titov1d47d312020-11-03 14:45:52 +0100680ios_builder("iOS64 Sim Debug (iOS 14.0)", "iOS|x64|14")
Mirko Bonadeif1e0c0b2021-01-12 13:05:52 +0100681ios_try_job("ios_sim_x64_dbg_ios14", cq = None)
Artem Titov1d47d312020-11-03 14:45:52 +0100682ios_builder("iOS64 Sim Debug (iOS 13)", "iOS|x64|13")
Artem Titove973dd62020-11-03 18:05:21 +0100683ios_try_job("ios_sim_x64_dbg_ios13", cq = None)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100684ios_builder("iOS64 Sim Debug (iOS 12)", "iOS|x64|12")
Andrey Logvin34ac6592021-02-04 13:03:49 +0000685
686# TODO(crbug.com/1174112): return ios_sim_x64_dbg_ios12 to cq.
687ios_try_job("ios_sim_x64_dbg_ios12", cq = None)
Mirko Bonadei84360ca2020-03-12 08:35:48 +0100688ios_builder_no_goma("iOS API Framework Builder", "iOS|fat|size", recipe = "ios_api_framework", prioritized = True)
Jeremy Leconteea2016b2020-07-01 09:47:22 +0200689ios_try_job_no_goma("ios_api_framework", recipe = "ios_api_framework", cq = None)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100690
691linux_builder("Linux32 Debug", "Linux|x86|dbg")
692linux_try_job("linux_x86_dbg")
693linux_builder("Linux32 Release", "Linux|x86|rel")
694linux_try_job("linux_x86_rel")
695linux_builder("Linux64 Debug", "Linux|x64|dbg")
696linux_try_job("linux_dbg", cq = None)
697linux_try_job("linux_compile_dbg")
698linux_builder("Linux64 Release", "Linux|x64|rel")
699linux_try_job("linux_rel")
700linux_builder("Linux64 Builder", "Linux|x64|size", perf_cat = "Linux|x64|Builder|", prioritized = True)
701linux_try_job("linux_compile_rel")
Mirko Bonadei12643772020-03-27 13:16:35 +0100702linux_perf_builder("Perf Linux Trusty", "Linux|x64|Tester|Trusty", triggered_by = ["Linux64 Builder"])
703linux_perf_builder("Perf Linux Xenial", None, triggered_by = ["Linux64 Builder"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100704linux_builder("Linux32 Debug (ARM)", "Linux|arm|dbg")
705linux_try_job("linux_compile_arm_dbg")
706linux_builder("Linux32 Release (ARM)", "Linux|arm|rel")
707linux_try_job("linux_compile_arm_rel")
708linux_builder("Linux64 Debug (ARM)", "Linux|arm64|dbg")
709linux_try_job("linux_compile_arm64_dbg")
710linux_builder("Linux64 Release (ARM)", "Linux|arm64|rel")
711linux_try_job("linux_compile_arm64_rel")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100712linux_builder("Linux Asan", "Linux|x64|asan")
713linux_try_job("linux_asan")
714linux_builder("Linux MSan", "Linux|x64|msan")
715linux_try_job("linux_msan")
716linux_builder("Linux Tsan v2", "Linux|x64|tsan")
717linux_try_job("linux_tsan2")
718linux_builder("Linux UBSan", "Linux|x64|ubsan")
719linux_try_job("linux_ubsan")
720linux_builder("Linux UBSan vptr", "Linux|x64|ubsan")
721linux_try_job("linux_ubsan_vptr")
722linux_builder("Linux64 Release (Libfuzzer)", "Linux|x64|fuzz", recipe = "libfuzzer")
723linux_try_job("linux_libfuzzer_rel", recipe = "libfuzzer")
724linux_builder("Linux (more configs)", "Linux|x64|more", recipe = "more_configs")
725linux_try_job("linux_more_configs", recipe = "more_configs")
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100726linux_try_job("linux_chromium_compile", recipe = "chromium_trybot", branch_cq = False)
Mirko Bonadei40de50f2019-11-07 11:23:47 +0100727linux_try_job("linux_chromium_compile_dbg", recipe = "chromium_trybot", branch_cq = False)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100728
729mac_builder("Mac64 Debug", "Mac|x64|dbg")
730mac_try_job("mac_dbg", cq = None)
731mac_try_job("mac_compile_dbg")
732mac_builder("Mac64 Release", "Mac|x64|rel")
733mac_try_job("mac_rel")
734mac_try_job("mac_compile_rel", cq = None)
735mac_builder("Mac64 Builder", ci_cat = None, perf_cat = "Mac|x64|Builder|")
Mirko Bonadei12643772020-03-27 13:16:35 +0100736mac_perf_builder("Perf Mac 10.11", "Mac|x64|Tester|10.11", triggered_by = ["Mac64 Builder"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100737mac_builder("Mac Asan", "Mac|x64|asan")
738mac_try_job("mac_asan")
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100739mac_try_job("mac_chromium_compile", recipe = "chromium_trybot", dimensions = {"cores": "8"}, branch_cq = False)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100740
741win_builder("Win32 Debug", "Win MSVC|x86|dbg")
742win_try_job("win_x86_msvc_dbg")
743win_try_job("win_compile_x86_msvc_dbg", cq = None)
Oleh Prypin63041b82019-03-19 22:56:01 +0000744win_builder("Win32 Release", "Win MSVC|x86|rel")
745win_try_job("win_x86_msvc_rel")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100746win_try_job("win_compile_x86_msvc_rel", cq = None)
747win_builder("Win64 Debug", "Win MSVC|x64|dbg")
748win_try_job("win_x64_msvc_dbg")
749win_try_job("win_compile_x64_msvc_dbg", cq = None)
750win_builder("Win64 Release", "Win MSVC|x64|rel")
751win_try_job("win_x64_msvc_rel")
752win_try_job("win_compile_x64_msvc_rel", cq = None)
753win_builder("Win32 Debug (Clang)", "Win Clang|x86|dbg")
754win_try_job("win_x86_clang_dbg", cq = None)
755win_try_job("win_compile_x86_clang_dbg")
756win_builder("Win32 Release (Clang)", "Win Clang|x86|rel")
757win_try_job("win_x86_clang_rel")
758win_try_job("win_compile_x86_clang_rel", cq = None)
759win_builder("Win32 Builder (Clang)", ci_cat = None, perf_cat = "Win|x86|Builder|")
Mirko Bonadei12643772020-03-27 13:16:35 +0100760win_perf_builder("Perf Win7", "Win|x86|Tester|7", triggered_by = ["Win32 Builder (Clang)"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100761win_builder("Win64 Debug (Clang)", "Win Clang|x64|dbg")
762win_try_job("win_x64_clang_dbg", cq = None)
763win_try_job("win_x64_clang_dbg_win8", cq = None)
764win_try_job("win_x64_clang_dbg_win10", cq = None)
765win_try_job("win_compile_x64_clang_dbg")
766win_builder("Win64 Release (Clang)", "Win Clang|x64|rel")
767win_try_job("win_x64_clang_rel", cq = None)
768win_try_job("win_compile_x64_clang_rel")
Mirko Bonadeiafbe33d2019-05-31 09:27:54 +0200769win_builder("Win64 ASan", "Win Clang|x64|asan")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100770win_try_job("win_asan")
771win_builder("Win64 UWP", ci_cat = None, fyi_cat = "")
772win_try_job("win_x64_uwp", cq = None, try_cat = None, fyi_cat = "")
773win_builder("Win (more configs)", "Win Clang|x86|more", recipe = "more_configs")
774win_try_job("win_x86_more_configs", recipe = "more_configs")
Mirko Bonadei825f9c22020-07-01 11:52:39 +0200775win_try_job("win_chromium_compile", recipe = "chromium_trybot", branch_cq = False, goma_jobs = 150)
776win_try_job("win_chromium_compile_dbg", recipe = "chromium_trybot", branch_cq = False, goma_jobs = 150)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100777
778linux_try_job(
779 "presubmit",
780 recipe = "run_presubmit",
781 properties = {"repo_name": "webrtc", "runhooks": True},
782 priority = 28,
783 cq = {"disable_reuse": True},
784)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100785
786cron_builder(
787 "Auto-roll - WebRTC DEPS",
788 recipe = "auto_roll_webrtc_deps",
Oleh Prypindc68a722019-06-25 10:43:13 +0200789 schedule = "0 */2 * * *", # Every 2 hours.
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100790)
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200791
Mirko Bonadei54bbd1c2020-12-14 15:54:46 +0100792cron_builder(
793 "WebRTC version update",
794 recipe = "update_webrtc_binary_version",
795 schedule = "0 4 * * *", # Every day at 4am.
796 service_account = "webrtc-version-updater@webrtc-ci.iam.gserviceaccount.com",
797)
798
Mirko Bonadeiab3ff262020-12-18 10:17:43 +0100799skipped_lkgr_bots = [
800 "iOS64 Sim Debug (iOS 14.0)",
801 "iOS64 Sim Debug (iOS 13)",
802 "iOS64 Sim Debug (iOS 12)",
803]
804
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200805lkgr_config = {
806 "project": "webrtc",
807 "source_url": WEBRTC_GIT,
808 "status_url": "https://webrtc-status.appspot.com",
809 "allowed_lag": 300, # hours
810 "allowed_gap": 12, # commits behind
Mirko Bonadei450e9b72020-07-10 19:19:22 +0000811 "error_recipients": "webrtc-troopers-robots@google.com",
Michael Achenbach09dd7dc2019-07-26 15:26:11 +0200812 "buckets": {
813 "webrtc/ci": {
814 # bucket alias: luci.webrtc.ci
Mirko Bonadeiab3ff262020-12-18 10:17:43 +0100815 "builders": [
816 b
817 for b in sorted(lkgr_builders)
818 if b not in skipped_lkgr_bots
819 ],
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200820 },
Michael Achenbach09dd7dc2019-07-26 15:26:11 +0200821 "chromium/webrtc.fyi": {
822 # bucket alias: luci.chromium.webrtc.fyi
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200823 "builders": [
824 "WebRTC Chromium FYI Android Builder (dbg)",
825 "WebRTC Chromium FYI Android Builder ARM64 (dbg)",
826 "WebRTC Chromium FYI Android Builder",
Mirko Bonadei04fe32f2020-05-01 00:45:58 +0200827 "WebRTC Chromium FYI Android Tests (dbg) (L Nexus5)",
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200828 "WebRTC Chromium FYI Android Tests (dbg) (M Nexus5X)",
829 "WebRTC Chromium FYI Linux Builder (dbg)",
830 "WebRTC Chromium FYI Linux Builder",
831 "WebRTC Chromium FYI Linux Tester",
832 "WebRTC Chromium FYI Mac Builder (dbg)",
833 "WebRTC Chromium FYI Mac Builder",
834 "WebRTC Chromium FYI Mac Tester",
835 "WebRTC Chromium FYI Win Builder (dbg)",
836 "WebRTC Chromium FYI Win Builder",
837 "WebRTC Chromium FYI Win10 Tester",
Mirko Bonadeif1ef9292021-03-25 18:07:48 +0100838 # TODO(bugs.webrtc.org/12616): Re-enable when the gclient
839 # runhooks step is fixed or skipped.
840 # "WebRTC Chromium FYI Win7 Tester",
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200841 "WebRTC Chromium FYI Win8 Tester",
Mirko Bonadei2f05cfe2020-11-25 10:41:58 +0100842 "WebRTC Chromium FYI ios-device",
843 "WebRTC Chromium FYI ios-simulator",
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200844 ],
845 },
846 },
847}
848
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100849cron_builder(
850 "WebRTC lkgr finder",
851 recipe = "lkgr_finder",
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200852 properties = {
853 "project": "webrtc",
854 "repo": WEBRTC_GIT,
855 "ref": "refs/heads/lkgr",
856 "lkgr_status_gs_path": "chromium-webrtc/lkgr-status",
857 "config": lkgr_config,
858 },
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100859 schedule = "*/10 * * * *", # Every 10 minutes.
860)