blob: 4f56b49f4f22c77486a879f49be1cbcfb13f1b07 [file] [log] [blame]
Oleh Prypin1a0593f2019-03-11 09:43:28 +01001#!/usr/bin/env lucicfg
2
3# Copyright (c) 2019 The WebRTC project authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7# https://chromium.googlesource.com/infra/luci/luci-go/+/master/lucicfg/doc/
8
Jeremy Leconteea2016b2020-07-01 09:47:22 +02009"""LUCI project configuration for WebRTC CQ and CI."""
10
11lucicfg.check_version("1.15.0")
Mirko Bonadei9189d9c2020-06-09 21:54:18 +020012
Oleh Prypin1a0593f2019-03-11 09:43:28 +010013WEBRTC_GIT = "https://webrtc.googlesource.com/src"
14WEBRTC_GERRIT = "https://webrtc-review.googlesource.com/src"
15
Mirko Bonadei32ca4862020-03-11 10:39:45 +010016GOMA_BACKEND_WEBRTC_RBE_PROD = {
Mirko Bonadei1e905342020-03-04 09:39:42 +010017 "$build/goma": {
18 "server_host": "goma.chromium.org",
Jeremy Leconteea2016b2020-07-01 09:47:22 +020019 "use_luci_auth": True,
20 },
Mirko Bonadei1e905342020-03-04 09:39:42 +010021}
22
Mirko Bonadei84360ca2020-03-12 08:35:48 +010023GOMA_BACKEND_WEBRTC_IOS_RBE_PROD = {
24 "$build/goma": {
25 "server_host": "goma.chromium.org",
Jeremy Leconteea2016b2020-07-01 09:47:22 +020026 "use_luci_auth": True,
Mirko Bonadei84360ca2020-03-12 08:35:48 +010027 },
28 "$depot_tools/osx_sdk": {"sdk_version": "11c29"},
29}
30
Patrik Höglund3916a972019-12-04 13:19:38 +010031GOMA_BACKEND_RBE_ATS_PROD = {
32 "$build/goma": {
33 "server_host": "goma.chromium.org",
Mirko Bonadei2688eca2020-03-25 09:15:54 +010034 "use_luci_auth": True,
Jeremy Leconteea2016b2020-07-01 09:47:22 +020035 "enable_ats": True,
36 },
Patrik Höglund3916a972019-12-04 13:19:38 +010037}
38
Oleh Prypin1a0593f2019-03-11 09:43:28 +010039# Top-level configs:
40
Owen Rodley6b721ba2020-05-26 10:04:20 +100041# Enable luci.tree_closer.
42lucicfg.enable_experiment("crbug.com/1054172")
43
Oleh Prypin1a0593f2019-03-11 09:43:28 +010044lucicfg.config(
45 config_dir = ".",
46 tracked_files = [
47 "commit-queue.cfg",
48 "cr-buildbucket.cfg",
49 "luci-logdog.cfg",
50 "luci-milo.cfg",
Oleh Prypin71d17422019-03-28 08:43:16 +010051 "luci-notify.cfg",
Oleh Prypin705b6a62019-04-03 23:10:51 +020052 "luci-notify/**/*",
Oleh Prypin1a0593f2019-03-11 09:43:28 +010053 "luci-scheduler.cfg",
54 "project.cfg",
55 ],
Jeremy Leconteea2016b2020-07-01 09:47:22 +020056 lint_checks = ["default"],
Oleh Prypin1a0593f2019-03-11 09:43:28 +010057)
58
59luci.project(
60 name = "webrtc",
61 buildbucket = "cr-buildbucket.appspot.com",
62 logdog = "luci-logdog.appspot.com",
63 milo = "luci-milo.appspot.com",
Oleh Prypin71d17422019-03-28 08:43:16 +010064 notify = "luci-notify.appspot.com",
Oleh Prypin1a0593f2019-03-11 09:43:28 +010065 scheduler = "luci-scheduler.appspot.com",
66 swarming = "chromium-swarm.appspot.com",
67 acls = [
68 acl.entry(
69 [acl.BUILDBUCKET_READER, acl.LOGDOG_READER, acl.PROJECT_CONFIGS_READER, acl.SCHEDULER_READER],
70 groups = ["all"],
71 ),
72 acl.entry(acl.LOGDOG_WRITER, groups = ["luci-logdog-chromium-writers"]),
73 acl.entry(acl.SCHEDULER_OWNER, groups = ["project-webrtc-admins"]),
74 ],
75)
76
77luci.logdog(
78 gs_bucket = "chromium-luci-logdog",
79)
80
81luci.milo(
82 logo = "https://storage.googleapis.com/chrome-infra/webrtc-logo-vert-retro-255x305.png",
83)
84
Owen Rodleya55c38e2020-06-15 11:16:10 +100085luci.notify(tree_closing_enabled = True)
86
Oleh Prypin1a0593f2019-03-11 09:43:28 +010087luci.cq(
88 status_host = "chromium-cq-status.appspot.com",
89 submit_max_burst = 1,
90 submit_burst_delay = 1 * time.minute,
91)
92
93luci.gitiles_poller(
94 name = "webrtc-gitiles-trigger-master",
95 bucket = "ci",
96 repo = WEBRTC_GIT,
97)
98
99# Bucket definitions:
100
101luci.bucket(
102 name = "try",
103 acls = [
104 acl.entry(acl.BUILDBUCKET_TRIGGERER, groups = [
105 "service-account-cq",
106 "project-webrtc-tryjob-access",
107 ]),
108 ],
109)
110
111luci.bucket(
112 name = "ci",
113 acls = [
Oleh Prypinf35939d2019-05-03 20:42:38 +0200114 acl.entry(acl.BUILDBUCKET_TRIGGERER, groups = [
115 "project-webrtc-ci-schedulers",
116 ], users = [
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100117 "luci-scheduler@appspot.gserviceaccount.com",
118 ]),
Jeremy Leconte2e25bb52020-07-16 09:38:56 +0200119 acl.entry(acl.BUILDBUCKET_TRIGGERER, groups = [
120 # Allow Pinpoint to trigger builds for bisection
121 "service-account-chromeperf",
122 ]),
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100123 ],
124)
125
126luci.bucket(
127 name = "perf",
128 acls = [
129 acl.entry(acl.BUILDBUCKET_TRIGGERER, users = [
130 "luci-scheduler@appspot.gserviceaccount.com",
131 "webrtc-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
132 ]),
133 ],
134)
135
136luci.bucket(
137 name = "cron",
138 acls = [
139 acl.entry(acl.BUILDBUCKET_TRIGGERER, users = [
140 "luci-scheduler@appspot.gserviceaccount.com",
141 ]),
142 ],
143)
144
145# Commit queue definitions:
146
147luci.cq_group(
148 name = "cq",
149 tree_status_host = "webrtc-status.appspot.com",
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100150 watch = [cq.refset(repo = WEBRTC_GERRIT, refs = ["refs/heads/master"])],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100151 acls = [
152 acl.entry(acl.CQ_COMMITTER, groups = ["project-webrtc-committers"]),
153 acl.entry(acl.CQ_DRY_RUNNER, groups = ["project-webrtc-tryjob-access"]),
154 ],
155 retry_config = cq.RETRY_ALL_FAILURES,
Oleh Prypin5416f2d2019-08-02 09:43:15 +0200156 cancel_stale_tryjobs = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100157)
158
159luci.cq_group(
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100160 name = "cq_branch",
161 watch = [cq.refset(repo = WEBRTC_GERRIT, refs = ["refs/branch-heads/.+"])],
162 acls = [
163 acl.entry(acl.CQ_COMMITTER, groups = ["project-webrtc-committers"]),
164 acl.entry(acl.CQ_DRY_RUNNER, groups = ["project-webrtc-tryjob-access"]),
165 ],
166 retry_config = cq.RETRY_ALL_FAILURES,
Oleh Prypin5416f2d2019-08-02 09:43:15 +0200167 cancel_stale_tryjobs = True,
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100168)
169
170luci.cq_group(
171 name = "cq_infra",
172 watch = [cq.refset(repo = WEBRTC_GERRIT, refs = ["refs/heads/infra/config"])],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100173 acls = [
174 acl.entry(acl.CQ_COMMITTER, groups = ["project-webrtc-admins"]),
175 acl.entry(acl.CQ_DRY_RUNNER, groups = ["project-webrtc-tryjob-access"]),
176 ],
177 retry_config = cq.RETRY_ALL_FAILURES,
Oleh Prypin5416f2d2019-08-02 09:43:15 +0200178 cancel_stale_tryjobs = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100179)
180
181luci.cq_tryjob_verifier(
182 builder = "presubmit",
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100183 cq_group = "cq_infra",
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100184)
185
Oleh Prypine56cd9b2019-05-29 21:14:31 +0200186luci.cq_tryjob_verifier(
187 builder = "webrtc-internal:g3.webrtc-internal.try/internal_compile_lite",
188 owner_whitelist = ["project-webrtc-internal-tryjob-access"],
189 cq_group = "cq",
190)
191
Oleh Prypin71d17422019-03-28 08:43:16 +0100192# Notifier definitions:
193
194luci.notifier(
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200195 name = "post_submit_failure_notifier",
196 on_new_status = ["FAILURE"],
Oleh Prypin71d17422019-03-28 08:43:16 +0100197 notify_emails = ["webrtc-sheriffs-robots@google.com"],
Oleh Prypin705b6a62019-04-03 23:10:51 +0200198 template = luci.notifier_template(
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200199 name = "build_failure",
200 body = io.read_file("luci-notify/email-templates/build_failure.template"),
Oleh Prypin705b6a62019-04-03 23:10:51 +0200201 ),
Oleh Prypin71d17422019-03-28 08:43:16 +0100202)
203
204luci.notifier(
205 name = "cron_notifier",
Jeremy Leconte67a904e2020-07-08 11:48:18 +0200206 on_new_status = ["FAILURE", "INFRA_FAILURE"],
Mirko Bonadei450e9b72020-07-10 19:19:22 +0000207 notify_emails = ["webrtc-troopers-robots@google.com"],
Oleh Prypin705b6a62019-04-03 23:10:51 +0200208 template = luci.notifier_template(
209 name = "cron",
210 body = io.read_file("luci-notify/email-templates/cron.template"),
211 ),
Oleh Prypin71d17422019-03-28 08:43:16 +0100212)
213
Jeremy Leconte67a904e2020-07-08 11:48:18 +0200214luci.notifier(
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200215 name = "infra_failure_notifier",
Jeremy Leconte67a904e2020-07-08 11:48:18 +0200216 on_new_status = ["INFRA_FAILURE"],
Mirko Bonadei450e9b72020-07-10 19:19:22 +0000217 notify_emails = ["webrtc-troopers-robots@google.com"],
Jeremy Leconte67a904e2020-07-08 11:48:18 +0200218 template = luci.notifier_template(
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200219 name = "infra_failure",
220 body = io.read_file("luci-notify/email-templates/infra_failure.template"),
Jeremy Leconte67a904e2020-07-08 11:48:18 +0200221 ),
222)
223
Owen Rodley6b721ba2020-05-26 10:04:20 +1000224# Tree closer definitions:
225
226luci.tree_closer(
227 name = "webrtc_tree_closer",
228 tree_status_host = "webrtc-status.appspot.com",
Owen Rodley6b721ba2020-05-26 10:04:20 +1000229 # TODO: These step filters are copied verbatim from Gatekeeper, for testing
230 # that LUCI-Notify would take the exact same actions. Once we've switched
231 # over, this should be updated - several of these steps don't exist in
232 # WebRTC recipes.
233 failed_step_regexp = [
234 "bot_update",
235 "compile",
236 "gclient runhooks",
237 "runhooks",
238 "update",
239 "extract build",
240 "cleanup_temp",
241 "taskkill",
242 "compile",
243 "gn",
244 ],
245 failed_step_regexp_exclude = ".*\\(experimental\\).*",
246)
247
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100248# Recipe definitions:
249
250def recipe(recipe, pkg = "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"):
251 return luci.recipe(
252 name = recipe.split("/")[-1],
253 cipd_package = pkg,
254 cipd_version = "refs/heads/master",
255 recipe = recipe,
256 )
257
258recipe("chromium_trybot")
259recipe("run_presubmit")
260recipe("webrtc/auto_roll_webrtc_deps")
261recipe("webrtc/ios")
262recipe("webrtc/ios_api_framework")
263recipe("webrtc/libfuzzer")
264recipe("webrtc/more_configs")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100265recipe("webrtc/standalone")
266recipe("lkgr_finder", pkg = "infra/recipe_bundles/chromium.googlesource.com/infra/infra")
267
268# Console definitions:
269
270luci.console_view(name = "ci", title = "Main", repo = WEBRTC_GIT, header = "console-header.textpb")
271luci.console_view(name = "perf", title = "Perf", repo = WEBRTC_GIT, header = "console-header.textpb")
272luci.list_view(name = "cron", title = "Cron")
273luci.console_view(name = "fyi", title = "FYI", repo = WEBRTC_GIT, header = "console-header.textpb")
274luci.list_view(name = "try", title = "Tryserver")
275
276def add_milo(builder, views):
277 """Add Milo console entries for the builder.
278
279 Args:
280 builder: builder name (str).
281 views: dict where keys are names of consoles and values are either a
282 category for the console (str, pipe-separated) or True, which means
283 adding to a list view rather than a console.
284 """
285 for view_name, category in views.items():
286 if category == None:
287 continue
288 elif type(category) == "string":
289 category, _, short_name = category.rpartition("|")
290 luci.console_view_entry(
291 console_view = view_name,
292 builder = builder,
293 category = category or None,
294 short_name = short_name or None,
295 )
296 elif category == True:
297 luci.list_view_entry(
298 list_view = view_name,
299 builder = builder,
300 )
301 else:
302 fail("Unexpected value for category: %r" % category)
303
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200304lkgr_builders = {}
305
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100306# Builder-defining functions:
307
Artem Titarenkof6588b72019-11-14 14:28:11 +0100308def webrtc_builder(
309 name,
310 recipe = "standalone",
311 dimensions = {},
Mirko Bonadei825f9c22020-07-01 11:52:39 +0200312 properties = {},
Artem Titarenkof6588b72019-11-14 14:28:11 +0100313 priority = 30,
314 execution_timeout = 2 * time.hour,
Mirko Bonadeib7ef4602020-07-01 13:28:26 +0200315 goma_jobs = None,
Artem Titarenkof6588b72019-11-14 14:28:11 +0100316 **kwargs):
Mirko Bonadeib7ef4602020-07-01 13:28:26 +0200317 """WebRTC specific wrapper around luci.builder.
318
319 Args:
320 name: builder name (str).
321 recipe: string with the name of the recipe to run.
322 dimensions: dict of Swarming dimensions (strings) to search machines by.
323 properties: dict of properties to pass to the recipe (on top of the default ones).
324 priority: int [1-255] or None, indicating swarming task priority, lower is
325 more important. If None, defer the decision to Buildbucket service.
326 execution_timeout: int or None, how long to wait for a running build to finish before
327 forcefully aborting it and marking the build as timed out. If None,
328 defer the decision to Buildbucket service.
329 goma_jobs: int or None, number of jobs to be used by the builder. If None, defer the
330 decition to the goma module.
331 **kwargs: Pass on to webrtc_builder / luci.builder.
332 Returns:
333 A luci.builder.
334 """
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100335 dimensions = merge_dicts({"cpu": "x86-64"}, dimensions)
336
Mirko Bonadeib7ef4602020-07-01 13:28:26 +0200337 if goma_jobs != None:
338 goma_additional_params = {"$build/goma": {"jobs": goma_jobs}}
339 properties = merge_dicts(properties, goma_additional_params)
Mirko Bonadei825f9c22020-07-01 11:52:39 +0200340
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100341 return luci.builder(
342 name = name,
Oleh Prypin705b6a62019-04-03 23:10:51 +0200343 executable = recipe,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100344 dimensions = {k: v for k, v in dimensions.items() if v != None},
Mirko Bonadeib7ef4602020-07-01 13:28:26 +0200345 properties = properties,
Artem Titarenkof6588b72019-11-14 14:28:11 +0100346 execution_timeout = execution_timeout,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100347 priority = priority,
348 build_numbers = True,
349 swarming_tags = ["vpython:native-python-wrapper"],
350 **kwargs
351 )
352
353def ci_builder(
354 name,
355 ci_cat,
356 perf_cat = None,
357 fyi_cat = None,
358 properties = {},
359 dimensions = {},
360 prioritized = False,
Oleh Prypinf5432c52019-03-19 15:10:37 +0100361 enabled = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100362 **kwargs):
Oleh Prypine5677672019-04-25 12:21:35 +0200363 """Add a post-submit builder.
364
365 Args:
366 name: builder name (str).
367 ci_cat: the category + name for the /ci/ console, or None to omit from the console.
368 perf_cat: the category + name for the /perf/ console, or None to omit from the console.
369 fyi_cat: the category + name for the /fyi/ console, or None to omit from the console.
370 properties: dict of properties to pass to the recipe (on top of the default ones).
371 dimensions: dict of Swarming dimensions (strings) to search machines by.
372 prioritized: True to make this builder have a higher priority and never batch builds.
373 enabled: False to exclude this builder from consoles and failure notifications.
374 **kwargs: Pass on to webrtc_builder / luci.builder.
375 Returns:
376 A luci.builder.
377
378 Notifications are also disabled if a builder is not on either of /ci/ or /perf/ consoles.
379 """
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100380 if prioritized:
381 kwargs["triggering_policy"] = scheduler.greedy_batching(
382 max_batch_size = 1,
383 max_concurrent_invocations = 3,
384 )
385 kwargs["priority"] = 29
386
Oleh Prypinf5432c52019-03-19 15:10:37 +0100387 if enabled:
388 add_milo(name, {"ci": ci_cat, "perf": perf_cat, "fyi": fyi_cat})
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200389 if ci_cat:
390 lkgr_builders[name] = True
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100391 return webrtc_builder(
392 name = name,
393 properties = merge_dicts({"mastername": "client.webrtc"}, properties),
394 dimensions = merge_dicts({"pool": "luci.webrtc.ci"}, dimensions),
395 bucket = "ci",
396 service_account = "webrtc-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
Oleh Prypinf5432c52019-03-19 15:10:37 +0100397 triggered_by = ["webrtc-gitiles-trigger-master"] if enabled else None,
Oleh Prypin71d17422019-03-28 08:43:16 +0100398 repo = WEBRTC_GIT,
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200399 notifies = ["post_submit_failure_notifier", "webrtc_tree_closer", "infra_failure_notifier"] if enabled and (ci_cat or perf_cat) else None,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100400 **kwargs
401 )
402
403def try_builder(
404 name,
405 try_cat = True,
406 fyi_cat = None,
407 properties = {},
408 dimensions = {},
409 cq = {},
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100410 branch_cq = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100411 **kwargs):
Oleh Prypine5677672019-04-25 12:21:35 +0200412 """Add a pre-submit builder.
413
414 Args:
415 name: builder name (str).
416 try_cat: boolean, whether to include this builder in the /try/ console. See also: `add_milo`.
417 fyi_cat: the category + name for the /fyi/ console, or None to omit from the console.
418 properties: dict of properties to pass to the recipe (on top of the default ones).
419 dimensions: dict of Swarming dimensions (strings) to search machines by.
420 cq: None to exclude this from all commit queues, or a dict of kwargs for cq_tryjob_verifier.
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100421 branch_cq: False to exclude this builder just from the release-branch CQ.
Oleh Prypine5677672019-04-25 12:21:35 +0200422 **kwargs: Pass on to webrtc_builder / luci.builder.
423 Returns:
424 A luci.builder.
425 """
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100426 add_milo(name, {"try": try_cat, "fyi": fyi_cat})
427 if cq != None:
428 luci.cq_tryjob_verifier(
429 builder = name,
430 cq_group = "cq",
431 **cq
432 )
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100433 if branch_cq:
434 luci.cq_tryjob_verifier(
435 builder = name,
436 cq_group = "cq_branch",
437 **cq
438 )
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100439
440 return webrtc_builder(
441 name = name,
442 properties = merge_dicts({"mastername": "tryserver.webrtc"}, properties),
443 dimensions = merge_dicts({"pool": "luci.webrtc.try"}, dimensions),
444 bucket = "try",
445 service_account = "webrtc-try-builder@chops-service-accounts.iam.gserviceaccount.com",
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200446 notifies = ["infra_failure_notifier"],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100447 **kwargs
448 )
449
450def perf_builder(
451 name,
452 perf_cat,
453 recipe = "standalone",
454 properties = {},
455 dimensions = {},
456 **kwargs):
457 add_milo(name, {"perf": perf_cat})
458 return webrtc_builder(
459 name = name,
460 recipe = recipe,
461 properties = merge_dicts({"mastername": "client.webrtc.perf"}, properties),
462 dimensions = merge_dicts({"pool": "luci.webrtc.perf", "os": "Linux", "cpu": None}, dimensions),
463 bucket = "perf",
464 service_account = "webrtc-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
465 # log_base of 1.7 means:
466 # when there are P pending builds, LUCI will batch the first B builds.
467 # P: 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 ...
468 # B: 1 1 2 2 3 3 3 3 4 4 4 4 4 4 5 ...
Jeremy Leconteea2016b2020-07-01 09:47:22 +0200469 triggering_policy = scheduler.logarithmic_batching(log_base = 1.7),
Artem Titarenkof6588b72019-11-14 14:28:11 +0100470 execution_timeout = 3 * time.hour,
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200471 notifies = ["post_submit_failure_notifier", "infra_failure_notifier"],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100472 **kwargs
473 )
474
Mirko Bonadei12643772020-03-27 13:16:35 +0100475def android_perf_builder(
476 name,
477 perf_cat,
478 recipe = "standalone",
479 properties = {},
480 dimensions = {},
481 **kwargs):
482 return perf_builder(
483 name = name,
484 perf_cat = perf_cat,
485 recipe = recipe,
486 properties = merge_dicts(GOMA_BACKEND_WEBRTC_RBE_PROD, properties),
487 dimensions = dimensions,
488 **kwargs
489 )
490
491def linux_perf_builder(
492 name,
493 perf_cat,
494 recipe = "standalone",
495 properties = {},
496 dimensions = {},
497 **kwargs):
498 return perf_builder(
499 name = name,
500 perf_cat = perf_cat,
501 recipe = recipe,
502 properties = merge_dicts(GOMA_BACKEND_WEBRTC_RBE_PROD, properties),
503 dimensions = dimensions,
504 **kwargs
505 )
506
507def mac_perf_builder(
508 name,
509 perf_cat,
510 recipe = "standalone",
511 properties = {},
512 dimensions = {},
513 **kwargs):
514 return perf_builder(
515 name = name,
516 perf_cat = perf_cat,
517 recipe = recipe,
518 properties = merge_dicts(GOMA_BACKEND_WEBRTC_RBE_PROD, properties),
519 dimensions = dimensions,
520 **kwargs
521 )
522
523def win_perf_builder(
524 name,
525 perf_cat,
526 recipe = "standalone",
527 properties = {},
528 dimensions = {},
529 **kwargs):
530 return perf_builder(
531 name = name,
532 perf_cat = perf_cat,
533 recipe = recipe,
534 properties = merge_dicts(GOMA_BACKEND_RBE_ATS_PROD, properties),
535 dimensions = dimensions,
536 **kwargs
537 )
538
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100539def cron_builder(name, dimensions = {}, **kwargs):
540 add_milo(name, {"cron": True})
541 return webrtc_builder(
542 name = name,
543 dimensions = merge_dicts({"pool": "luci.webrtc.cron", "os": "Linux"}, dimensions),
544 bucket = "cron",
545 service_account = "chromium-webrtc-autoroll@webrtc-ci.iam.gserviceaccount.com",
Oleh Prypin71d17422019-03-28 08:43:16 +0100546 notifies = ["cron_notifier"],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100547 **kwargs
548 )
549
550# Helpers:
551
552def merge_dicts(a, b):
553 """Return the result of merging two dicts.
554
555 If matching values are both dicts or both lists, they will be merged (non-recursively).
556
557 Args:
558 a: first dict.
559 b: second dict (takes priority).
560 Returns:
561 Merged dict.
562 """
563 a = dict(a)
564 for k, bv in b.items():
565 av = a.get(k)
566 if type(av) == "dict" and type(bv) == "dict":
567 a[k] = dict(av)
568 a[k].update(bv)
569 elif type(av) == "list" and type(bv) == "list":
570 a[k] = av + bv
571 else:
572 a[k] = bv
573 return a
574
575def normal_builder_factory(**common_kwargs):
576 def builder(*args, **kwargs):
577 return ci_builder(*args, **merge_dicts(common_kwargs, kwargs))
578
579 def try_job(*args, **kwargs):
580 return try_builder(*args, **merge_dicts(common_kwargs, kwargs))
581
582 return builder, try_job
583
584# Mixins:
585
586linux_builder, linux_try_job = normal_builder_factory(
587 dimensions = {"os": "Linux", "inside_docker": "0"},
Mirko Bonadei32ca4862020-03-11 10:39:45 +0100588 properties = GOMA_BACKEND_WEBRTC_RBE_PROD,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100589)
590
591android_builder, android_try_job = normal_builder_factory(
592 dimensions = {"os": "Linux"},
Mirko Bonadei32ca4862020-03-11 10:39:45 +0100593 properties = GOMA_BACKEND_WEBRTC_RBE_PROD,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100594)
595
596win_builder, win_try_job = normal_builder_factory(
597 dimensions = {"os": "Windows"},
Patrik Höglund3916a972019-12-04 13:19:38 +0100598 properties = GOMA_BACKEND_RBE_ATS_PROD,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100599)
600
601mac_builder, mac_try_job = normal_builder_factory(
602 dimensions = {"os": "Mac"},
Mirko Bonadei32ca4862020-03-11 10:39:45 +0100603 properties = GOMA_BACKEND_WEBRTC_RBE_PROD,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100604)
605
606ios_builder, ios_try_job = normal_builder_factory(
Patrik Höglundd5dcd0e2020-02-19 21:50:39 +0100607 dimensions = {"os": "Mac-10.14"},
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100608 recipe = "ios",
Mirko Bonadei84360ca2020-03-12 08:35:48 +0100609 properties = GOMA_BACKEND_WEBRTC_IOS_RBE_PROD,
610 caches = [swarming.cache("osx_sdk")],
611)
612
613# Some iOS builders don't use goma because they build using
614# `use_xcode_clang=true` which is not supported by goma.
615# https://ci.chromium.org/p/webrtc/builders/ci/iOS%20API%20Framework%20Builder
616# https://ci.chromium.org/p/webrtc/builders/try/ios_api_framework
617ios_builder_no_goma, ios_try_job_no_goma = normal_builder_factory(
618 dimensions = {"os": "Mac-10.14"},
619 recipe = "ios",
Patrik Höglundb681c162020-02-21 07:34:53 +0100620 properties = {"$depot_tools/osx_sdk": {"sdk_version": "11c29"}},
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100621 caches = [swarming.cache("osx_sdk")],
622)
623
624# Actual builder configuration:
625
626android_builder("Android32 (M Nexus5X)(dbg)", "Android|arm|dbg")
627android_try_job("android_compile_arm_dbg", cq = None)
628android_try_job("android_arm_dbg")
629android_builder("Android32 (M Nexus5X)", "Android|arm|rel")
630android_try_job("android_arm_rel")
631android_builder("Android32 Builder arm", "Android|arm|size", perf_cat = "Android|arm|Builder|", prioritized = True)
632android_try_job("android_compile_arm_rel")
Mirko Bonadei86b4f2d2020-07-08 14:24:48 +0200633android_perf_builder("Perf Android32 (M Nexus5)", "Android|arm|Tester|M Nexus5", triggered_by = ["Android32 Builder arm"])
Mirko Bonadei12643772020-03-27 13:16:35 +0100634android_perf_builder("Perf Android32 (M AOSP Nexus6)", "Android|arm|Tester|M AOSP Nexus6", triggered_by = ["Android32 Builder arm"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100635android_try_job("android_compile_arm64_dbg", cq = None)
636android_try_job("android_arm64_dbg", cq = None)
637android_builder("Android64 (M Nexus5X)", "Android|arm64|rel")
638android_try_job("android_arm64_rel")
639android_builder("Android64 Builder arm64", "Android|arm64|size", perf_cat = "Android|arm64|Builder|", prioritized = True)
Mirko Bonadei12643772020-03-27 13:16:35 +0100640android_perf_builder("Perf Android64 (M Nexus5X)", "Android|arm64|Tester|M Nexus5X", triggered_by = ["Android64 Builder arm64"])
641android_perf_builder("Perf Android64 (O Pixel2)", "Android|arm64|Tester|O Pixel2", triggered_by = ["Android64 Builder arm64"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100642android_try_job("android_compile_arm64_rel")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100643android_builder("Android64 Builder x64 (dbg)", "Android|x64|dbg")
644android_try_job("android_compile_x64_dbg")
645android_try_job("android_compile_x64_rel", cq = None)
646android_builder("Android32 Builder x86 (dbg)", "Android|x86|dbg")
647android_try_job("android_compile_x86_dbg")
648android_builder("Android32 Builder x86", "Android|x86|rel")
649android_try_job("android_compile_x86_rel")
650android_builder("Android32 (more configs)", "Android|arm|more", recipe = "more_configs")
651android_try_job("android_arm_more_configs", recipe = "more_configs")
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100652android_try_job("android_chromium_compile", recipe = "chromium_trybot", branch_cq = False)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100653
654ios_builder("iOS32 Debug", "iOS|arm|dbg")
655ios_try_job("ios_compile_arm_dbg")
656ios_builder("iOS32 Release", "iOS|arm|rel")
657ios_try_job("ios_compile_arm_rel")
658ios_builder("iOS64 Debug", "iOS|arm64|dbg")
659ios_try_job("ios_compile_arm64_dbg")
660ios_builder("iOS64 Release", "iOS|arm64|rel")
661ios_try_job("ios_compile_arm64_rel")
662ios_builder("iOS64 Sim Debug (iOS 10.0)", "iOS|x64|10")
Patrik Höglund17bed672020-02-20 13:31:36 +0000663ios_try_job("ios_sim_x64_dbg_ios10")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100664ios_builder("iOS64 Sim Debug (iOS 11)", "iOS|x64|11")
Patrik Höglund17bed672020-02-20 13:31:36 +0000665ios_try_job("ios_sim_x64_dbg_ios11")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100666ios_builder("iOS64 Sim Debug (iOS 12)", "iOS|x64|12")
Patrik Höglund17bed672020-02-20 13:31:36 +0000667ios_try_job("ios_sim_x64_dbg_ios12")
Mirko Bonadei84360ca2020-03-12 08:35:48 +0100668ios_builder_no_goma("iOS API Framework Builder", "iOS|fat|size", recipe = "ios_api_framework", prioritized = True)
Jeremy Leconteea2016b2020-07-01 09:47:22 +0200669ios_try_job_no_goma("ios_api_framework", recipe = "ios_api_framework", cq = None)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100670
671linux_builder("Linux32 Debug", "Linux|x86|dbg")
672linux_try_job("linux_x86_dbg")
673linux_builder("Linux32 Release", "Linux|x86|rel")
674linux_try_job("linux_x86_rel")
675linux_builder("Linux64 Debug", "Linux|x64|dbg")
676linux_try_job("linux_dbg", cq = None)
677linux_try_job("linux_compile_dbg")
678linux_builder("Linux64 Release", "Linux|x64|rel")
679linux_try_job("linux_rel")
680linux_builder("Linux64 Builder", "Linux|x64|size", perf_cat = "Linux|x64|Builder|", prioritized = True)
681linux_try_job("linux_compile_rel")
Mirko Bonadei12643772020-03-27 13:16:35 +0100682linux_perf_builder("Perf Linux Trusty", "Linux|x64|Tester|Trusty", triggered_by = ["Linux64 Builder"])
683linux_perf_builder("Perf Linux Xenial", None, triggered_by = ["Linux64 Builder"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100684linux_builder("Linux32 Debug (ARM)", "Linux|arm|dbg")
685linux_try_job("linux_compile_arm_dbg")
686linux_builder("Linux32 Release (ARM)", "Linux|arm|rel")
687linux_try_job("linux_compile_arm_rel")
688linux_builder("Linux64 Debug (ARM)", "Linux|arm64|dbg")
689linux_try_job("linux_compile_arm64_dbg")
690linux_builder("Linux64 Release (ARM)", "Linux|arm64|rel")
691linux_try_job("linux_compile_arm64_rel")
692linux_builder("Linux64 Release (GCC)", "Linux|x64|gcc")
693linux_try_job("linux_compile_gcc_rel")
694linux_builder("Linux Asan", "Linux|x64|asan")
695linux_try_job("linux_asan")
696linux_builder("Linux MSan", "Linux|x64|msan")
697linux_try_job("linux_msan")
698linux_builder("Linux Tsan v2", "Linux|x64|tsan")
699linux_try_job("linux_tsan2")
700linux_builder("Linux UBSan", "Linux|x64|ubsan")
701linux_try_job("linux_ubsan")
702linux_builder("Linux UBSan vptr", "Linux|x64|ubsan")
703linux_try_job("linux_ubsan_vptr")
704linux_builder("Linux64 Release (Libfuzzer)", "Linux|x64|fuzz", recipe = "libfuzzer")
705linux_try_job("linux_libfuzzer_rel", recipe = "libfuzzer")
706linux_builder("Linux (more configs)", "Linux|x64|more", recipe = "more_configs")
707linux_try_job("linux_more_configs", recipe = "more_configs")
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100708linux_try_job("linux_chromium_compile", recipe = "chromium_trybot", branch_cq = False)
Mirko Bonadei40de50f2019-11-07 11:23:47 +0100709linux_try_job("linux_chromium_compile_dbg", recipe = "chromium_trybot", branch_cq = False)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100710
711mac_builder("Mac64 Debug", "Mac|x64|dbg")
712mac_try_job("mac_dbg", cq = None)
713mac_try_job("mac_compile_dbg")
714mac_builder("Mac64 Release", "Mac|x64|rel")
715mac_try_job("mac_rel")
716mac_try_job("mac_compile_rel", cq = None)
717mac_builder("Mac64 Builder", ci_cat = None, perf_cat = "Mac|x64|Builder|")
Mirko Bonadei12643772020-03-27 13:16:35 +0100718mac_perf_builder("Perf Mac 10.11", "Mac|x64|Tester|10.11", triggered_by = ["Mac64 Builder"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100719mac_builder("Mac Asan", "Mac|x64|asan")
720mac_try_job("mac_asan")
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100721mac_try_job("mac_chromium_compile", recipe = "chromium_trybot", dimensions = {"cores": "8"}, branch_cq = False)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100722
723win_builder("Win32 Debug", "Win MSVC|x86|dbg")
724win_try_job("win_x86_msvc_dbg")
725win_try_job("win_compile_x86_msvc_dbg", cq = None)
Oleh Prypin63041b82019-03-19 22:56:01 +0000726win_builder("Win32 Release", "Win MSVC|x86|rel")
727win_try_job("win_x86_msvc_rel")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100728win_try_job("win_compile_x86_msvc_rel", cq = None)
729win_builder("Win64 Debug", "Win MSVC|x64|dbg")
730win_try_job("win_x64_msvc_dbg")
731win_try_job("win_compile_x64_msvc_dbg", cq = None)
732win_builder("Win64 Release", "Win MSVC|x64|rel")
733win_try_job("win_x64_msvc_rel")
734win_try_job("win_compile_x64_msvc_rel", cq = None)
735win_builder("Win32 Debug (Clang)", "Win Clang|x86|dbg")
736win_try_job("win_x86_clang_dbg", cq = None)
737win_try_job("win_compile_x86_clang_dbg")
738win_builder("Win32 Release (Clang)", "Win Clang|x86|rel")
739win_try_job("win_x86_clang_rel")
740win_try_job("win_compile_x86_clang_rel", cq = None)
741win_builder("Win32 Builder (Clang)", ci_cat = None, perf_cat = "Win|x86|Builder|")
Mirko Bonadei12643772020-03-27 13:16:35 +0100742win_perf_builder("Perf Win7", "Win|x86|Tester|7", triggered_by = ["Win32 Builder (Clang)"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100743win_builder("Win64 Debug (Clang)", "Win Clang|x64|dbg")
744win_try_job("win_x64_clang_dbg", cq = None)
745win_try_job("win_x64_clang_dbg_win8", cq = None)
746win_try_job("win_x64_clang_dbg_win10", cq = None)
747win_try_job("win_compile_x64_clang_dbg")
748win_builder("Win64 Release (Clang)", "Win Clang|x64|rel")
749win_try_job("win_x64_clang_rel", cq = None)
750win_try_job("win_compile_x64_clang_rel")
Mirko Bonadeiafbe33d2019-05-31 09:27:54 +0200751win_builder("Win64 ASan", "Win Clang|x64|asan")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100752win_try_job("win_asan")
753win_builder("Win64 UWP", ci_cat = None, fyi_cat = "")
754win_try_job("win_x64_uwp", cq = None, try_cat = None, fyi_cat = "")
755win_builder("Win (more configs)", "Win Clang|x86|more", recipe = "more_configs")
756win_try_job("win_x86_more_configs", recipe = "more_configs")
Mirko Bonadei825f9c22020-07-01 11:52:39 +0200757win_try_job("win_chromium_compile", recipe = "chromium_trybot", branch_cq = False, goma_jobs = 150)
758win_try_job("win_chromium_compile_dbg", recipe = "chromium_trybot", branch_cq = False, goma_jobs = 150)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100759
760linux_try_job(
761 "presubmit",
762 recipe = "run_presubmit",
763 properties = {"repo_name": "webrtc", "runhooks": True},
764 priority = 28,
765 cq = {"disable_reuse": True},
766)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100767
768cron_builder(
769 "Auto-roll - WebRTC DEPS",
770 recipe = "auto_roll_webrtc_deps",
Oleh Prypindc68a722019-06-25 10:43:13 +0200771 schedule = "0 */2 * * *", # Every 2 hours.
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100772)
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200773
774lkgr_config = {
775 "project": "webrtc",
776 "source_url": WEBRTC_GIT,
777 "status_url": "https://webrtc-status.appspot.com",
778 "allowed_lag": 300, # hours
779 "allowed_gap": 12, # commits behind
Mirko Bonadei450e9b72020-07-10 19:19:22 +0000780 "error_recipients": "webrtc-troopers-robots@google.com",
Michael Achenbach09dd7dc2019-07-26 15:26:11 +0200781 "buckets": {
782 "webrtc/ci": {
783 # bucket alias: luci.webrtc.ci
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200784 "builders": sorted(lkgr_builders),
785 },
Michael Achenbach09dd7dc2019-07-26 15:26:11 +0200786 "chromium/webrtc.fyi": {
787 # bucket alias: luci.chromium.webrtc.fyi
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200788 "builders": [
789 "WebRTC Chromium FYI Android Builder (dbg)",
790 "WebRTC Chromium FYI Android Builder ARM64 (dbg)",
791 "WebRTC Chromium FYI Android Builder",
Mirko Bonadei04fe32f2020-05-01 00:45:58 +0200792 "WebRTC Chromium FYI Android Tests (dbg) (L Nexus5)",
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200793 "WebRTC Chromium FYI Android Tests (dbg) (M Nexus5X)",
794 "WebRTC Chromium FYI Linux Builder (dbg)",
795 "WebRTC Chromium FYI Linux Builder",
796 "WebRTC Chromium FYI Linux Tester",
797 "WebRTC Chromium FYI Mac Builder (dbg)",
798 "WebRTC Chromium FYI Mac Builder",
799 "WebRTC Chromium FYI Mac Tester",
800 "WebRTC Chromium FYI Win Builder (dbg)",
801 "WebRTC Chromium FYI Win Builder",
802 "WebRTC Chromium FYI Win10 Tester",
803 "WebRTC Chromium FYI Win7 Tester",
804 "WebRTC Chromium FYI Win8 Tester",
805 "WebRTC Chromium FYI ios-device",
Mirko Bonadei37071f62020-02-15 10:42:49 +0100806 "WebRTC Chromium FYI ios-simulator",
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200807 ],
808 },
809 },
810}
811
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100812cron_builder(
813 "WebRTC lkgr finder",
814 recipe = "lkgr_finder",
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200815 properties = {
816 "project": "webrtc",
817 "repo": WEBRTC_GIT,
818 "ref": "refs/heads/lkgr",
819 "lkgr_status_gs_path": "chromium-webrtc/lkgr-status",
820 "config": lkgr_config,
821 },
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100822 schedule = "*/10 * * * *", # Every 10 minutes.
823)