blob: bed0a4961c4ecd70b553176a315a56329ecd5e74 [file] [log] [blame]
Oleh Prypin1a0593f2019-03-11 09:43:28 +01001#!/usr/bin/env lucicfg
2
3# Copyright (c) 2019 The WebRTC project authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7# https://chromium.googlesource.com/infra/luci/luci-go/+/master/lucicfg/doc/
8
Jeremy Leconteea2016b2020-07-01 09:47:22 +02009"""LUCI project configuration for WebRTC CQ and CI."""
10
11lucicfg.check_version("1.15.0")
Mirko Bonadei9189d9c2020-06-09 21:54:18 +020012
Oleh Prypin1a0593f2019-03-11 09:43:28 +010013WEBRTC_GIT = "https://webrtc.googlesource.com/src"
14WEBRTC_GERRIT = "https://webrtc-review.googlesource.com/src"
15
Mirko Bonadei32ca4862020-03-11 10:39:45 +010016GOMA_BACKEND_WEBRTC_RBE_PROD = {
Mirko Bonadei1e905342020-03-04 09:39:42 +010017 "$build/goma": {
18 "server_host": "goma.chromium.org",
Jeremy Leconteea2016b2020-07-01 09:47:22 +020019 "use_luci_auth": True,
20 },
Mirko Bonadei1e905342020-03-04 09:39:42 +010021}
22
Mirko Bonadei84360ca2020-03-12 08:35:48 +010023GOMA_BACKEND_WEBRTC_IOS_RBE_PROD = {
24 "$build/goma": {
25 "server_host": "goma.chromium.org",
Jeremy Leconteea2016b2020-07-01 09:47:22 +020026 "use_luci_auth": True,
Mirko Bonadei84360ca2020-03-12 08:35:48 +010027 },
Artem Titov1d47d312020-11-03 14:45:52 +010028 "$depot_tools/osx_sdk": {"sdk_version": "12a7209"},
Mirko Bonadei84360ca2020-03-12 08:35:48 +010029}
30
Patrik Höglund3916a972019-12-04 13:19:38 +010031GOMA_BACKEND_RBE_ATS_PROD = {
32 "$build/goma": {
33 "server_host": "goma.chromium.org",
Mirko Bonadei2688eca2020-03-25 09:15:54 +010034 "use_luci_auth": True,
Jeremy Leconteea2016b2020-07-01 09:47:22 +020035 "enable_ats": True,
36 },
Patrik Höglund3916a972019-12-04 13:19:38 +010037}
38
Oleh Prypin1a0593f2019-03-11 09:43:28 +010039# Top-level configs:
40
Owen Rodley6b721ba2020-05-26 10:04:20 +100041# Enable luci.tree_closer.
42lucicfg.enable_experiment("crbug.com/1054172")
43
Oleh Prypin1a0593f2019-03-11 09:43:28 +010044lucicfg.config(
45 config_dir = ".",
46 tracked_files = [
47 "commit-queue.cfg",
48 "cr-buildbucket.cfg",
49 "luci-logdog.cfg",
50 "luci-milo.cfg",
Oleh Prypin71d17422019-03-28 08:43:16 +010051 "luci-notify.cfg",
Oleh Prypin705b6a62019-04-03 23:10:51 +020052 "luci-notify/**/*",
Oleh Prypin1a0593f2019-03-11 09:43:28 +010053 "luci-scheduler.cfg",
54 "project.cfg",
55 ],
Jeremy Leconteea2016b2020-07-01 09:47:22 +020056 lint_checks = ["default"],
Oleh Prypin1a0593f2019-03-11 09:43:28 +010057)
58
59luci.project(
60 name = "webrtc",
61 buildbucket = "cr-buildbucket.appspot.com",
62 logdog = "luci-logdog.appspot.com",
63 milo = "luci-milo.appspot.com",
Oleh Prypin71d17422019-03-28 08:43:16 +010064 notify = "luci-notify.appspot.com",
Oleh Prypin1a0593f2019-03-11 09:43:28 +010065 scheduler = "luci-scheduler.appspot.com",
66 swarming = "chromium-swarm.appspot.com",
67 acls = [
68 acl.entry(
69 [acl.BUILDBUCKET_READER, acl.LOGDOG_READER, acl.PROJECT_CONFIGS_READER, acl.SCHEDULER_READER],
70 groups = ["all"],
71 ),
72 acl.entry(acl.LOGDOG_WRITER, groups = ["luci-logdog-chromium-writers"]),
73 acl.entry(acl.SCHEDULER_OWNER, groups = ["project-webrtc-admins"]),
74 ],
75)
76
77luci.logdog(
78 gs_bucket = "chromium-luci-logdog",
79)
80
81luci.milo(
82 logo = "https://storage.googleapis.com/chrome-infra/webrtc-logo-vert-retro-255x305.png",
83)
84
Owen Rodleya55c38e2020-06-15 11:16:10 +100085luci.notify(tree_closing_enabled = True)
86
Oleh Prypin1a0593f2019-03-11 09:43:28 +010087luci.cq(
88 status_host = "chromium-cq-status.appspot.com",
89 submit_max_burst = 1,
90 submit_burst_delay = 1 * time.minute,
91)
92
93luci.gitiles_poller(
94 name = "webrtc-gitiles-trigger-master",
95 bucket = "ci",
96 repo = WEBRTC_GIT,
97)
98
99# Bucket definitions:
100
101luci.bucket(
102 name = "try",
103 acls = [
104 acl.entry(acl.BUILDBUCKET_TRIGGERER, groups = [
105 "service-account-cq",
106 "project-webrtc-tryjob-access",
107 ]),
108 ],
109)
110
111luci.bucket(
112 name = "ci",
113 acls = [
Oleh Prypinf35939d2019-05-03 20:42:38 +0200114 acl.entry(acl.BUILDBUCKET_TRIGGERER, groups = [
115 "project-webrtc-ci-schedulers",
116 ], users = [
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100117 "luci-scheduler@appspot.gserviceaccount.com",
118 ]),
Jeremy Leconte2e25bb52020-07-16 09:38:56 +0200119 acl.entry(acl.BUILDBUCKET_TRIGGERER, groups = [
120 # Allow Pinpoint to trigger builds for bisection
121 "service-account-chromeperf",
122 ]),
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100123 ],
124)
125
126luci.bucket(
127 name = "perf",
128 acls = [
129 acl.entry(acl.BUILDBUCKET_TRIGGERER, users = [
130 "luci-scheduler@appspot.gserviceaccount.com",
131 "webrtc-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
132 ]),
133 ],
134)
135
136luci.bucket(
137 name = "cron",
138 acls = [
139 acl.entry(acl.BUILDBUCKET_TRIGGERER, users = [
140 "luci-scheduler@appspot.gserviceaccount.com",
141 ]),
142 ],
143)
144
145# Commit queue definitions:
146
147luci.cq_group(
148 name = "cq",
149 tree_status_host = "webrtc-status.appspot.com",
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100150 watch = [cq.refset(repo = WEBRTC_GERRIT, refs = ["refs/heads/master"])],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100151 acls = [
152 acl.entry(acl.CQ_COMMITTER, groups = ["project-webrtc-committers"]),
153 acl.entry(acl.CQ_DRY_RUNNER, groups = ["project-webrtc-tryjob-access"]),
154 ],
155 retry_config = cq.RETRY_ALL_FAILURES,
Oleh Prypin5416f2d2019-08-02 09:43:15 +0200156 cancel_stale_tryjobs = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100157)
158
159luci.cq_group(
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100160 name = "cq_branch",
161 watch = [cq.refset(repo = WEBRTC_GERRIT, refs = ["refs/branch-heads/.+"])],
162 acls = [
163 acl.entry(acl.CQ_COMMITTER, groups = ["project-webrtc-committers"]),
164 acl.entry(acl.CQ_DRY_RUNNER, groups = ["project-webrtc-tryjob-access"]),
165 ],
166 retry_config = cq.RETRY_ALL_FAILURES,
Oleh Prypin5416f2d2019-08-02 09:43:15 +0200167 cancel_stale_tryjobs = True,
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100168)
169
170luci.cq_group(
171 name = "cq_infra",
172 watch = [cq.refset(repo = WEBRTC_GERRIT, refs = ["refs/heads/infra/config"])],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100173 acls = [
174 acl.entry(acl.CQ_COMMITTER, groups = ["project-webrtc-admins"]),
175 acl.entry(acl.CQ_DRY_RUNNER, groups = ["project-webrtc-tryjob-access"]),
176 ],
177 retry_config = cq.RETRY_ALL_FAILURES,
Oleh Prypin5416f2d2019-08-02 09:43:15 +0200178 cancel_stale_tryjobs = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100179)
180
181luci.cq_tryjob_verifier(
182 builder = "presubmit",
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100183 cq_group = "cq_infra",
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100184)
185
Oleh Prypine56cd9b2019-05-29 21:14:31 +0200186luci.cq_tryjob_verifier(
187 builder = "webrtc-internal:g3.webrtc-internal.try/internal_compile_lite",
188 owner_whitelist = ["project-webrtc-internal-tryjob-access"],
189 cq_group = "cq",
190)
191
Oleh Prypin71d17422019-03-28 08:43:16 +0100192# Notifier definitions:
193
194luci.notifier(
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200195 name = "post_submit_failure_notifier",
196 on_new_status = ["FAILURE"],
Oleh Prypin71d17422019-03-28 08:43:16 +0100197 notify_emails = ["webrtc-sheriffs-robots@google.com"],
Oleh Prypin705b6a62019-04-03 23:10:51 +0200198 template = luci.notifier_template(
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200199 name = "build_failure",
200 body = io.read_file("luci-notify/email-templates/build_failure.template"),
Oleh Prypin705b6a62019-04-03 23:10:51 +0200201 ),
Oleh Prypin71d17422019-03-28 08:43:16 +0100202)
203
204luci.notifier(
205 name = "cron_notifier",
Jeremy Leconte67a904e2020-07-08 11:48:18 +0200206 on_new_status = ["FAILURE", "INFRA_FAILURE"],
Mirko Bonadei450e9b72020-07-10 19:19:22 +0000207 notify_emails = ["webrtc-troopers-robots@google.com"],
Oleh Prypin705b6a62019-04-03 23:10:51 +0200208 template = luci.notifier_template(
209 name = "cron",
210 body = io.read_file("luci-notify/email-templates/cron.template"),
211 ),
Oleh Prypin71d17422019-03-28 08:43:16 +0100212)
213
Jeremy Leconte67a904e2020-07-08 11:48:18 +0200214luci.notifier(
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200215 name = "infra_failure_notifier",
Jeremy Leconte67a904e2020-07-08 11:48:18 +0200216 on_new_status = ["INFRA_FAILURE"],
Mirko Bonadei450e9b72020-07-10 19:19:22 +0000217 notify_emails = ["webrtc-troopers-robots@google.com"],
Jeremy Leconte67a904e2020-07-08 11:48:18 +0200218 template = luci.notifier_template(
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200219 name = "infra_failure",
220 body = io.read_file("luci-notify/email-templates/infra_failure.template"),
Jeremy Leconte67a904e2020-07-08 11:48:18 +0200221 ),
222)
223
Owen Rodley6b721ba2020-05-26 10:04:20 +1000224# Tree closer definitions:
225
226luci.tree_closer(
227 name = "webrtc_tree_closer",
228 tree_status_host = "webrtc-status.appspot.com",
Owen Rodley6b721ba2020-05-26 10:04:20 +1000229 # TODO: These step filters are copied verbatim from Gatekeeper, for testing
230 # that LUCI-Notify would take the exact same actions. Once we've switched
231 # over, this should be updated - several of these steps don't exist in
232 # WebRTC recipes.
233 failed_step_regexp = [
234 "bot_update",
235 "compile",
236 "gclient runhooks",
237 "runhooks",
238 "update",
239 "extract build",
240 "cleanup_temp",
241 "taskkill",
242 "compile",
243 "gn",
244 ],
245 failed_step_regexp_exclude = ".*\\(experimental\\).*",
246)
247
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100248# Recipe definitions:
249
250def recipe(recipe, pkg = "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"):
251 return luci.recipe(
252 name = recipe.split("/")[-1],
253 cipd_package = pkg,
254 cipd_version = "refs/heads/master",
255 recipe = recipe,
256 )
257
258recipe("chromium_trybot")
259recipe("run_presubmit")
260recipe("webrtc/auto_roll_webrtc_deps")
261recipe("webrtc/ios")
262recipe("webrtc/ios_api_framework")
263recipe("webrtc/libfuzzer")
264recipe("webrtc/more_configs")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100265recipe("webrtc/standalone")
266recipe("lkgr_finder", pkg = "infra/recipe_bundles/chromium.googlesource.com/infra/infra")
267
268# Console definitions:
269
270luci.console_view(name = "ci", title = "Main", repo = WEBRTC_GIT, header = "console-header.textpb")
271luci.console_view(name = "perf", title = "Perf", repo = WEBRTC_GIT, header = "console-header.textpb")
272luci.list_view(name = "cron", title = "Cron")
273luci.console_view(name = "fyi", title = "FYI", repo = WEBRTC_GIT, header = "console-header.textpb")
274luci.list_view(name = "try", title = "Tryserver")
275
276def add_milo(builder, views):
277 """Add Milo console entries for the builder.
278
279 Args:
280 builder: builder name (str).
281 views: dict where keys are names of consoles and values are either a
282 category for the console (str, pipe-separated) or True, which means
283 adding to a list view rather than a console.
284 """
285 for view_name, category in views.items():
286 if category == None:
287 continue
288 elif type(category) == "string":
289 category, _, short_name = category.rpartition("|")
290 luci.console_view_entry(
291 console_view = view_name,
292 builder = builder,
293 category = category or None,
294 short_name = short_name or None,
295 )
296 elif category == True:
297 luci.list_view_entry(
298 list_view = view_name,
299 builder = builder,
300 )
301 else:
302 fail("Unexpected value for category: %r" % category)
303
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200304lkgr_builders = {}
305
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100306# Builder-defining functions:
307
Artem Titarenkof6588b72019-11-14 14:28:11 +0100308def webrtc_builder(
309 name,
310 recipe = "standalone",
311 dimensions = {},
Mirko Bonadei825f9c22020-07-01 11:52:39 +0200312 properties = {},
Artem Titarenkof6588b72019-11-14 14:28:11 +0100313 priority = 30,
314 execution_timeout = 2 * time.hour,
Mirko Bonadeib7ef4602020-07-01 13:28:26 +0200315 goma_jobs = None,
Artem Titarenkof6588b72019-11-14 14:28:11 +0100316 **kwargs):
Mirko Bonadeib7ef4602020-07-01 13:28:26 +0200317 """WebRTC specific wrapper around luci.builder.
318
319 Args:
320 name: builder name (str).
321 recipe: string with the name of the recipe to run.
322 dimensions: dict of Swarming dimensions (strings) to search machines by.
323 properties: dict of properties to pass to the recipe (on top of the default ones).
324 priority: int [1-255] or None, indicating swarming task priority, lower is
325 more important. If None, defer the decision to Buildbucket service.
326 execution_timeout: int or None, how long to wait for a running build to finish before
327 forcefully aborting it and marking the build as timed out. If None,
328 defer the decision to Buildbucket service.
329 goma_jobs: int or None, number of jobs to be used by the builder. If None, defer the
330 decition to the goma module.
331 **kwargs: Pass on to webrtc_builder / luci.builder.
332 Returns:
333 A luci.builder.
334 """
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100335 dimensions = merge_dicts({"cpu": "x86-64"}, dimensions)
336
Mirko Bonadeib7ef4602020-07-01 13:28:26 +0200337 if goma_jobs != None:
338 goma_additional_params = {"$build/goma": {"jobs": goma_jobs}}
339 properties = merge_dicts(properties, goma_additional_params)
Ye Kuang8df20492020-10-15 14:54:17 +0900340 properties = merge_dicts(properties, {
341 "$recipe_engine/isolated": {
342 "server": "https://isolateserver.appspot.com",
343 },
344 })
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100345 return luci.builder(
346 name = name,
Oleh Prypin705b6a62019-04-03 23:10:51 +0200347 executable = recipe,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100348 dimensions = {k: v for k, v in dimensions.items() if v != None},
Mirko Bonadeib7ef4602020-07-01 13:28:26 +0200349 properties = properties,
Artem Titarenkof6588b72019-11-14 14:28:11 +0100350 execution_timeout = execution_timeout,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100351 priority = priority,
352 build_numbers = True,
353 swarming_tags = ["vpython:native-python-wrapper"],
354 **kwargs
355 )
356
357def ci_builder(
358 name,
359 ci_cat,
360 perf_cat = None,
361 fyi_cat = None,
362 properties = {},
363 dimensions = {},
364 prioritized = False,
Oleh Prypinf5432c52019-03-19 15:10:37 +0100365 enabled = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100366 **kwargs):
Oleh Prypine5677672019-04-25 12:21:35 +0200367 """Add a post-submit builder.
368
369 Args:
370 name: builder name (str).
371 ci_cat: the category + name for the /ci/ console, or None to omit from the console.
372 perf_cat: the category + name for the /perf/ console, or None to omit from the console.
373 fyi_cat: the category + name for the /fyi/ console, or None to omit from the console.
374 properties: dict of properties to pass to the recipe (on top of the default ones).
375 dimensions: dict of Swarming dimensions (strings) to search machines by.
376 prioritized: True to make this builder have a higher priority and never batch builds.
377 enabled: False to exclude this builder from consoles and failure notifications.
378 **kwargs: Pass on to webrtc_builder / luci.builder.
379 Returns:
380 A luci.builder.
381
382 Notifications are also disabled if a builder is not on either of /ci/ or /perf/ consoles.
383 """
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100384 if prioritized:
385 kwargs["triggering_policy"] = scheduler.greedy_batching(
386 max_batch_size = 1,
387 max_concurrent_invocations = 3,
388 )
389 kwargs["priority"] = 29
390
Oleh Prypinf5432c52019-03-19 15:10:37 +0100391 if enabled:
392 add_milo(name, {"ci": ci_cat, "perf": perf_cat, "fyi": fyi_cat})
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200393 if ci_cat:
394 lkgr_builders[name] = True
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100395 return webrtc_builder(
396 name = name,
Garrett Beaty4a106ce2020-09-09 14:51:23 -0700397 properties = merge_dicts({"builder_group": "client.webrtc"}, properties),
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100398 dimensions = merge_dicts({"pool": "luci.webrtc.ci"}, dimensions),
399 bucket = "ci",
400 service_account = "webrtc-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
Oleh Prypinf5432c52019-03-19 15:10:37 +0100401 triggered_by = ["webrtc-gitiles-trigger-master"] if enabled else None,
Oleh Prypin71d17422019-03-28 08:43:16 +0100402 repo = WEBRTC_GIT,
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200403 notifies = ["post_submit_failure_notifier", "webrtc_tree_closer", "infra_failure_notifier"] if enabled and (ci_cat or perf_cat) else None,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100404 **kwargs
405 )
406
407def try_builder(
408 name,
409 try_cat = True,
410 fyi_cat = None,
411 properties = {},
412 dimensions = {},
413 cq = {},
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100414 branch_cq = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100415 **kwargs):
Oleh Prypine5677672019-04-25 12:21:35 +0200416 """Add a pre-submit builder.
417
418 Args:
419 name: builder name (str).
420 try_cat: boolean, whether to include this builder in the /try/ console. See also: `add_milo`.
421 fyi_cat: the category + name for the /fyi/ console, or None to omit from the console.
422 properties: dict of properties to pass to the recipe (on top of the default ones).
423 dimensions: dict of Swarming dimensions (strings) to search machines by.
424 cq: None to exclude this from all commit queues, or a dict of kwargs for cq_tryjob_verifier.
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100425 branch_cq: False to exclude this builder just from the release-branch CQ.
Oleh Prypine5677672019-04-25 12:21:35 +0200426 **kwargs: Pass on to webrtc_builder / luci.builder.
427 Returns:
428 A luci.builder.
429 """
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100430 add_milo(name, {"try": try_cat, "fyi": fyi_cat})
431 if cq != None:
432 luci.cq_tryjob_verifier(
433 builder = name,
434 cq_group = "cq",
435 **cq
436 )
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100437 if branch_cq:
438 luci.cq_tryjob_verifier(
439 builder = name,
440 cq_group = "cq_branch",
441 **cq
442 )
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100443
444 return webrtc_builder(
445 name = name,
Garrett Beaty4a106ce2020-09-09 14:51:23 -0700446 properties = merge_dicts({"builder_group": "tryserver.webrtc"}, properties),
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100447 dimensions = merge_dicts({"pool": "luci.webrtc.try"}, dimensions),
448 bucket = "try",
449 service_account = "webrtc-try-builder@chops-service-accounts.iam.gserviceaccount.com",
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200450 notifies = ["infra_failure_notifier"],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100451 **kwargs
452 )
453
454def perf_builder(
455 name,
456 perf_cat,
457 recipe = "standalone",
458 properties = {},
459 dimensions = {},
460 **kwargs):
461 add_milo(name, {"perf": perf_cat})
462 return webrtc_builder(
463 name = name,
464 recipe = recipe,
Garrett Beaty4a106ce2020-09-09 14:51:23 -0700465 properties = merge_dicts({"builder_group": "client.webrtc.perf"}, properties),
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100466 dimensions = merge_dicts({"pool": "luci.webrtc.perf", "os": "Linux", "cpu": None}, dimensions),
467 bucket = "perf",
468 service_account = "webrtc-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
469 # log_base of 1.7 means:
470 # when there are P pending builds, LUCI will batch the first B builds.
471 # P: 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 ...
472 # B: 1 1 2 2 3 3 3 3 4 4 4 4 4 4 5 ...
Jeremy Leconteea2016b2020-07-01 09:47:22 +0200473 triggering_policy = scheduler.logarithmic_batching(log_base = 1.7),
Artem Titarenkof6588b72019-11-14 14:28:11 +0100474 execution_timeout = 3 * time.hour,
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200475 notifies = ["post_submit_failure_notifier", "infra_failure_notifier"],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100476 **kwargs
477 )
478
Mirko Bonadei12643772020-03-27 13:16:35 +0100479def android_perf_builder(
480 name,
481 perf_cat,
482 recipe = "standalone",
483 properties = {},
484 dimensions = {},
485 **kwargs):
486 return perf_builder(
487 name = name,
488 perf_cat = perf_cat,
489 recipe = recipe,
490 properties = merge_dicts(GOMA_BACKEND_WEBRTC_RBE_PROD, properties),
491 dimensions = dimensions,
492 **kwargs
493 )
494
495def linux_perf_builder(
496 name,
497 perf_cat,
498 recipe = "standalone",
499 properties = {},
500 dimensions = {},
501 **kwargs):
502 return perf_builder(
503 name = name,
504 perf_cat = perf_cat,
505 recipe = recipe,
506 properties = merge_dicts(GOMA_BACKEND_WEBRTC_RBE_PROD, properties),
507 dimensions = dimensions,
508 **kwargs
509 )
510
511def mac_perf_builder(
512 name,
513 perf_cat,
514 recipe = "standalone",
515 properties = {},
516 dimensions = {},
517 **kwargs):
518 return perf_builder(
519 name = name,
520 perf_cat = perf_cat,
521 recipe = recipe,
522 properties = merge_dicts(GOMA_BACKEND_WEBRTC_RBE_PROD, properties),
523 dimensions = dimensions,
524 **kwargs
525 )
526
527def win_perf_builder(
528 name,
529 perf_cat,
530 recipe = "standalone",
531 properties = {},
532 dimensions = {},
533 **kwargs):
534 return perf_builder(
535 name = name,
536 perf_cat = perf_cat,
537 recipe = recipe,
538 properties = merge_dicts(GOMA_BACKEND_RBE_ATS_PROD, properties),
539 dimensions = dimensions,
540 **kwargs
541 )
542
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100543def cron_builder(name, dimensions = {}, **kwargs):
544 add_milo(name, {"cron": True})
545 return webrtc_builder(
546 name = name,
547 dimensions = merge_dicts({"pool": "luci.webrtc.cron", "os": "Linux"}, dimensions),
548 bucket = "cron",
549 service_account = "chromium-webrtc-autoroll@webrtc-ci.iam.gserviceaccount.com",
Oleh Prypin71d17422019-03-28 08:43:16 +0100550 notifies = ["cron_notifier"],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100551 **kwargs
552 )
553
554# Helpers:
555
556def merge_dicts(a, b):
557 """Return the result of merging two dicts.
558
559 If matching values are both dicts or both lists, they will be merged (non-recursively).
560
561 Args:
562 a: first dict.
563 b: second dict (takes priority).
564 Returns:
565 Merged dict.
566 """
567 a = dict(a)
568 for k, bv in b.items():
569 av = a.get(k)
570 if type(av) == "dict" and type(bv) == "dict":
571 a[k] = dict(av)
572 a[k].update(bv)
573 elif type(av) == "list" and type(bv) == "list":
574 a[k] = av + bv
575 else:
576 a[k] = bv
577 return a
578
579def normal_builder_factory(**common_kwargs):
580 def builder(*args, **kwargs):
581 return ci_builder(*args, **merge_dicts(common_kwargs, kwargs))
582
583 def try_job(*args, **kwargs):
584 return try_builder(*args, **merge_dicts(common_kwargs, kwargs))
585
586 return builder, try_job
587
588# Mixins:
589
590linux_builder, linux_try_job = normal_builder_factory(
591 dimensions = {"os": "Linux", "inside_docker": "0"},
Mirko Bonadei32ca4862020-03-11 10:39:45 +0100592 properties = GOMA_BACKEND_WEBRTC_RBE_PROD,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100593)
594
595android_builder, android_try_job = normal_builder_factory(
596 dimensions = {"os": "Linux"},
Mirko Bonadei32ca4862020-03-11 10:39:45 +0100597 properties = GOMA_BACKEND_WEBRTC_RBE_PROD,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100598)
599
600win_builder, win_try_job = normal_builder_factory(
601 dimensions = {"os": "Windows"},
Patrik Höglund3916a972019-12-04 13:19:38 +0100602 properties = GOMA_BACKEND_RBE_ATS_PROD,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100603)
604
605mac_builder, mac_try_job = normal_builder_factory(
606 dimensions = {"os": "Mac"},
Mirko Bonadei32ca4862020-03-11 10:39:45 +0100607 properties = GOMA_BACKEND_WEBRTC_RBE_PROD,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100608)
609
610ios_builder, ios_try_job = normal_builder_factory(
Mirko Bonadei8096c232020-08-26 21:44:42 +0200611 dimensions = {"os": "Mac-10.15"},
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100612 recipe = "ios",
Mirko Bonadei84360ca2020-03-12 08:35:48 +0100613 properties = GOMA_BACKEND_WEBRTC_IOS_RBE_PROD,
614 caches = [swarming.cache("osx_sdk")],
615)
616
617# Some iOS builders don't use goma because they build using
618# `use_xcode_clang=true` which is not supported by goma.
619# https://ci.chromium.org/p/webrtc/builders/ci/iOS%20API%20Framework%20Builder
620# https://ci.chromium.org/p/webrtc/builders/try/ios_api_framework
621ios_builder_no_goma, ios_try_job_no_goma = normal_builder_factory(
Mirko Bonadei8096c232020-08-26 21:44:42 +0200622 dimensions = {"os": "Mac-10.15"},
Mirko Bonadei84360ca2020-03-12 08:35:48 +0100623 recipe = "ios",
Artem Titov1d47d312020-11-03 14:45:52 +0100624 properties = {"$depot_tools/osx_sdk": {"sdk_version": "12a7209"}},
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100625 caches = [swarming.cache("osx_sdk")],
626)
627
628# Actual builder configuration:
629
630android_builder("Android32 (M Nexus5X)(dbg)", "Android|arm|dbg")
631android_try_job("android_compile_arm_dbg", cq = None)
632android_try_job("android_arm_dbg")
633android_builder("Android32 (M Nexus5X)", "Android|arm|rel")
634android_try_job("android_arm_rel")
635android_builder("Android32 Builder arm", "Android|arm|size", perf_cat = "Android|arm|Builder|", prioritized = True)
636android_try_job("android_compile_arm_rel")
Mirko Bonadei86b4f2d2020-07-08 14:24:48 +0200637android_perf_builder("Perf Android32 (M Nexus5)", "Android|arm|Tester|M Nexus5", triggered_by = ["Android32 Builder arm"])
Mirko Bonadei12643772020-03-27 13:16:35 +0100638android_perf_builder("Perf Android32 (M AOSP Nexus6)", "Android|arm|Tester|M AOSP Nexus6", triggered_by = ["Android32 Builder arm"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100639android_try_job("android_compile_arm64_dbg", cq = None)
640android_try_job("android_arm64_dbg", cq = None)
641android_builder("Android64 (M Nexus5X)", "Android|arm64|rel")
642android_try_job("android_arm64_rel")
643android_builder("Android64 Builder arm64", "Android|arm64|size", perf_cat = "Android|arm64|Builder|", prioritized = True)
Mirko Bonadei12643772020-03-27 13:16:35 +0100644android_perf_builder("Perf Android64 (M Nexus5X)", "Android|arm64|Tester|M Nexus5X", triggered_by = ["Android64 Builder arm64"])
645android_perf_builder("Perf Android64 (O Pixel2)", "Android|arm64|Tester|O Pixel2", triggered_by = ["Android64 Builder arm64"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100646android_try_job("android_compile_arm64_rel")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100647android_builder("Android64 Builder x64 (dbg)", "Android|x64|dbg")
648android_try_job("android_compile_x64_dbg")
649android_try_job("android_compile_x64_rel", cq = None)
650android_builder("Android32 Builder x86 (dbg)", "Android|x86|dbg")
651android_try_job("android_compile_x86_dbg")
652android_builder("Android32 Builder x86", "Android|x86|rel")
653android_try_job("android_compile_x86_rel")
654android_builder("Android32 (more configs)", "Android|arm|more", recipe = "more_configs")
655android_try_job("android_arm_more_configs", recipe = "more_configs")
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100656android_try_job("android_chromium_compile", recipe = "chromium_trybot", branch_cq = False)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100657
658ios_builder("iOS32 Debug", "iOS|arm|dbg")
659ios_try_job("ios_compile_arm_dbg")
660ios_builder("iOS32 Release", "iOS|arm|rel")
661ios_try_job("ios_compile_arm_rel")
662ios_builder("iOS64 Debug", "iOS|arm64|dbg")
663ios_try_job("ios_compile_arm64_dbg")
664ios_builder("iOS64 Release", "iOS|arm64|rel")
665ios_try_job("ios_compile_arm64_rel")
Artem Titov1d47d312020-11-03 14:45:52 +0100666
667ios_builder("iOS64 Sim Debug (iOS 14.0)", "iOS|x64|14")
668ios_try_job("ios_sim_x64_dbg_ios14")
669ios_builder("iOS64 Sim Debug (iOS 13)", "iOS|x64|13")
670ios_try_job("ios_sim_x64_dbg_ios13")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100671ios_builder("iOS64 Sim Debug (iOS 12)", "iOS|x64|12")
Patrik Höglund17bed672020-02-20 13:31:36 +0000672ios_try_job("ios_sim_x64_dbg_ios12")
Mirko Bonadei84360ca2020-03-12 08:35:48 +0100673ios_builder_no_goma("iOS API Framework Builder", "iOS|fat|size", recipe = "ios_api_framework", prioritized = True)
Jeremy Leconteea2016b2020-07-01 09:47:22 +0200674ios_try_job_no_goma("ios_api_framework", recipe = "ios_api_framework", cq = None)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100675
676linux_builder("Linux32 Debug", "Linux|x86|dbg")
677linux_try_job("linux_x86_dbg")
678linux_builder("Linux32 Release", "Linux|x86|rel")
679linux_try_job("linux_x86_rel")
680linux_builder("Linux64 Debug", "Linux|x64|dbg")
681linux_try_job("linux_dbg", cq = None)
682linux_try_job("linux_compile_dbg")
683linux_builder("Linux64 Release", "Linux|x64|rel")
684linux_try_job("linux_rel")
685linux_builder("Linux64 Builder", "Linux|x64|size", perf_cat = "Linux|x64|Builder|", prioritized = True)
686linux_try_job("linux_compile_rel")
Mirko Bonadei12643772020-03-27 13:16:35 +0100687linux_perf_builder("Perf Linux Trusty", "Linux|x64|Tester|Trusty", triggered_by = ["Linux64 Builder"])
688linux_perf_builder("Perf Linux Xenial", None, triggered_by = ["Linux64 Builder"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100689linux_builder("Linux32 Debug (ARM)", "Linux|arm|dbg")
690linux_try_job("linux_compile_arm_dbg")
691linux_builder("Linux32 Release (ARM)", "Linux|arm|rel")
692linux_try_job("linux_compile_arm_rel")
693linux_builder("Linux64 Debug (ARM)", "Linux|arm64|dbg")
694linux_try_job("linux_compile_arm64_dbg")
695linux_builder("Linux64 Release (ARM)", "Linux|arm64|rel")
696linux_try_job("linux_compile_arm64_rel")
697linux_builder("Linux64 Release (GCC)", "Linux|x64|gcc")
698linux_try_job("linux_compile_gcc_rel")
699linux_builder("Linux Asan", "Linux|x64|asan")
700linux_try_job("linux_asan")
701linux_builder("Linux MSan", "Linux|x64|msan")
702linux_try_job("linux_msan")
703linux_builder("Linux Tsan v2", "Linux|x64|tsan")
704linux_try_job("linux_tsan2")
705linux_builder("Linux UBSan", "Linux|x64|ubsan")
706linux_try_job("linux_ubsan")
707linux_builder("Linux UBSan vptr", "Linux|x64|ubsan")
708linux_try_job("linux_ubsan_vptr")
709linux_builder("Linux64 Release (Libfuzzer)", "Linux|x64|fuzz", recipe = "libfuzzer")
710linux_try_job("linux_libfuzzer_rel", recipe = "libfuzzer")
711linux_builder("Linux (more configs)", "Linux|x64|more", recipe = "more_configs")
712linux_try_job("linux_more_configs", recipe = "more_configs")
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100713linux_try_job("linux_chromium_compile", recipe = "chromium_trybot", branch_cq = False)
Mirko Bonadei40de50f2019-11-07 11:23:47 +0100714linux_try_job("linux_chromium_compile_dbg", recipe = "chromium_trybot", branch_cq = False)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100715
716mac_builder("Mac64 Debug", "Mac|x64|dbg")
717mac_try_job("mac_dbg", cq = None)
718mac_try_job("mac_compile_dbg")
719mac_builder("Mac64 Release", "Mac|x64|rel")
720mac_try_job("mac_rel")
721mac_try_job("mac_compile_rel", cq = None)
722mac_builder("Mac64 Builder", ci_cat = None, perf_cat = "Mac|x64|Builder|")
Mirko Bonadei12643772020-03-27 13:16:35 +0100723mac_perf_builder("Perf Mac 10.11", "Mac|x64|Tester|10.11", triggered_by = ["Mac64 Builder"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100724mac_builder("Mac Asan", "Mac|x64|asan")
725mac_try_job("mac_asan")
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100726mac_try_job("mac_chromium_compile", recipe = "chromium_trybot", dimensions = {"cores": "8"}, branch_cq = False)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100727
728win_builder("Win32 Debug", "Win MSVC|x86|dbg")
729win_try_job("win_x86_msvc_dbg")
730win_try_job("win_compile_x86_msvc_dbg", cq = None)
Oleh Prypin63041b82019-03-19 22:56:01 +0000731win_builder("Win32 Release", "Win MSVC|x86|rel")
732win_try_job("win_x86_msvc_rel")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100733win_try_job("win_compile_x86_msvc_rel", cq = None)
734win_builder("Win64 Debug", "Win MSVC|x64|dbg")
735win_try_job("win_x64_msvc_dbg")
736win_try_job("win_compile_x64_msvc_dbg", cq = None)
737win_builder("Win64 Release", "Win MSVC|x64|rel")
738win_try_job("win_x64_msvc_rel")
739win_try_job("win_compile_x64_msvc_rel", cq = None)
740win_builder("Win32 Debug (Clang)", "Win Clang|x86|dbg")
741win_try_job("win_x86_clang_dbg", cq = None)
742win_try_job("win_compile_x86_clang_dbg")
743win_builder("Win32 Release (Clang)", "Win Clang|x86|rel")
744win_try_job("win_x86_clang_rel")
745win_try_job("win_compile_x86_clang_rel", cq = None)
746win_builder("Win32 Builder (Clang)", ci_cat = None, perf_cat = "Win|x86|Builder|")
Mirko Bonadei12643772020-03-27 13:16:35 +0100747win_perf_builder("Perf Win7", "Win|x86|Tester|7", triggered_by = ["Win32 Builder (Clang)"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100748win_builder("Win64 Debug (Clang)", "Win Clang|x64|dbg")
749win_try_job("win_x64_clang_dbg", cq = None)
750win_try_job("win_x64_clang_dbg_win8", cq = None)
751win_try_job("win_x64_clang_dbg_win10", cq = None)
752win_try_job("win_compile_x64_clang_dbg")
753win_builder("Win64 Release (Clang)", "Win Clang|x64|rel")
754win_try_job("win_x64_clang_rel", cq = None)
755win_try_job("win_compile_x64_clang_rel")
Mirko Bonadeiafbe33d2019-05-31 09:27:54 +0200756win_builder("Win64 ASan", "Win Clang|x64|asan")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100757win_try_job("win_asan")
758win_builder("Win64 UWP", ci_cat = None, fyi_cat = "")
759win_try_job("win_x64_uwp", cq = None, try_cat = None, fyi_cat = "")
760win_builder("Win (more configs)", "Win Clang|x86|more", recipe = "more_configs")
761win_try_job("win_x86_more_configs", recipe = "more_configs")
Mirko Bonadei825f9c22020-07-01 11:52:39 +0200762win_try_job("win_chromium_compile", recipe = "chromium_trybot", branch_cq = False, goma_jobs = 150)
763win_try_job("win_chromium_compile_dbg", recipe = "chromium_trybot", branch_cq = False, goma_jobs = 150)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100764
765linux_try_job(
766 "presubmit",
767 recipe = "run_presubmit",
768 properties = {"repo_name": "webrtc", "runhooks": True},
769 priority = 28,
770 cq = {"disable_reuse": True},
771)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100772
773cron_builder(
774 "Auto-roll - WebRTC DEPS",
775 recipe = "auto_roll_webrtc_deps",
Oleh Prypindc68a722019-06-25 10:43:13 +0200776 schedule = "0 */2 * * *", # Every 2 hours.
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100777)
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200778
779lkgr_config = {
780 "project": "webrtc",
781 "source_url": WEBRTC_GIT,
782 "status_url": "https://webrtc-status.appspot.com",
783 "allowed_lag": 300, # hours
784 "allowed_gap": 12, # commits behind
Mirko Bonadei450e9b72020-07-10 19:19:22 +0000785 "error_recipients": "webrtc-troopers-robots@google.com",
Michael Achenbach09dd7dc2019-07-26 15:26:11 +0200786 "buckets": {
787 "webrtc/ci": {
788 # bucket alias: luci.webrtc.ci
Mirko Bonadei55fde3e2020-10-19 14:22:00 +0200789 "builders": sorted(lkgr_builders),
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200790 },
Michael Achenbach09dd7dc2019-07-26 15:26:11 +0200791 "chromium/webrtc.fyi": {
792 # bucket alias: luci.chromium.webrtc.fyi
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200793 "builders": [
794 "WebRTC Chromium FYI Android Builder (dbg)",
795 "WebRTC Chromium FYI Android Builder ARM64 (dbg)",
796 "WebRTC Chromium FYI Android Builder",
Mirko Bonadei04fe32f2020-05-01 00:45:58 +0200797 "WebRTC Chromium FYI Android Tests (dbg) (L Nexus5)",
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200798 "WebRTC Chromium FYI Android Tests (dbg) (M Nexus5X)",
799 "WebRTC Chromium FYI Linux Builder (dbg)",
800 "WebRTC Chromium FYI Linux Builder",
801 "WebRTC Chromium FYI Linux Tester",
802 "WebRTC Chromium FYI Mac Builder (dbg)",
803 "WebRTC Chromium FYI Mac Builder",
804 "WebRTC Chromium FYI Mac Tester",
805 "WebRTC Chromium FYI Win Builder (dbg)",
806 "WebRTC Chromium FYI Win Builder",
807 "WebRTC Chromium FYI Win10 Tester",
808 "WebRTC Chromium FYI Win7 Tester",
809 "WebRTC Chromium FYI Win8 Tester",
Artem Titov95e94522020-11-03 13:02:15 +0100810 #"WebRTC Chromium FYI ios-device",
811 #"WebRTC Chromium FYI ios-simulator",
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200812 ],
813 },
814 },
815}
816
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100817cron_builder(
818 "WebRTC lkgr finder",
819 recipe = "lkgr_finder",
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200820 properties = {
821 "project": "webrtc",
822 "repo": WEBRTC_GIT,
823 "ref": "refs/heads/lkgr",
824 "lkgr_status_gs_path": "chromium-webrtc/lkgr-status",
825 "config": lkgr_config,
826 },
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100827 schedule = "*/10 * * * *", # Every 10 minutes.
828)