blob: bba14e7497aacbe917f4bfa85e7c0a2c1c46a7c3 [file] [log] [blame]
Oleh Prypin1a0593f2019-03-11 09:43:28 +01001#!/usr/bin/env lucicfg
2
3# Copyright (c) 2019 The WebRTC project authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7# https://chromium.googlesource.com/infra/luci/luci-go/+/master/lucicfg/doc/
8
Jeremy Leconteea2016b2020-07-01 09:47:22 +02009"""LUCI project configuration for WebRTC CQ and CI."""
10
11lucicfg.check_version("1.15.0")
Mirko Bonadei9189d9c2020-06-09 21:54:18 +020012
Oleh Prypin1a0593f2019-03-11 09:43:28 +010013WEBRTC_GIT = "https://webrtc.googlesource.com/src"
14WEBRTC_GERRIT = "https://webrtc-review.googlesource.com/src"
15
Mirko Bonadei32ca4862020-03-11 10:39:45 +010016GOMA_BACKEND_WEBRTC_RBE_PROD = {
Mirko Bonadei1e905342020-03-04 09:39:42 +010017 "$build/goma": {
18 "server_host": "goma.chromium.org",
Jeremy Leconteea2016b2020-07-01 09:47:22 +020019 "use_luci_auth": True,
20 },
Mirko Bonadei1e905342020-03-04 09:39:42 +010021}
22
Mirko Bonadei84360ca2020-03-12 08:35:48 +010023GOMA_BACKEND_WEBRTC_IOS_RBE_PROD = {
24 "$build/goma": {
25 "server_host": "goma.chromium.org",
Jeremy Leconteea2016b2020-07-01 09:47:22 +020026 "use_luci_auth": True,
Mirko Bonadei84360ca2020-03-12 08:35:48 +010027 },
Artem Titov1d47d312020-11-03 14:45:52 +010028 "$depot_tools/osx_sdk": {"sdk_version": "12a7209"},
Mirko Bonadei84360ca2020-03-12 08:35:48 +010029}
30
Patrik Höglund3916a972019-12-04 13:19:38 +010031GOMA_BACKEND_RBE_ATS_PROD = {
32 "$build/goma": {
33 "server_host": "goma.chromium.org",
Mirko Bonadei2688eca2020-03-25 09:15:54 +010034 "use_luci_auth": True,
Jeremy Leconteea2016b2020-07-01 09:47:22 +020035 "enable_ats": True,
36 },
Patrik Höglund3916a972019-12-04 13:19:38 +010037}
38
Oleh Prypin1a0593f2019-03-11 09:43:28 +010039# Top-level configs:
40
Owen Rodley6b721ba2020-05-26 10:04:20 +100041# Enable luci.tree_closer.
42lucicfg.enable_experiment("crbug.com/1054172")
43
Oleh Prypin1a0593f2019-03-11 09:43:28 +010044lucicfg.config(
45 config_dir = ".",
46 tracked_files = [
47 "commit-queue.cfg",
48 "cr-buildbucket.cfg",
49 "luci-logdog.cfg",
50 "luci-milo.cfg",
Oleh Prypin71d17422019-03-28 08:43:16 +010051 "luci-notify.cfg",
Oleh Prypin705b6a62019-04-03 23:10:51 +020052 "luci-notify/**/*",
Oleh Prypin1a0593f2019-03-11 09:43:28 +010053 "luci-scheduler.cfg",
54 "project.cfg",
55 ],
Jeremy Leconteea2016b2020-07-01 09:47:22 +020056 lint_checks = ["default"],
Oleh Prypin1a0593f2019-03-11 09:43:28 +010057)
58
59luci.project(
60 name = "webrtc",
61 buildbucket = "cr-buildbucket.appspot.com",
62 logdog = "luci-logdog.appspot.com",
63 milo = "luci-milo.appspot.com",
Oleh Prypin71d17422019-03-28 08:43:16 +010064 notify = "luci-notify.appspot.com",
Oleh Prypin1a0593f2019-03-11 09:43:28 +010065 scheduler = "luci-scheduler.appspot.com",
66 swarming = "chromium-swarm.appspot.com",
67 acls = [
68 acl.entry(
69 [acl.BUILDBUCKET_READER, acl.LOGDOG_READER, acl.PROJECT_CONFIGS_READER, acl.SCHEDULER_READER],
70 groups = ["all"],
71 ),
72 acl.entry(acl.LOGDOG_WRITER, groups = ["luci-logdog-chromium-writers"]),
73 acl.entry(acl.SCHEDULER_OWNER, groups = ["project-webrtc-admins"]),
74 ],
75)
76
77luci.logdog(
78 gs_bucket = "chromium-luci-logdog",
79)
80
81luci.milo(
82 logo = "https://storage.googleapis.com/chrome-infra/webrtc-logo-vert-retro-255x305.png",
83)
84
Owen Rodleya55c38e2020-06-15 11:16:10 +100085luci.notify(tree_closing_enabled = True)
86
Oleh Prypin1a0593f2019-03-11 09:43:28 +010087luci.cq(
88 status_host = "chromium-cq-status.appspot.com",
89 submit_max_burst = 1,
90 submit_burst_delay = 1 * time.minute,
91)
92
93luci.gitiles_poller(
94 name = "webrtc-gitiles-trigger-master",
95 bucket = "ci",
96 repo = WEBRTC_GIT,
97)
98
99# Bucket definitions:
100
101luci.bucket(
102 name = "try",
103 acls = [
104 acl.entry(acl.BUILDBUCKET_TRIGGERER, groups = [
105 "service-account-cq",
106 "project-webrtc-tryjob-access",
107 ]),
108 ],
109)
110
111luci.bucket(
112 name = "ci",
113 acls = [
Oleh Prypinf35939d2019-05-03 20:42:38 +0200114 acl.entry(acl.BUILDBUCKET_TRIGGERER, groups = [
115 "project-webrtc-ci-schedulers",
116 ], users = [
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100117 "luci-scheduler@appspot.gserviceaccount.com",
118 ]),
Jeremy Leconte2e25bb52020-07-16 09:38:56 +0200119 acl.entry(acl.BUILDBUCKET_TRIGGERER, groups = [
120 # Allow Pinpoint to trigger builds for bisection
121 "service-account-chromeperf",
122 ]),
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100123 ],
124)
125
126luci.bucket(
127 name = "perf",
128 acls = [
129 acl.entry(acl.BUILDBUCKET_TRIGGERER, users = [
130 "luci-scheduler@appspot.gserviceaccount.com",
131 "webrtc-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
132 ]),
133 ],
134)
135
136luci.bucket(
137 name = "cron",
138 acls = [
139 acl.entry(acl.BUILDBUCKET_TRIGGERER, users = [
140 "luci-scheduler@appspot.gserviceaccount.com",
141 ]),
142 ],
143)
144
145# Commit queue definitions:
146
147luci.cq_group(
148 name = "cq",
149 tree_status_host = "webrtc-status.appspot.com",
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100150 watch = [cq.refset(repo = WEBRTC_GERRIT, refs = ["refs/heads/master"])],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100151 acls = [
152 acl.entry(acl.CQ_COMMITTER, groups = ["project-webrtc-committers"]),
153 acl.entry(acl.CQ_DRY_RUNNER, groups = ["project-webrtc-tryjob-access"]),
154 ],
155 retry_config = cq.RETRY_ALL_FAILURES,
Oleh Prypin5416f2d2019-08-02 09:43:15 +0200156 cancel_stale_tryjobs = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100157)
158
159luci.cq_group(
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100160 name = "cq_branch",
161 watch = [cq.refset(repo = WEBRTC_GERRIT, refs = ["refs/branch-heads/.+"])],
162 acls = [
163 acl.entry(acl.CQ_COMMITTER, groups = ["project-webrtc-committers"]),
164 acl.entry(acl.CQ_DRY_RUNNER, groups = ["project-webrtc-tryjob-access"]),
165 ],
166 retry_config = cq.RETRY_ALL_FAILURES,
Oleh Prypin5416f2d2019-08-02 09:43:15 +0200167 cancel_stale_tryjobs = True,
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100168)
169
170luci.cq_group(
171 name = "cq_infra",
172 watch = [cq.refset(repo = WEBRTC_GERRIT, refs = ["refs/heads/infra/config"])],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100173 acls = [
174 acl.entry(acl.CQ_COMMITTER, groups = ["project-webrtc-admins"]),
175 acl.entry(acl.CQ_DRY_RUNNER, groups = ["project-webrtc-tryjob-access"]),
176 ],
177 retry_config = cq.RETRY_ALL_FAILURES,
Oleh Prypin5416f2d2019-08-02 09:43:15 +0200178 cancel_stale_tryjobs = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100179)
180
181luci.cq_tryjob_verifier(
182 builder = "presubmit",
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100183 cq_group = "cq_infra",
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100184)
185
Oleh Prypine56cd9b2019-05-29 21:14:31 +0200186luci.cq_tryjob_verifier(
187 builder = "webrtc-internal:g3.webrtc-internal.try/internal_compile_lite",
188 owner_whitelist = ["project-webrtc-internal-tryjob-access"],
189 cq_group = "cq",
190)
191
Oleh Prypin71d17422019-03-28 08:43:16 +0100192# Notifier definitions:
193
194luci.notifier(
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200195 name = "post_submit_failure_notifier",
196 on_new_status = ["FAILURE"],
Oleh Prypin71d17422019-03-28 08:43:16 +0100197 notify_emails = ["webrtc-sheriffs-robots@google.com"],
Oleh Prypin705b6a62019-04-03 23:10:51 +0200198 template = luci.notifier_template(
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200199 name = "build_failure",
200 body = io.read_file("luci-notify/email-templates/build_failure.template"),
Oleh Prypin705b6a62019-04-03 23:10:51 +0200201 ),
Oleh Prypin71d17422019-03-28 08:43:16 +0100202)
203
204luci.notifier(
205 name = "cron_notifier",
Jeremy Leconte67a904e2020-07-08 11:48:18 +0200206 on_new_status = ["FAILURE", "INFRA_FAILURE"],
Mirko Bonadei450e9b72020-07-10 19:19:22 +0000207 notify_emails = ["webrtc-troopers-robots@google.com"],
Oleh Prypin705b6a62019-04-03 23:10:51 +0200208 template = luci.notifier_template(
209 name = "cron",
210 body = io.read_file("luci-notify/email-templates/cron.template"),
211 ),
Oleh Prypin71d17422019-03-28 08:43:16 +0100212)
213
Jeremy Leconte67a904e2020-07-08 11:48:18 +0200214luci.notifier(
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200215 name = "infra_failure_notifier",
Jeremy Leconte67a904e2020-07-08 11:48:18 +0200216 on_new_status = ["INFRA_FAILURE"],
Mirko Bonadei450e9b72020-07-10 19:19:22 +0000217 notify_emails = ["webrtc-troopers-robots@google.com"],
Jeremy Leconte67a904e2020-07-08 11:48:18 +0200218 template = luci.notifier_template(
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200219 name = "infra_failure",
220 body = io.read_file("luci-notify/email-templates/infra_failure.template"),
Jeremy Leconte67a904e2020-07-08 11:48:18 +0200221 ),
222)
223
Owen Rodley6b721ba2020-05-26 10:04:20 +1000224# Tree closer definitions:
225
226luci.tree_closer(
227 name = "webrtc_tree_closer",
228 tree_status_host = "webrtc-status.appspot.com",
Owen Rodley6b721ba2020-05-26 10:04:20 +1000229 # TODO: These step filters are copied verbatim from Gatekeeper, for testing
230 # that LUCI-Notify would take the exact same actions. Once we've switched
231 # over, this should be updated - several of these steps don't exist in
232 # WebRTC recipes.
233 failed_step_regexp = [
234 "bot_update",
235 "compile",
236 "gclient runhooks",
237 "runhooks",
238 "update",
239 "extract build",
240 "cleanup_temp",
241 "taskkill",
242 "compile",
243 "gn",
244 ],
245 failed_step_regexp_exclude = ".*\\(experimental\\).*",
246)
247
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100248# Recipe definitions:
249
250def recipe(recipe, pkg = "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"):
251 return luci.recipe(
252 name = recipe.split("/")[-1],
253 cipd_package = pkg,
254 cipd_version = "refs/heads/master",
255 recipe = recipe,
256 )
257
258recipe("chromium_trybot")
259recipe("run_presubmit")
260recipe("webrtc/auto_roll_webrtc_deps")
261recipe("webrtc/ios")
262recipe("webrtc/ios_api_framework")
263recipe("webrtc/libfuzzer")
264recipe("webrtc/more_configs")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100265recipe("webrtc/standalone")
Mirko Bonadei54bbd1c2020-12-14 15:54:46 +0100266recipe("webrtc/update_webrtc_binary_version")
Jeff Yoonf421c682020-12-17 00:16:22 -0800267recipe("lkgr_finder", pkg = "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100268
269# Console definitions:
270
271luci.console_view(name = "ci", title = "Main", repo = WEBRTC_GIT, header = "console-header.textpb")
272luci.console_view(name = "perf", title = "Perf", repo = WEBRTC_GIT, header = "console-header.textpb")
273luci.list_view(name = "cron", title = "Cron")
274luci.console_view(name = "fyi", title = "FYI", repo = WEBRTC_GIT, header = "console-header.textpb")
275luci.list_view(name = "try", title = "Tryserver")
276
277def add_milo(builder, views):
278 """Add Milo console entries for the builder.
279
280 Args:
281 builder: builder name (str).
282 views: dict where keys are names of consoles and values are either a
283 category for the console (str, pipe-separated) or True, which means
284 adding to a list view rather than a console.
285 """
286 for view_name, category in views.items():
287 if category == None:
288 continue
289 elif type(category) == "string":
290 category, _, short_name = category.rpartition("|")
291 luci.console_view_entry(
292 console_view = view_name,
293 builder = builder,
294 category = category or None,
295 short_name = short_name or None,
296 )
297 elif category == True:
298 luci.list_view_entry(
299 list_view = view_name,
300 builder = builder,
301 )
302 else:
303 fail("Unexpected value for category: %r" % category)
304
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200305lkgr_builders = {}
306
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100307# Builder-defining functions:
308
Artem Titarenkof6588b72019-11-14 14:28:11 +0100309def webrtc_builder(
310 name,
311 recipe = "standalone",
312 dimensions = {},
Mirko Bonadei825f9c22020-07-01 11:52:39 +0200313 properties = {},
Artem Titarenkof6588b72019-11-14 14:28:11 +0100314 priority = 30,
315 execution_timeout = 2 * time.hour,
Mirko Bonadeib7ef4602020-07-01 13:28:26 +0200316 goma_jobs = None,
Artem Titarenkof6588b72019-11-14 14:28:11 +0100317 **kwargs):
Mirko Bonadeib7ef4602020-07-01 13:28:26 +0200318 """WebRTC specific wrapper around luci.builder.
319
320 Args:
321 name: builder name (str).
322 recipe: string with the name of the recipe to run.
323 dimensions: dict of Swarming dimensions (strings) to search machines by.
324 properties: dict of properties to pass to the recipe (on top of the default ones).
325 priority: int [1-255] or None, indicating swarming task priority, lower is
326 more important. If None, defer the decision to Buildbucket service.
327 execution_timeout: int or None, how long to wait for a running build to finish before
328 forcefully aborting it and marking the build as timed out. If None,
329 defer the decision to Buildbucket service.
330 goma_jobs: int or None, number of jobs to be used by the builder. If None, defer the
331 decition to the goma module.
332 **kwargs: Pass on to webrtc_builder / luci.builder.
333 Returns:
334 A luci.builder.
335 """
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100336 dimensions = merge_dicts({"cpu": "x86-64"}, dimensions)
337
Mirko Bonadeib7ef4602020-07-01 13:28:26 +0200338 if goma_jobs != None:
339 goma_additional_params = {"$build/goma": {"jobs": goma_jobs}}
340 properties = merge_dicts(properties, goma_additional_params)
Ye Kuang8df20492020-10-15 14:54:17 +0900341 properties = merge_dicts(properties, {
342 "$recipe_engine/isolated": {
343 "server": "https://isolateserver.appspot.com",
344 },
345 })
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100346 return luci.builder(
347 name = name,
Oleh Prypin705b6a62019-04-03 23:10:51 +0200348 executable = recipe,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100349 dimensions = {k: v for k, v in dimensions.items() if v != None},
Mirko Bonadeib7ef4602020-07-01 13:28:26 +0200350 properties = properties,
Artem Titarenkof6588b72019-11-14 14:28:11 +0100351 execution_timeout = execution_timeout,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100352 priority = priority,
353 build_numbers = True,
354 swarming_tags = ["vpython:native-python-wrapper"],
355 **kwargs
356 )
357
358def ci_builder(
359 name,
360 ci_cat,
361 perf_cat = None,
362 fyi_cat = None,
363 properties = {},
364 dimensions = {},
365 prioritized = False,
Oleh Prypinf5432c52019-03-19 15:10:37 +0100366 enabled = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100367 **kwargs):
Oleh Prypine5677672019-04-25 12:21:35 +0200368 """Add a post-submit builder.
369
370 Args:
371 name: builder name (str).
372 ci_cat: the category + name for the /ci/ console, or None to omit from the console.
373 perf_cat: the category + name for the /perf/ console, or None to omit from the console.
374 fyi_cat: the category + name for the /fyi/ console, or None to omit from the console.
375 properties: dict of properties to pass to the recipe (on top of the default ones).
376 dimensions: dict of Swarming dimensions (strings) to search machines by.
377 prioritized: True to make this builder have a higher priority and never batch builds.
378 enabled: False to exclude this builder from consoles and failure notifications.
379 **kwargs: Pass on to webrtc_builder / luci.builder.
380 Returns:
381 A luci.builder.
382
383 Notifications are also disabled if a builder is not on either of /ci/ or /perf/ consoles.
384 """
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100385 if prioritized:
386 kwargs["triggering_policy"] = scheduler.greedy_batching(
387 max_batch_size = 1,
388 max_concurrent_invocations = 3,
389 )
390 kwargs["priority"] = 29
391
Oleh Prypinf5432c52019-03-19 15:10:37 +0100392 if enabled:
393 add_milo(name, {"ci": ci_cat, "perf": perf_cat, "fyi": fyi_cat})
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200394 if ci_cat:
395 lkgr_builders[name] = True
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100396 return webrtc_builder(
397 name = name,
Garrett Beaty4a106ce2020-09-09 14:51:23 -0700398 properties = merge_dicts({"builder_group": "client.webrtc"}, properties),
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100399 dimensions = merge_dicts({"pool": "luci.webrtc.ci"}, dimensions),
400 bucket = "ci",
401 service_account = "webrtc-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
Oleh Prypinf5432c52019-03-19 15:10:37 +0100402 triggered_by = ["webrtc-gitiles-trigger-master"] if enabled else None,
Oleh Prypin71d17422019-03-28 08:43:16 +0100403 repo = WEBRTC_GIT,
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200404 notifies = ["post_submit_failure_notifier", "webrtc_tree_closer", "infra_failure_notifier"] if enabled and (ci_cat or perf_cat) else None,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100405 **kwargs
406 )
407
408def try_builder(
409 name,
410 try_cat = True,
411 fyi_cat = None,
412 properties = {},
413 dimensions = {},
414 cq = {},
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100415 branch_cq = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100416 **kwargs):
Oleh Prypine5677672019-04-25 12:21:35 +0200417 """Add a pre-submit builder.
418
419 Args:
420 name: builder name (str).
421 try_cat: boolean, whether to include this builder in the /try/ console. See also: `add_milo`.
422 fyi_cat: the category + name for the /fyi/ console, or None to omit from the console.
423 properties: dict of properties to pass to the recipe (on top of the default ones).
424 dimensions: dict of Swarming dimensions (strings) to search machines by.
425 cq: None to exclude this from all commit queues, or a dict of kwargs for cq_tryjob_verifier.
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100426 branch_cq: False to exclude this builder just from the release-branch CQ.
Oleh Prypine5677672019-04-25 12:21:35 +0200427 **kwargs: Pass on to webrtc_builder / luci.builder.
428 Returns:
429 A luci.builder.
430 """
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100431 add_milo(name, {"try": try_cat, "fyi": fyi_cat})
432 if cq != None:
433 luci.cq_tryjob_verifier(
434 builder = name,
435 cq_group = "cq",
436 **cq
437 )
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100438 if branch_cq:
439 luci.cq_tryjob_verifier(
440 builder = name,
441 cq_group = "cq_branch",
442 **cq
443 )
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100444
445 return webrtc_builder(
446 name = name,
Garrett Beaty4a106ce2020-09-09 14:51:23 -0700447 properties = merge_dicts({"builder_group": "tryserver.webrtc"}, properties),
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100448 dimensions = merge_dicts({"pool": "luci.webrtc.try"}, dimensions),
449 bucket = "try",
450 service_account = "webrtc-try-builder@chops-service-accounts.iam.gserviceaccount.com",
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200451 notifies = ["infra_failure_notifier"],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100452 **kwargs
453 )
454
455def perf_builder(
456 name,
457 perf_cat,
458 recipe = "standalone",
459 properties = {},
460 dimensions = {},
461 **kwargs):
462 add_milo(name, {"perf": perf_cat})
463 return webrtc_builder(
464 name = name,
465 recipe = recipe,
Garrett Beaty4a106ce2020-09-09 14:51:23 -0700466 properties = merge_dicts({"builder_group": "client.webrtc.perf"}, properties),
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100467 dimensions = merge_dicts({"pool": "luci.webrtc.perf", "os": "Linux", "cpu": None}, dimensions),
468 bucket = "perf",
469 service_account = "webrtc-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
470 # log_base of 1.7 means:
471 # when there are P pending builds, LUCI will batch the first B builds.
472 # P: 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 ...
473 # B: 1 1 2 2 3 3 3 3 4 4 4 4 4 4 5 ...
Jeremy Leconteea2016b2020-07-01 09:47:22 +0200474 triggering_policy = scheduler.logarithmic_batching(log_base = 1.7),
Artem Titarenkof6588b72019-11-14 14:28:11 +0100475 execution_timeout = 3 * time.hour,
Mirko Bonadei875a27a2020-07-11 15:04:58 +0200476 notifies = ["post_submit_failure_notifier", "infra_failure_notifier"],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100477 **kwargs
478 )
479
Mirko Bonadei12643772020-03-27 13:16:35 +0100480def android_perf_builder(
481 name,
482 perf_cat,
483 recipe = "standalone",
484 properties = {},
485 dimensions = {},
486 **kwargs):
487 return perf_builder(
488 name = name,
489 perf_cat = perf_cat,
490 recipe = recipe,
491 properties = merge_dicts(GOMA_BACKEND_WEBRTC_RBE_PROD, properties),
492 dimensions = dimensions,
493 **kwargs
494 )
495
496def linux_perf_builder(
497 name,
498 perf_cat,
499 recipe = "standalone",
500 properties = {},
501 dimensions = {},
502 **kwargs):
503 return perf_builder(
504 name = name,
505 perf_cat = perf_cat,
506 recipe = recipe,
507 properties = merge_dicts(GOMA_BACKEND_WEBRTC_RBE_PROD, properties),
508 dimensions = dimensions,
509 **kwargs
510 )
511
512def mac_perf_builder(
513 name,
514 perf_cat,
515 recipe = "standalone",
516 properties = {},
517 dimensions = {},
518 **kwargs):
519 return perf_builder(
520 name = name,
521 perf_cat = perf_cat,
522 recipe = recipe,
523 properties = merge_dicts(GOMA_BACKEND_WEBRTC_RBE_PROD, properties),
524 dimensions = dimensions,
525 **kwargs
526 )
527
528def win_perf_builder(
529 name,
530 perf_cat,
531 recipe = "standalone",
532 properties = {},
533 dimensions = {},
534 **kwargs):
535 return perf_builder(
536 name = name,
537 perf_cat = perf_cat,
538 recipe = recipe,
539 properties = merge_dicts(GOMA_BACKEND_RBE_ATS_PROD, properties),
540 dimensions = dimensions,
541 **kwargs
542 )
543
Mirko Bonadei54bbd1c2020-12-14 15:54:46 +0100544def cron_builder(name, dimensions = {}, service_account = None, **kwargs):
545 if service_account == None:
546 service_account = "chromium-webrtc-autoroll@webrtc-ci.iam.gserviceaccount.com"
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100547 add_milo(name, {"cron": True})
548 return webrtc_builder(
549 name = name,
550 dimensions = merge_dicts({"pool": "luci.webrtc.cron", "os": "Linux"}, dimensions),
551 bucket = "cron",
Mirko Bonadei54bbd1c2020-12-14 15:54:46 +0100552 service_account = service_account,
Oleh Prypin71d17422019-03-28 08:43:16 +0100553 notifies = ["cron_notifier"],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100554 **kwargs
555 )
556
557# Helpers:
558
559def merge_dicts(a, b):
560 """Return the result of merging two dicts.
561
562 If matching values are both dicts or both lists, they will be merged (non-recursively).
563
564 Args:
565 a: first dict.
566 b: second dict (takes priority).
567 Returns:
568 Merged dict.
569 """
570 a = dict(a)
571 for k, bv in b.items():
572 av = a.get(k)
573 if type(av) == "dict" and type(bv) == "dict":
574 a[k] = dict(av)
575 a[k].update(bv)
576 elif type(av) == "list" and type(bv) == "list":
577 a[k] = av + bv
578 else:
579 a[k] = bv
580 return a
581
582def normal_builder_factory(**common_kwargs):
583 def builder(*args, **kwargs):
584 return ci_builder(*args, **merge_dicts(common_kwargs, kwargs))
585
586 def try_job(*args, **kwargs):
587 return try_builder(*args, **merge_dicts(common_kwargs, kwargs))
588
589 return builder, try_job
590
591# Mixins:
592
593linux_builder, linux_try_job = normal_builder_factory(
594 dimensions = {"os": "Linux", "inside_docker": "0"},
Mirko Bonadei32ca4862020-03-11 10:39:45 +0100595 properties = GOMA_BACKEND_WEBRTC_RBE_PROD,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100596)
597
598android_builder, android_try_job = normal_builder_factory(
599 dimensions = {"os": "Linux"},
Mirko Bonadei32ca4862020-03-11 10:39:45 +0100600 properties = GOMA_BACKEND_WEBRTC_RBE_PROD,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100601)
602
603win_builder, win_try_job = normal_builder_factory(
604 dimensions = {"os": "Windows"},
Patrik Höglund3916a972019-12-04 13:19:38 +0100605 properties = GOMA_BACKEND_RBE_ATS_PROD,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100606)
607
608mac_builder, mac_try_job = normal_builder_factory(
609 dimensions = {"os": "Mac"},
Mirko Bonadei32ca4862020-03-11 10:39:45 +0100610 properties = GOMA_BACKEND_WEBRTC_RBE_PROD,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100611)
612
613ios_builder, ios_try_job = normal_builder_factory(
Mirko Bonadei8096c232020-08-26 21:44:42 +0200614 dimensions = {"os": "Mac-10.15"},
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100615 recipe = "ios",
Mirko Bonadei84360ca2020-03-12 08:35:48 +0100616 properties = GOMA_BACKEND_WEBRTC_IOS_RBE_PROD,
617 caches = [swarming.cache("osx_sdk")],
618)
619
620# Some iOS builders don't use goma because they build using
621# `use_xcode_clang=true` which is not supported by goma.
622# https://ci.chromium.org/p/webrtc/builders/ci/iOS%20API%20Framework%20Builder
623# https://ci.chromium.org/p/webrtc/builders/try/ios_api_framework
624ios_builder_no_goma, ios_try_job_no_goma = normal_builder_factory(
Mirko Bonadei8096c232020-08-26 21:44:42 +0200625 dimensions = {"os": "Mac-10.15"},
Mirko Bonadei84360ca2020-03-12 08:35:48 +0100626 recipe = "ios",
Artem Titov1d47d312020-11-03 14:45:52 +0100627 properties = {"$depot_tools/osx_sdk": {"sdk_version": "12a7209"}},
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100628 caches = [swarming.cache("osx_sdk")],
629)
630
631# Actual builder configuration:
632
633android_builder("Android32 (M Nexus5X)(dbg)", "Android|arm|dbg")
634android_try_job("android_compile_arm_dbg", cq = None)
635android_try_job("android_arm_dbg")
636android_builder("Android32 (M Nexus5X)", "Android|arm|rel")
637android_try_job("android_arm_rel")
638android_builder("Android32 Builder arm", "Android|arm|size", perf_cat = "Android|arm|Builder|", prioritized = True)
639android_try_job("android_compile_arm_rel")
Mirko Bonadei86b4f2d2020-07-08 14:24:48 +0200640android_perf_builder("Perf Android32 (M Nexus5)", "Android|arm|Tester|M Nexus5", triggered_by = ["Android32 Builder arm"])
Mirko Bonadei12643772020-03-27 13:16:35 +0100641android_perf_builder("Perf Android32 (M AOSP Nexus6)", "Android|arm|Tester|M AOSP Nexus6", triggered_by = ["Android32 Builder arm"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100642android_try_job("android_compile_arm64_dbg", cq = None)
643android_try_job("android_arm64_dbg", cq = None)
644android_builder("Android64 (M Nexus5X)", "Android|arm64|rel")
645android_try_job("android_arm64_rel")
646android_builder("Android64 Builder arm64", "Android|arm64|size", perf_cat = "Android|arm64|Builder|", prioritized = True)
Mirko Bonadei12643772020-03-27 13:16:35 +0100647android_perf_builder("Perf Android64 (M Nexus5X)", "Android|arm64|Tester|M Nexus5X", triggered_by = ["Android64 Builder arm64"])
648android_perf_builder("Perf Android64 (O Pixel2)", "Android|arm64|Tester|O Pixel2", triggered_by = ["Android64 Builder arm64"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100649android_try_job("android_compile_arm64_rel")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100650android_builder("Android64 Builder x64 (dbg)", "Android|x64|dbg")
651android_try_job("android_compile_x64_dbg")
652android_try_job("android_compile_x64_rel", cq = None)
653android_builder("Android32 Builder x86 (dbg)", "Android|x86|dbg")
654android_try_job("android_compile_x86_dbg")
655android_builder("Android32 Builder x86", "Android|x86|rel")
656android_try_job("android_compile_x86_rel")
657android_builder("Android32 (more configs)", "Android|arm|more", recipe = "more_configs")
658android_try_job("android_arm_more_configs", recipe = "more_configs")
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100659android_try_job("android_chromium_compile", recipe = "chromium_trybot", branch_cq = False)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100660
661ios_builder("iOS32 Debug", "iOS|arm|dbg")
662ios_try_job("ios_compile_arm_dbg")
663ios_builder("iOS32 Release", "iOS|arm|rel")
664ios_try_job("ios_compile_arm_rel")
665ios_builder("iOS64 Debug", "iOS|arm64|dbg")
666ios_try_job("ios_compile_arm64_dbg")
667ios_builder("iOS64 Release", "iOS|arm64|rel")
668ios_try_job("ios_compile_arm64_rel")
Artem Titov1d47d312020-11-03 14:45:52 +0100669ios_builder("iOS64 Sim Debug (iOS 14.0)", "iOS|x64|14")
Mirko Bonadei83b5f232020-12-01 16:56:48 +0100670ios_try_job("ios_sim_x64_dbg_ios14")
Artem Titov1d47d312020-11-03 14:45:52 +0100671ios_builder("iOS64 Sim Debug (iOS 13)", "iOS|x64|13")
Artem Titove973dd62020-11-03 18:05:21 +0100672ios_try_job("ios_sim_x64_dbg_ios13", cq = None)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100673ios_builder("iOS64 Sim Debug (iOS 12)", "iOS|x64|12")
Mirko Bonadei83b5f232020-12-01 16:56:48 +0100674ios_try_job("ios_sim_x64_dbg_ios12")
Mirko Bonadei84360ca2020-03-12 08:35:48 +0100675ios_builder_no_goma("iOS API Framework Builder", "iOS|fat|size", recipe = "ios_api_framework", prioritized = True)
Jeremy Leconteea2016b2020-07-01 09:47:22 +0200676ios_try_job_no_goma("ios_api_framework", recipe = "ios_api_framework", cq = None)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100677
678linux_builder("Linux32 Debug", "Linux|x86|dbg")
679linux_try_job("linux_x86_dbg")
680linux_builder("Linux32 Release", "Linux|x86|rel")
681linux_try_job("linux_x86_rel")
682linux_builder("Linux64 Debug", "Linux|x64|dbg")
683linux_try_job("linux_dbg", cq = None)
684linux_try_job("linux_compile_dbg")
685linux_builder("Linux64 Release", "Linux|x64|rel")
686linux_try_job("linux_rel")
687linux_builder("Linux64 Builder", "Linux|x64|size", perf_cat = "Linux|x64|Builder|", prioritized = True)
688linux_try_job("linux_compile_rel")
Mirko Bonadei12643772020-03-27 13:16:35 +0100689linux_perf_builder("Perf Linux Trusty", "Linux|x64|Tester|Trusty", triggered_by = ["Linux64 Builder"])
690linux_perf_builder("Perf Linux Xenial", None, triggered_by = ["Linux64 Builder"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100691linux_builder("Linux32 Debug (ARM)", "Linux|arm|dbg")
692linux_try_job("linux_compile_arm_dbg")
693linux_builder("Linux32 Release (ARM)", "Linux|arm|rel")
694linux_try_job("linux_compile_arm_rel")
695linux_builder("Linux64 Debug (ARM)", "Linux|arm64|dbg")
696linux_try_job("linux_compile_arm64_dbg")
697linux_builder("Linux64 Release (ARM)", "Linux|arm64|rel")
698linux_try_job("linux_compile_arm64_rel")
699linux_builder("Linux64 Release (GCC)", "Linux|x64|gcc")
700linux_try_job("linux_compile_gcc_rel")
701linux_builder("Linux Asan", "Linux|x64|asan")
702linux_try_job("linux_asan")
703linux_builder("Linux MSan", "Linux|x64|msan")
704linux_try_job("linux_msan")
705linux_builder("Linux Tsan v2", "Linux|x64|tsan")
706linux_try_job("linux_tsan2")
707linux_builder("Linux UBSan", "Linux|x64|ubsan")
708linux_try_job("linux_ubsan")
709linux_builder("Linux UBSan vptr", "Linux|x64|ubsan")
710linux_try_job("linux_ubsan_vptr")
711linux_builder("Linux64 Release (Libfuzzer)", "Linux|x64|fuzz", recipe = "libfuzzer")
712linux_try_job("linux_libfuzzer_rel", recipe = "libfuzzer")
713linux_builder("Linux (more configs)", "Linux|x64|more", recipe = "more_configs")
714linux_try_job("linux_more_configs", recipe = "more_configs")
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100715linux_try_job("linux_chromium_compile", recipe = "chromium_trybot", branch_cq = False)
Mirko Bonadei40de50f2019-11-07 11:23:47 +0100716linux_try_job("linux_chromium_compile_dbg", recipe = "chromium_trybot", branch_cq = False)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100717
718mac_builder("Mac64 Debug", "Mac|x64|dbg")
719mac_try_job("mac_dbg", cq = None)
720mac_try_job("mac_compile_dbg")
721mac_builder("Mac64 Release", "Mac|x64|rel")
722mac_try_job("mac_rel")
723mac_try_job("mac_compile_rel", cq = None)
724mac_builder("Mac64 Builder", ci_cat = None, perf_cat = "Mac|x64|Builder|")
Mirko Bonadei12643772020-03-27 13:16:35 +0100725mac_perf_builder("Perf Mac 10.11", "Mac|x64|Tester|10.11", triggered_by = ["Mac64 Builder"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100726mac_builder("Mac Asan", "Mac|x64|asan")
727mac_try_job("mac_asan")
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100728mac_try_job("mac_chromium_compile", recipe = "chromium_trybot", dimensions = {"cores": "8"}, branch_cq = False)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100729
730win_builder("Win32 Debug", "Win MSVC|x86|dbg")
731win_try_job("win_x86_msvc_dbg")
732win_try_job("win_compile_x86_msvc_dbg", cq = None)
Oleh Prypin63041b82019-03-19 22:56:01 +0000733win_builder("Win32 Release", "Win MSVC|x86|rel")
734win_try_job("win_x86_msvc_rel")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100735win_try_job("win_compile_x86_msvc_rel", cq = None)
736win_builder("Win64 Debug", "Win MSVC|x64|dbg")
737win_try_job("win_x64_msvc_dbg")
738win_try_job("win_compile_x64_msvc_dbg", cq = None)
739win_builder("Win64 Release", "Win MSVC|x64|rel")
740win_try_job("win_x64_msvc_rel")
741win_try_job("win_compile_x64_msvc_rel", cq = None)
742win_builder("Win32 Debug (Clang)", "Win Clang|x86|dbg")
743win_try_job("win_x86_clang_dbg", cq = None)
744win_try_job("win_compile_x86_clang_dbg")
745win_builder("Win32 Release (Clang)", "Win Clang|x86|rel")
746win_try_job("win_x86_clang_rel")
747win_try_job("win_compile_x86_clang_rel", cq = None)
748win_builder("Win32 Builder (Clang)", ci_cat = None, perf_cat = "Win|x86|Builder|")
Mirko Bonadei12643772020-03-27 13:16:35 +0100749win_perf_builder("Perf Win7", "Win|x86|Tester|7", triggered_by = ["Win32 Builder (Clang)"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100750win_builder("Win64 Debug (Clang)", "Win Clang|x64|dbg")
751win_try_job("win_x64_clang_dbg", cq = None)
752win_try_job("win_x64_clang_dbg_win8", cq = None)
753win_try_job("win_x64_clang_dbg_win10", cq = None)
754win_try_job("win_compile_x64_clang_dbg")
755win_builder("Win64 Release (Clang)", "Win Clang|x64|rel")
756win_try_job("win_x64_clang_rel", cq = None)
757win_try_job("win_compile_x64_clang_rel")
Mirko Bonadeiafbe33d2019-05-31 09:27:54 +0200758win_builder("Win64 ASan", "Win Clang|x64|asan")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100759win_try_job("win_asan")
760win_builder("Win64 UWP", ci_cat = None, fyi_cat = "")
761win_try_job("win_x64_uwp", cq = None, try_cat = None, fyi_cat = "")
762win_builder("Win (more configs)", "Win Clang|x86|more", recipe = "more_configs")
763win_try_job("win_x86_more_configs", recipe = "more_configs")
Mirko Bonadei825f9c22020-07-01 11:52:39 +0200764win_try_job("win_chromium_compile", recipe = "chromium_trybot", branch_cq = False, goma_jobs = 150)
765win_try_job("win_chromium_compile_dbg", recipe = "chromium_trybot", branch_cq = False, goma_jobs = 150)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100766
767linux_try_job(
768 "presubmit",
769 recipe = "run_presubmit",
770 properties = {"repo_name": "webrtc", "runhooks": True},
771 priority = 28,
772 cq = {"disable_reuse": True},
773)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100774
775cron_builder(
776 "Auto-roll - WebRTC DEPS",
777 recipe = "auto_roll_webrtc_deps",
Oleh Prypindc68a722019-06-25 10:43:13 +0200778 schedule = "0 */2 * * *", # Every 2 hours.
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100779)
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200780
Mirko Bonadei54bbd1c2020-12-14 15:54:46 +0100781cron_builder(
782 "WebRTC version update",
783 recipe = "update_webrtc_binary_version",
784 schedule = "0 4 * * *", # Every day at 4am.
785 service_account = "webrtc-version-updater@webrtc-ci.iam.gserviceaccount.com",
786)
787
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200788lkgr_config = {
789 "project": "webrtc",
790 "source_url": WEBRTC_GIT,
791 "status_url": "https://webrtc-status.appspot.com",
792 "allowed_lag": 300, # hours
793 "allowed_gap": 12, # commits behind
Mirko Bonadei450e9b72020-07-10 19:19:22 +0000794 "error_recipients": "webrtc-troopers-robots@google.com",
Michael Achenbach09dd7dc2019-07-26 15:26:11 +0200795 "buckets": {
796 "webrtc/ci": {
797 # bucket alias: luci.webrtc.ci
Mirko Bonadei83b5f232020-12-01 16:56:48 +0100798 "builders": sorted(lkgr_builders),
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200799 },
Michael Achenbach09dd7dc2019-07-26 15:26:11 +0200800 "chromium/webrtc.fyi": {
801 # bucket alias: luci.chromium.webrtc.fyi
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200802 "builders": [
803 "WebRTC Chromium FYI Android Builder (dbg)",
804 "WebRTC Chromium FYI Android Builder ARM64 (dbg)",
805 "WebRTC Chromium FYI Android Builder",
Mirko Bonadei04fe32f2020-05-01 00:45:58 +0200806 "WebRTC Chromium FYI Android Tests (dbg) (L Nexus5)",
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200807 "WebRTC Chromium FYI Android Tests (dbg) (M Nexus5X)",
808 "WebRTC Chromium FYI Linux Builder (dbg)",
809 "WebRTC Chromium FYI Linux Builder",
810 "WebRTC Chromium FYI Linux Tester",
811 "WebRTC Chromium FYI Mac Builder (dbg)",
812 "WebRTC Chromium FYI Mac Builder",
813 "WebRTC Chromium FYI Mac Tester",
814 "WebRTC Chromium FYI Win Builder (dbg)",
815 "WebRTC Chromium FYI Win Builder",
816 "WebRTC Chromium FYI Win10 Tester",
817 "WebRTC Chromium FYI Win7 Tester",
818 "WebRTC Chromium FYI Win8 Tester",
Mirko Bonadei2f05cfe2020-11-25 10:41:58 +0100819 "WebRTC Chromium FYI ios-device",
820 "WebRTC Chromium FYI ios-simulator",
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200821 ],
822 },
823 },
824}
825
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100826cron_builder(
827 "WebRTC lkgr finder",
828 recipe = "lkgr_finder",
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200829 properties = {
830 "project": "webrtc",
831 "repo": WEBRTC_GIT,
832 "ref": "refs/heads/lkgr",
833 "lkgr_status_gs_path": "chromium-webrtc/lkgr-status",
834 "config": lkgr_config,
835 },
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100836 schedule = "*/10 * * * *", # Every 10 minutes.
837)