blob: daef1ac689179dd256536641954277b254c52569 [file] [log] [blame]
Oleh Prypin1a0593f2019-03-11 09:43:28 +01001#!/usr/bin/env lucicfg
2
3# Copyright (c) 2019 The WebRTC project authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7# https://chromium.googlesource.com/infra/luci/luci-go/+/master/lucicfg/doc/
8
Jeremy Leconteea2016b2020-07-01 09:47:22 +02009"""LUCI project configuration for WebRTC CQ and CI."""
10
11lucicfg.check_version("1.15.0")
Mirko Bonadei9189d9c2020-06-09 21:54:18 +020012
Oleh Prypin1a0593f2019-03-11 09:43:28 +010013WEBRTC_GIT = "https://webrtc.googlesource.com/src"
14WEBRTC_GERRIT = "https://webrtc-review.googlesource.com/src"
15
Mirko Bonadei32ca4862020-03-11 10:39:45 +010016GOMA_BACKEND_WEBRTC_RBE_PROD = {
Mirko Bonadei1e905342020-03-04 09:39:42 +010017 "$build/goma": {
18 "server_host": "goma.chromium.org",
Jeremy Leconteea2016b2020-07-01 09:47:22 +020019 "use_luci_auth": True,
20 },
Mirko Bonadei1e905342020-03-04 09:39:42 +010021}
22
Mirko Bonadei84360ca2020-03-12 08:35:48 +010023GOMA_BACKEND_WEBRTC_IOS_RBE_PROD = {
24 "$build/goma": {
25 "server_host": "goma.chromium.org",
Jeremy Leconteea2016b2020-07-01 09:47:22 +020026 "use_luci_auth": True,
Mirko Bonadei84360ca2020-03-12 08:35:48 +010027 },
28 "$depot_tools/osx_sdk": {"sdk_version": "11c29"},
29}
30
Patrik Höglund3916a972019-12-04 13:19:38 +010031GOMA_BACKEND_RBE_ATS_PROD = {
32 "$build/goma": {
33 "server_host": "goma.chromium.org",
Mirko Bonadei2688eca2020-03-25 09:15:54 +010034 "use_luci_auth": True,
Jeremy Leconteea2016b2020-07-01 09:47:22 +020035 "enable_ats": True,
36 },
Patrik Höglund3916a972019-12-04 13:19:38 +010037}
38
Oleh Prypin1a0593f2019-03-11 09:43:28 +010039# Top-level configs:
40
Owen Rodley6b721ba2020-05-26 10:04:20 +100041# Enable luci.tree_closer.
42lucicfg.enable_experiment("crbug.com/1054172")
43
Oleh Prypin1a0593f2019-03-11 09:43:28 +010044lucicfg.config(
45 config_dir = ".",
46 tracked_files = [
47 "commit-queue.cfg",
48 "cr-buildbucket.cfg",
49 "luci-logdog.cfg",
50 "luci-milo.cfg",
Oleh Prypin71d17422019-03-28 08:43:16 +010051 "luci-notify.cfg",
Oleh Prypin705b6a62019-04-03 23:10:51 +020052 "luci-notify/**/*",
Oleh Prypin1a0593f2019-03-11 09:43:28 +010053 "luci-scheduler.cfg",
54 "project.cfg",
55 ],
Jeremy Leconteea2016b2020-07-01 09:47:22 +020056 lint_checks = ["default"],
Oleh Prypin1a0593f2019-03-11 09:43:28 +010057)
58
59luci.project(
60 name = "webrtc",
61 buildbucket = "cr-buildbucket.appspot.com",
62 logdog = "luci-logdog.appspot.com",
63 milo = "luci-milo.appspot.com",
Oleh Prypin71d17422019-03-28 08:43:16 +010064 notify = "luci-notify.appspot.com",
Oleh Prypin1a0593f2019-03-11 09:43:28 +010065 scheduler = "luci-scheduler.appspot.com",
66 swarming = "chromium-swarm.appspot.com",
67 acls = [
68 acl.entry(
69 [acl.BUILDBUCKET_READER, acl.LOGDOG_READER, acl.PROJECT_CONFIGS_READER, acl.SCHEDULER_READER],
70 groups = ["all"],
71 ),
72 acl.entry(acl.LOGDOG_WRITER, groups = ["luci-logdog-chromium-writers"]),
73 acl.entry(acl.SCHEDULER_OWNER, groups = ["project-webrtc-admins"]),
74 ],
75)
76
77luci.logdog(
78 gs_bucket = "chromium-luci-logdog",
79)
80
81luci.milo(
82 logo = "https://storage.googleapis.com/chrome-infra/webrtc-logo-vert-retro-255x305.png",
83)
84
Owen Rodleya55c38e2020-06-15 11:16:10 +100085luci.notify(tree_closing_enabled = True)
86
Oleh Prypin1a0593f2019-03-11 09:43:28 +010087luci.cq(
88 status_host = "chromium-cq-status.appspot.com",
89 submit_max_burst = 1,
90 submit_burst_delay = 1 * time.minute,
91)
92
93luci.gitiles_poller(
94 name = "webrtc-gitiles-trigger-master",
95 bucket = "ci",
96 repo = WEBRTC_GIT,
97)
98
99# Bucket definitions:
100
101luci.bucket(
102 name = "try",
103 acls = [
104 acl.entry(acl.BUILDBUCKET_TRIGGERER, groups = [
105 "service-account-cq",
106 "project-webrtc-tryjob-access",
107 ]),
108 ],
109)
110
111luci.bucket(
112 name = "ci",
113 acls = [
Oleh Prypinf35939d2019-05-03 20:42:38 +0200114 acl.entry(acl.BUILDBUCKET_TRIGGERER, groups = [
115 "project-webrtc-ci-schedulers",
116 ], users = [
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100117 "luci-scheduler@appspot.gserviceaccount.com",
118 ]),
119 ],
120)
121
122luci.bucket(
123 name = "perf",
124 acls = [
125 acl.entry(acl.BUILDBUCKET_TRIGGERER, users = [
126 "luci-scheduler@appspot.gserviceaccount.com",
127 "webrtc-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
128 ]),
129 ],
130)
131
132luci.bucket(
133 name = "cron",
134 acls = [
135 acl.entry(acl.BUILDBUCKET_TRIGGERER, users = [
136 "luci-scheduler@appspot.gserviceaccount.com",
137 ]),
138 ],
139)
140
141# Commit queue definitions:
142
143luci.cq_group(
144 name = "cq",
145 tree_status_host = "webrtc-status.appspot.com",
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100146 watch = [cq.refset(repo = WEBRTC_GERRIT, refs = ["refs/heads/master"])],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100147 acls = [
148 acl.entry(acl.CQ_COMMITTER, groups = ["project-webrtc-committers"]),
149 acl.entry(acl.CQ_DRY_RUNNER, groups = ["project-webrtc-tryjob-access"]),
150 ],
151 retry_config = cq.RETRY_ALL_FAILURES,
Oleh Prypin5416f2d2019-08-02 09:43:15 +0200152 cancel_stale_tryjobs = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100153)
154
155luci.cq_group(
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100156 name = "cq_branch",
157 watch = [cq.refset(repo = WEBRTC_GERRIT, refs = ["refs/branch-heads/.+"])],
158 acls = [
159 acl.entry(acl.CQ_COMMITTER, groups = ["project-webrtc-committers"]),
160 acl.entry(acl.CQ_DRY_RUNNER, groups = ["project-webrtc-tryjob-access"]),
161 ],
162 retry_config = cq.RETRY_ALL_FAILURES,
Oleh Prypin5416f2d2019-08-02 09:43:15 +0200163 cancel_stale_tryjobs = True,
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100164)
165
166luci.cq_group(
167 name = "cq_infra",
168 watch = [cq.refset(repo = WEBRTC_GERRIT, refs = ["refs/heads/infra/config"])],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100169 acls = [
170 acl.entry(acl.CQ_COMMITTER, groups = ["project-webrtc-admins"]),
171 acl.entry(acl.CQ_DRY_RUNNER, groups = ["project-webrtc-tryjob-access"]),
172 ],
173 retry_config = cq.RETRY_ALL_FAILURES,
Oleh Prypin5416f2d2019-08-02 09:43:15 +0200174 cancel_stale_tryjobs = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100175)
176
177luci.cq_tryjob_verifier(
178 builder = "presubmit",
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100179 cq_group = "cq_infra",
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100180)
181
Oleh Prypine56cd9b2019-05-29 21:14:31 +0200182luci.cq_tryjob_verifier(
183 builder = "webrtc-internal:g3.webrtc-internal.try/internal_compile_lite",
184 owner_whitelist = ["project-webrtc-internal-tryjob-access"],
185 cq_group = "cq",
186)
187
Oleh Prypin71d17422019-03-28 08:43:16 +0100188# Notifier definitions:
189
190luci.notifier(
191 name = "ci_notifier",
192 on_failure = True,
193 notify_emails = ["webrtc-sheriffs-robots@google.com"],
Oleh Prypin705b6a62019-04-03 23:10:51 +0200194 template = luci.notifier_template(
195 name = "ci",
196 body = io.read_file("luci-notify/email-templates/ci.template"),
197 ),
Oleh Prypin71d17422019-03-28 08:43:16 +0100198)
199
200luci.notifier(
201 name = "cron_notifier",
202 on_failure = True,
203 notify_emails = ["webrtc-troopers-robots@google.com"],
Oleh Prypin705b6a62019-04-03 23:10:51 +0200204 template = luci.notifier_template(
205 name = "cron",
206 body = io.read_file("luci-notify/email-templates/cron.template"),
207 ),
Oleh Prypin71d17422019-03-28 08:43:16 +0100208)
209
Owen Rodley6b721ba2020-05-26 10:04:20 +1000210# Tree closer definitions:
211
212luci.tree_closer(
213 name = "webrtc_tree_closer",
214 tree_status_host = "webrtc-status.appspot.com",
215 template = luci.notifier_template(
216 name = "status",
217 body = io.read_file("luci-notify/email-templates/status.template"),
218 ),
219 # TODO: These step filters are copied verbatim from Gatekeeper, for testing
220 # that LUCI-Notify would take the exact same actions. Once we've switched
221 # over, this should be updated - several of these steps don't exist in
222 # WebRTC recipes.
223 failed_step_regexp = [
224 "bot_update",
225 "compile",
226 "gclient runhooks",
227 "runhooks",
228 "update",
229 "extract build",
230 "cleanup_temp",
231 "taskkill",
232 "compile",
233 "gn",
234 ],
235 failed_step_regexp_exclude = ".*\\(experimental\\).*",
236)
237
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100238# Recipe definitions:
239
240def recipe(recipe, pkg = "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"):
241 return luci.recipe(
242 name = recipe.split("/")[-1],
243 cipd_package = pkg,
244 cipd_version = "refs/heads/master",
245 recipe = recipe,
246 )
247
248recipe("chromium_trybot")
249recipe("run_presubmit")
250recipe("webrtc/auto_roll_webrtc_deps")
251recipe("webrtc/ios")
252recipe("webrtc/ios_api_framework")
253recipe("webrtc/libfuzzer")
254recipe("webrtc/more_configs")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100255recipe("webrtc/standalone")
256recipe("lkgr_finder", pkg = "infra/recipe_bundles/chromium.googlesource.com/infra/infra")
257
258# Console definitions:
259
260luci.console_view(name = "ci", title = "Main", repo = WEBRTC_GIT, header = "console-header.textpb")
261luci.console_view(name = "perf", title = "Perf", repo = WEBRTC_GIT, header = "console-header.textpb")
262luci.list_view(name = "cron", title = "Cron")
263luci.console_view(name = "fyi", title = "FYI", repo = WEBRTC_GIT, header = "console-header.textpb")
264luci.list_view(name = "try", title = "Tryserver")
265
266def add_milo(builder, views):
267 """Add Milo console entries for the builder.
268
269 Args:
270 builder: builder name (str).
271 views: dict where keys are names of consoles and values are either a
272 category for the console (str, pipe-separated) or True, which means
273 adding to a list view rather than a console.
274 """
275 for view_name, category in views.items():
276 if category == None:
277 continue
278 elif type(category) == "string":
279 category, _, short_name = category.rpartition("|")
280 luci.console_view_entry(
281 console_view = view_name,
282 builder = builder,
283 category = category or None,
284 short_name = short_name or None,
285 )
286 elif category == True:
287 luci.list_view_entry(
288 list_view = view_name,
289 builder = builder,
290 )
291 else:
292 fail("Unexpected value for category: %r" % category)
293
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200294lkgr_builders = {}
295
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100296# Builder-defining functions:
297
Artem Titarenkof6588b72019-11-14 14:28:11 +0100298def webrtc_builder(
299 name,
300 recipe = "standalone",
301 dimensions = {},
Mirko Bonadei825f9c22020-07-01 11:52:39 +0200302 properties = {},
Artem Titarenkof6588b72019-11-14 14:28:11 +0100303 priority = 30,
304 execution_timeout = 2 * time.hour,
Mirko Bonadei825f9c22020-07-01 11:52:39 +0200305 goma_jobs = 80,
Artem Titarenkof6588b72019-11-14 14:28:11 +0100306 **kwargs):
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100307 dimensions = merge_dicts({"cpu": "x86-64"}, dimensions)
308
Mirko Bonadei825f9c22020-07-01 11:52:39 +0200309 goma_additional_params = {"$build/goma": {"jobs": goma_jobs}}
310
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100311 return luci.builder(
312 name = name,
Oleh Prypin705b6a62019-04-03 23:10:51 +0200313 executable = recipe,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100314 dimensions = {k: v for k, v in dimensions.items() if v != None},
Mirko Bonadei825f9c22020-07-01 11:52:39 +0200315 properties = merge_dicts(properties, goma_additional_params),
Artem Titarenkof6588b72019-11-14 14:28:11 +0100316 execution_timeout = execution_timeout,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100317 priority = priority,
318 build_numbers = True,
319 swarming_tags = ["vpython:native-python-wrapper"],
320 **kwargs
321 )
322
323def ci_builder(
324 name,
325 ci_cat,
326 perf_cat = None,
327 fyi_cat = None,
328 properties = {},
329 dimensions = {},
330 prioritized = False,
Oleh Prypinf5432c52019-03-19 15:10:37 +0100331 enabled = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100332 **kwargs):
Oleh Prypine5677672019-04-25 12:21:35 +0200333 """Add a post-submit builder.
334
335 Args:
336 name: builder name (str).
337 ci_cat: the category + name for the /ci/ console, or None to omit from the console.
338 perf_cat: the category + name for the /perf/ console, or None to omit from the console.
339 fyi_cat: the category + name for the /fyi/ console, or None to omit from the console.
340 properties: dict of properties to pass to the recipe (on top of the default ones).
341 dimensions: dict of Swarming dimensions (strings) to search machines by.
342 prioritized: True to make this builder have a higher priority and never batch builds.
343 enabled: False to exclude this builder from consoles and failure notifications.
344 **kwargs: Pass on to webrtc_builder / luci.builder.
345 Returns:
346 A luci.builder.
347
348 Notifications are also disabled if a builder is not on either of /ci/ or /perf/ consoles.
349 """
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100350 if prioritized:
351 kwargs["triggering_policy"] = scheduler.greedy_batching(
352 max_batch_size = 1,
353 max_concurrent_invocations = 3,
354 )
355 kwargs["priority"] = 29
356
Oleh Prypinf5432c52019-03-19 15:10:37 +0100357 if enabled:
358 add_milo(name, {"ci": ci_cat, "perf": perf_cat, "fyi": fyi_cat})
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200359 if ci_cat:
360 lkgr_builders[name] = True
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100361 return webrtc_builder(
362 name = name,
363 properties = merge_dicts({"mastername": "client.webrtc"}, properties),
364 dimensions = merge_dicts({"pool": "luci.webrtc.ci"}, dimensions),
365 bucket = "ci",
366 service_account = "webrtc-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
Oleh Prypinf5432c52019-03-19 15:10:37 +0100367 triggered_by = ["webrtc-gitiles-trigger-master"] if enabled else None,
Oleh Prypin71d17422019-03-28 08:43:16 +0100368 repo = WEBRTC_GIT,
Owen Rodley6b721ba2020-05-26 10:04:20 +1000369 notifies = ["ci_notifier", "webrtc_tree_closer"] if enabled and (ci_cat or perf_cat) else None,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100370 **kwargs
371 )
372
373def try_builder(
374 name,
375 try_cat = True,
376 fyi_cat = None,
377 properties = {},
378 dimensions = {},
379 cq = {},
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100380 branch_cq = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100381 **kwargs):
Oleh Prypine5677672019-04-25 12:21:35 +0200382 """Add a pre-submit builder.
383
384 Args:
385 name: builder name (str).
386 try_cat: boolean, whether to include this builder in the /try/ console. See also: `add_milo`.
387 fyi_cat: the category + name for the /fyi/ console, or None to omit from the console.
388 properties: dict of properties to pass to the recipe (on top of the default ones).
389 dimensions: dict of Swarming dimensions (strings) to search machines by.
390 cq: None to exclude this from all commit queues, or a dict of kwargs for cq_tryjob_verifier.
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100391 branch_cq: False to exclude this builder just from the release-branch CQ.
Oleh Prypine5677672019-04-25 12:21:35 +0200392 **kwargs: Pass on to webrtc_builder / luci.builder.
393 Returns:
394 A luci.builder.
395 """
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100396 add_milo(name, {"try": try_cat, "fyi": fyi_cat})
397 if cq != None:
398 luci.cq_tryjob_verifier(
399 builder = name,
400 cq_group = "cq",
401 **cq
402 )
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100403 if branch_cq:
404 luci.cq_tryjob_verifier(
405 builder = name,
406 cq_group = "cq_branch",
407 **cq
408 )
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100409
410 return webrtc_builder(
411 name = name,
412 properties = merge_dicts({"mastername": "tryserver.webrtc"}, properties),
413 dimensions = merge_dicts({"pool": "luci.webrtc.try"}, dimensions),
414 bucket = "try",
415 service_account = "webrtc-try-builder@chops-service-accounts.iam.gserviceaccount.com",
416 **kwargs
417 )
418
419def perf_builder(
420 name,
421 perf_cat,
422 recipe = "standalone",
423 properties = {},
424 dimensions = {},
425 **kwargs):
426 add_milo(name, {"perf": perf_cat})
427 return webrtc_builder(
428 name = name,
429 recipe = recipe,
430 properties = merge_dicts({"mastername": "client.webrtc.perf"}, properties),
431 dimensions = merge_dicts({"pool": "luci.webrtc.perf", "os": "Linux", "cpu": None}, dimensions),
432 bucket = "perf",
433 service_account = "webrtc-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
434 # log_base of 1.7 means:
435 # when there are P pending builds, LUCI will batch the first B builds.
436 # P: 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 ...
437 # B: 1 1 2 2 3 3 3 3 4 4 4 4 4 4 5 ...
Jeremy Leconteea2016b2020-07-01 09:47:22 +0200438 triggering_policy = scheduler.logarithmic_batching(log_base = 1.7),
Artem Titarenkof6588b72019-11-14 14:28:11 +0100439 execution_timeout = 3 * time.hour,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100440 **kwargs
441 )
442
Mirko Bonadei12643772020-03-27 13:16:35 +0100443def android_perf_builder(
444 name,
445 perf_cat,
446 recipe = "standalone",
447 properties = {},
448 dimensions = {},
449 **kwargs):
450 return perf_builder(
451 name = name,
452 perf_cat = perf_cat,
453 recipe = recipe,
454 properties = merge_dicts(GOMA_BACKEND_WEBRTC_RBE_PROD, properties),
455 dimensions = dimensions,
456 **kwargs
457 )
458
459def linux_perf_builder(
460 name,
461 perf_cat,
462 recipe = "standalone",
463 properties = {},
464 dimensions = {},
465 **kwargs):
466 return perf_builder(
467 name = name,
468 perf_cat = perf_cat,
469 recipe = recipe,
470 properties = merge_dicts(GOMA_BACKEND_WEBRTC_RBE_PROD, properties),
471 dimensions = dimensions,
472 **kwargs
473 )
474
475def mac_perf_builder(
476 name,
477 perf_cat,
478 recipe = "standalone",
479 properties = {},
480 dimensions = {},
481 **kwargs):
482 return perf_builder(
483 name = name,
484 perf_cat = perf_cat,
485 recipe = recipe,
486 properties = merge_dicts(GOMA_BACKEND_WEBRTC_RBE_PROD, properties),
487 dimensions = dimensions,
488 **kwargs
489 )
490
491def win_perf_builder(
492 name,
493 perf_cat,
494 recipe = "standalone",
495 properties = {},
496 dimensions = {},
497 **kwargs):
498 return perf_builder(
499 name = name,
500 perf_cat = perf_cat,
501 recipe = recipe,
502 properties = merge_dicts(GOMA_BACKEND_RBE_ATS_PROD, properties),
503 dimensions = dimensions,
504 **kwargs
505 )
506
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100507def cron_builder(name, dimensions = {}, **kwargs):
508 add_milo(name, {"cron": True})
509 return webrtc_builder(
510 name = name,
511 dimensions = merge_dicts({"pool": "luci.webrtc.cron", "os": "Linux"}, dimensions),
512 bucket = "cron",
513 service_account = "chromium-webrtc-autoroll@webrtc-ci.iam.gserviceaccount.com",
Oleh Prypin71d17422019-03-28 08:43:16 +0100514 repo = WEBRTC_GIT,
515 notifies = ["cron_notifier"],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100516 **kwargs
517 )
518
519# Helpers:
520
521def merge_dicts(a, b):
522 """Return the result of merging two dicts.
523
524 If matching values are both dicts or both lists, they will be merged (non-recursively).
525
526 Args:
527 a: first dict.
528 b: second dict (takes priority).
529 Returns:
530 Merged dict.
531 """
532 a = dict(a)
533 for k, bv in b.items():
534 av = a.get(k)
535 if type(av) == "dict" and type(bv) == "dict":
536 a[k] = dict(av)
537 a[k].update(bv)
538 elif type(av) == "list" and type(bv) == "list":
539 a[k] = av + bv
540 else:
541 a[k] = bv
542 return a
543
544def normal_builder_factory(**common_kwargs):
545 def builder(*args, **kwargs):
546 return ci_builder(*args, **merge_dicts(common_kwargs, kwargs))
547
548 def try_job(*args, **kwargs):
549 return try_builder(*args, **merge_dicts(common_kwargs, kwargs))
550
551 return builder, try_job
552
553# Mixins:
554
555linux_builder, linux_try_job = normal_builder_factory(
556 dimensions = {"os": "Linux", "inside_docker": "0"},
Mirko Bonadei32ca4862020-03-11 10:39:45 +0100557 properties = GOMA_BACKEND_WEBRTC_RBE_PROD,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100558)
559
560android_builder, android_try_job = normal_builder_factory(
561 dimensions = {"os": "Linux"},
Mirko Bonadei32ca4862020-03-11 10:39:45 +0100562 properties = GOMA_BACKEND_WEBRTC_RBE_PROD,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100563)
564
565win_builder, win_try_job = normal_builder_factory(
566 dimensions = {"os": "Windows"},
Patrik Höglund3916a972019-12-04 13:19:38 +0100567 properties = GOMA_BACKEND_RBE_ATS_PROD,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100568)
569
570mac_builder, mac_try_job = normal_builder_factory(
571 dimensions = {"os": "Mac"},
Mirko Bonadei32ca4862020-03-11 10:39:45 +0100572 properties = GOMA_BACKEND_WEBRTC_RBE_PROD,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100573)
574
575ios_builder, ios_try_job = normal_builder_factory(
Patrik Höglundd5dcd0e2020-02-19 21:50:39 +0100576 dimensions = {"os": "Mac-10.14"},
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100577 recipe = "ios",
Mirko Bonadei84360ca2020-03-12 08:35:48 +0100578 properties = GOMA_BACKEND_WEBRTC_IOS_RBE_PROD,
579 caches = [swarming.cache("osx_sdk")],
580)
581
582# Some iOS builders don't use goma because they build using
583# `use_xcode_clang=true` which is not supported by goma.
584# https://ci.chromium.org/p/webrtc/builders/ci/iOS%20API%20Framework%20Builder
585# https://ci.chromium.org/p/webrtc/builders/try/ios_api_framework
586ios_builder_no_goma, ios_try_job_no_goma = normal_builder_factory(
587 dimensions = {"os": "Mac-10.14"},
588 recipe = "ios",
Patrik Höglundb681c162020-02-21 07:34:53 +0100589 properties = {"$depot_tools/osx_sdk": {"sdk_version": "11c29"}},
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100590 caches = [swarming.cache("osx_sdk")],
591)
592
593# Actual builder configuration:
594
595android_builder("Android32 (M Nexus5X)(dbg)", "Android|arm|dbg")
596android_try_job("android_compile_arm_dbg", cq = None)
597android_try_job("android_arm_dbg")
598android_builder("Android32 (M Nexus5X)", "Android|arm|rel")
599android_try_job("android_arm_rel")
600android_builder("Android32 Builder arm", "Android|arm|size", perf_cat = "Android|arm|Builder|", prioritized = True)
601android_try_job("android_compile_arm_rel")
Mirko Bonadei12643772020-03-27 13:16:35 +0100602android_perf_builder("Perf Android32 (K Nexus5)", "Android|arm|Tester|K Nexus5", triggered_by = ["Android32 Builder arm"])
603android_perf_builder("Perf Android32 (M AOSP Nexus6)", "Android|arm|Tester|M AOSP Nexus6", triggered_by = ["Android32 Builder arm"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100604android_try_job("android_compile_arm64_dbg", cq = None)
605android_try_job("android_arm64_dbg", cq = None)
606android_builder("Android64 (M Nexus5X)", "Android|arm64|rel")
607android_try_job("android_arm64_rel")
608android_builder("Android64 Builder arm64", "Android|arm64|size", perf_cat = "Android|arm64|Builder|", prioritized = True)
Mirko Bonadei12643772020-03-27 13:16:35 +0100609android_perf_builder("Perf Android64 (M Nexus5X)", "Android|arm64|Tester|M Nexus5X", triggered_by = ["Android64 Builder arm64"])
610android_perf_builder("Perf Android64 (O Pixel2)", "Android|arm64|Tester|O Pixel2", triggered_by = ["Android64 Builder arm64"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100611android_try_job("android_compile_arm64_rel")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100612android_builder("Android64 Builder x64 (dbg)", "Android|x64|dbg")
613android_try_job("android_compile_x64_dbg")
614android_try_job("android_compile_x64_rel", cq = None)
615android_builder("Android32 Builder x86 (dbg)", "Android|x86|dbg")
616android_try_job("android_compile_x86_dbg")
617android_builder("Android32 Builder x86", "Android|x86|rel")
618android_try_job("android_compile_x86_rel")
619android_builder("Android32 (more configs)", "Android|arm|more", recipe = "more_configs")
620android_try_job("android_arm_more_configs", recipe = "more_configs")
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100621android_try_job("android_chromium_compile", recipe = "chromium_trybot", branch_cq = False)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100622
623ios_builder("iOS32 Debug", "iOS|arm|dbg")
624ios_try_job("ios_compile_arm_dbg")
625ios_builder("iOS32 Release", "iOS|arm|rel")
626ios_try_job("ios_compile_arm_rel")
627ios_builder("iOS64 Debug", "iOS|arm64|dbg")
628ios_try_job("ios_compile_arm64_dbg")
629ios_builder("iOS64 Release", "iOS|arm64|rel")
630ios_try_job("ios_compile_arm64_rel")
631ios_builder("iOS64 Sim Debug (iOS 10.0)", "iOS|x64|10")
Patrik Höglund17bed672020-02-20 13:31:36 +0000632ios_try_job("ios_sim_x64_dbg_ios10")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100633ios_builder("iOS64 Sim Debug (iOS 11)", "iOS|x64|11")
Patrik Höglund17bed672020-02-20 13:31:36 +0000634ios_try_job("ios_sim_x64_dbg_ios11")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100635ios_builder("iOS64 Sim Debug (iOS 12)", "iOS|x64|12")
Patrik Höglund17bed672020-02-20 13:31:36 +0000636ios_try_job("ios_sim_x64_dbg_ios12")
Mirko Bonadei84360ca2020-03-12 08:35:48 +0100637ios_builder_no_goma("iOS API Framework Builder", "iOS|fat|size", recipe = "ios_api_framework", prioritized = True)
Jeremy Leconteea2016b2020-07-01 09:47:22 +0200638ios_try_job_no_goma("ios_api_framework", recipe = "ios_api_framework", cq = None)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100639
640linux_builder("Linux32 Debug", "Linux|x86|dbg")
641linux_try_job("linux_x86_dbg")
642linux_builder("Linux32 Release", "Linux|x86|rel")
643linux_try_job("linux_x86_rel")
644linux_builder("Linux64 Debug", "Linux|x64|dbg")
645linux_try_job("linux_dbg", cq = None)
646linux_try_job("linux_compile_dbg")
647linux_builder("Linux64 Release", "Linux|x64|rel")
648linux_try_job("linux_rel")
649linux_builder("Linux64 Builder", "Linux|x64|size", perf_cat = "Linux|x64|Builder|", prioritized = True)
650linux_try_job("linux_compile_rel")
Mirko Bonadei12643772020-03-27 13:16:35 +0100651linux_perf_builder("Perf Linux Trusty", "Linux|x64|Tester|Trusty", triggered_by = ["Linux64 Builder"])
652linux_perf_builder("Perf Linux Xenial", None, triggered_by = ["Linux64 Builder"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100653linux_builder("Linux32 Debug (ARM)", "Linux|arm|dbg")
654linux_try_job("linux_compile_arm_dbg")
655linux_builder("Linux32 Release (ARM)", "Linux|arm|rel")
656linux_try_job("linux_compile_arm_rel")
657linux_builder("Linux64 Debug (ARM)", "Linux|arm64|dbg")
658linux_try_job("linux_compile_arm64_dbg")
659linux_builder("Linux64 Release (ARM)", "Linux|arm64|rel")
660linux_try_job("linux_compile_arm64_rel")
661linux_builder("Linux64 Release (GCC)", "Linux|x64|gcc")
662linux_try_job("linux_compile_gcc_rel")
663linux_builder("Linux Asan", "Linux|x64|asan")
664linux_try_job("linux_asan")
665linux_builder("Linux MSan", "Linux|x64|msan")
666linux_try_job("linux_msan")
667linux_builder("Linux Tsan v2", "Linux|x64|tsan")
668linux_try_job("linux_tsan2")
669linux_builder("Linux UBSan", "Linux|x64|ubsan")
670linux_try_job("linux_ubsan")
671linux_builder("Linux UBSan vptr", "Linux|x64|ubsan")
672linux_try_job("linux_ubsan_vptr")
673linux_builder("Linux64 Release (Libfuzzer)", "Linux|x64|fuzz", recipe = "libfuzzer")
674linux_try_job("linux_libfuzzer_rel", recipe = "libfuzzer")
675linux_builder("Linux (more configs)", "Linux|x64|more", recipe = "more_configs")
676linux_try_job("linux_more_configs", recipe = "more_configs")
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100677linux_try_job("linux_chromium_compile", recipe = "chromium_trybot", branch_cq = False)
Mirko Bonadei40de50f2019-11-07 11:23:47 +0100678linux_try_job("linux_chromium_compile_dbg", recipe = "chromium_trybot", branch_cq = False)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100679
680mac_builder("Mac64 Debug", "Mac|x64|dbg")
681mac_try_job("mac_dbg", cq = None)
682mac_try_job("mac_compile_dbg")
683mac_builder("Mac64 Release", "Mac|x64|rel")
684mac_try_job("mac_rel")
685mac_try_job("mac_compile_rel", cq = None)
686mac_builder("Mac64 Builder", ci_cat = None, perf_cat = "Mac|x64|Builder|")
Mirko Bonadei12643772020-03-27 13:16:35 +0100687mac_perf_builder("Perf Mac 10.11", "Mac|x64|Tester|10.11", triggered_by = ["Mac64 Builder"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100688mac_builder("Mac Asan", "Mac|x64|asan")
689mac_try_job("mac_asan")
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100690mac_try_job("mac_chromium_compile", recipe = "chromium_trybot", dimensions = {"cores": "8"}, branch_cq = False)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100691
692win_builder("Win32 Debug", "Win MSVC|x86|dbg")
693win_try_job("win_x86_msvc_dbg")
694win_try_job("win_compile_x86_msvc_dbg", cq = None)
Oleh Prypin63041b82019-03-19 22:56:01 +0000695win_builder("Win32 Release", "Win MSVC|x86|rel")
696win_try_job("win_x86_msvc_rel")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100697win_try_job("win_compile_x86_msvc_rel", cq = None)
698win_builder("Win64 Debug", "Win MSVC|x64|dbg")
699win_try_job("win_x64_msvc_dbg")
700win_try_job("win_compile_x64_msvc_dbg", cq = None)
701win_builder("Win64 Release", "Win MSVC|x64|rel")
702win_try_job("win_x64_msvc_rel")
703win_try_job("win_compile_x64_msvc_rel", cq = None)
704win_builder("Win32 Debug (Clang)", "Win Clang|x86|dbg")
705win_try_job("win_x86_clang_dbg", cq = None)
706win_try_job("win_compile_x86_clang_dbg")
707win_builder("Win32 Release (Clang)", "Win Clang|x86|rel")
708win_try_job("win_x86_clang_rel")
709win_try_job("win_compile_x86_clang_rel", cq = None)
710win_builder("Win32 Builder (Clang)", ci_cat = None, perf_cat = "Win|x86|Builder|")
Mirko Bonadei12643772020-03-27 13:16:35 +0100711win_perf_builder("Perf Win7", "Win|x86|Tester|7", triggered_by = ["Win32 Builder (Clang)"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100712win_builder("Win64 Debug (Clang)", "Win Clang|x64|dbg")
713win_try_job("win_x64_clang_dbg", cq = None)
714win_try_job("win_x64_clang_dbg_win8", cq = None)
715win_try_job("win_x64_clang_dbg_win10", cq = None)
716win_try_job("win_compile_x64_clang_dbg")
717win_builder("Win64 Release (Clang)", "Win Clang|x64|rel")
718win_try_job("win_x64_clang_rel", cq = None)
719win_try_job("win_compile_x64_clang_rel")
Mirko Bonadeiafbe33d2019-05-31 09:27:54 +0200720win_builder("Win64 ASan", "Win Clang|x64|asan")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100721win_try_job("win_asan")
722win_builder("Win64 UWP", ci_cat = None, fyi_cat = "")
723win_try_job("win_x64_uwp", cq = None, try_cat = None, fyi_cat = "")
724win_builder("Win (more configs)", "Win Clang|x86|more", recipe = "more_configs")
725win_try_job("win_x86_more_configs", recipe = "more_configs")
Mirko Bonadei825f9c22020-07-01 11:52:39 +0200726win_try_job("win_chromium_compile", recipe = "chromium_trybot", branch_cq = False, goma_jobs = 150)
727win_try_job("win_chromium_compile_dbg", recipe = "chromium_trybot", branch_cq = False, goma_jobs = 150)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100728
729linux_try_job(
730 "presubmit",
731 recipe = "run_presubmit",
732 properties = {"repo_name": "webrtc", "runhooks": True},
733 priority = 28,
734 cq = {"disable_reuse": True},
735)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100736
737cron_builder(
738 "Auto-roll - WebRTC DEPS",
739 recipe = "auto_roll_webrtc_deps",
Oleh Prypindc68a722019-06-25 10:43:13 +0200740 schedule = "0 */2 * * *", # Every 2 hours.
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100741)
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200742
743lkgr_config = {
744 "project": "webrtc",
745 "source_url": WEBRTC_GIT,
746 "status_url": "https://webrtc-status.appspot.com",
747 "allowed_lag": 300, # hours
748 "allowed_gap": 12, # commits behind
749 "error_recipients": "webrtc-troopers-robots@google.com",
Michael Achenbach09dd7dc2019-07-26 15:26:11 +0200750 "buckets": {
751 "webrtc/ci": {
752 # bucket alias: luci.webrtc.ci
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200753 "builders": sorted(lkgr_builders),
754 },
Michael Achenbach09dd7dc2019-07-26 15:26:11 +0200755 "chromium/webrtc.fyi": {
756 # bucket alias: luci.chromium.webrtc.fyi
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200757 "builders": [
758 "WebRTC Chromium FYI Android Builder (dbg)",
759 "WebRTC Chromium FYI Android Builder ARM64 (dbg)",
760 "WebRTC Chromium FYI Android Builder",
Mirko Bonadei04fe32f2020-05-01 00:45:58 +0200761 "WebRTC Chromium FYI Android Tests (dbg) (L Nexus5)",
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200762 "WebRTC Chromium FYI Android Tests (dbg) (M Nexus5X)",
763 "WebRTC Chromium FYI Linux Builder (dbg)",
764 "WebRTC Chromium FYI Linux Builder",
765 "WebRTC Chromium FYI Linux Tester",
766 "WebRTC Chromium FYI Mac Builder (dbg)",
767 "WebRTC Chromium FYI Mac Builder",
768 "WebRTC Chromium FYI Mac Tester",
769 "WebRTC Chromium FYI Win Builder (dbg)",
770 "WebRTC Chromium FYI Win Builder",
771 "WebRTC Chromium FYI Win10 Tester",
772 "WebRTC Chromium FYI Win7 Tester",
773 "WebRTC Chromium FYI Win8 Tester",
774 "WebRTC Chromium FYI ios-device",
Mirko Bonadei37071f62020-02-15 10:42:49 +0100775 "WebRTC Chromium FYI ios-simulator",
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200776 ],
777 },
778 },
779}
780
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100781cron_builder(
782 "WebRTC lkgr finder",
783 recipe = "lkgr_finder",
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200784 properties = {
785 "project": "webrtc",
786 "repo": WEBRTC_GIT,
787 "ref": "refs/heads/lkgr",
788 "lkgr_status_gs_path": "chromium-webrtc/lkgr-status",
789 "config": lkgr_config,
790 },
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100791 schedule = "*/10 * * * *", # Every 10 minutes.
792)