blob: 2cbb3668e424f3cae45dd39a0c0cf2fbab8aec6f [file] [log] [blame]
Oleh Prypin1a0593f2019-03-11 09:43:28 +01001#!/usr/bin/env lucicfg
2
3# Copyright (c) 2019 The WebRTC project authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7# https://chromium.googlesource.com/infra/luci/luci-go/+/master/lucicfg/doc/
8
Jeremy Leconteea2016b2020-07-01 09:47:22 +02009"""LUCI project configuration for WebRTC CQ and CI."""
10
11lucicfg.check_version("1.15.0")
Mirko Bonadei9189d9c2020-06-09 21:54:18 +020012
Oleh Prypin1a0593f2019-03-11 09:43:28 +010013WEBRTC_GIT = "https://webrtc.googlesource.com/src"
14WEBRTC_GERRIT = "https://webrtc-review.googlesource.com/src"
15
Mirko Bonadei32ca4862020-03-11 10:39:45 +010016GOMA_BACKEND_WEBRTC_RBE_PROD = {
Mirko Bonadei1e905342020-03-04 09:39:42 +010017 "$build/goma": {
18 "server_host": "goma.chromium.org",
Jeremy Leconteea2016b2020-07-01 09:47:22 +020019 "use_luci_auth": True,
20 },
Mirko Bonadei1e905342020-03-04 09:39:42 +010021}
22
Mirko Bonadei84360ca2020-03-12 08:35:48 +010023GOMA_BACKEND_WEBRTC_IOS_RBE_PROD = {
24 "$build/goma": {
25 "server_host": "goma.chromium.org",
Jeremy Leconteea2016b2020-07-01 09:47:22 +020026 "use_luci_auth": True,
Mirko Bonadei84360ca2020-03-12 08:35:48 +010027 },
28 "$depot_tools/osx_sdk": {"sdk_version": "11c29"},
29}
30
Patrik Höglund3916a972019-12-04 13:19:38 +010031GOMA_BACKEND_RBE_ATS_PROD = {
32 "$build/goma": {
33 "server_host": "goma.chromium.org",
Mirko Bonadei2688eca2020-03-25 09:15:54 +010034 "use_luci_auth": True,
Jeremy Leconteea2016b2020-07-01 09:47:22 +020035 "enable_ats": True,
36 },
Patrik Höglund3916a972019-12-04 13:19:38 +010037}
38
Oleh Prypin1a0593f2019-03-11 09:43:28 +010039# Top-level configs:
40
Owen Rodley6b721ba2020-05-26 10:04:20 +100041# Enable luci.tree_closer.
42lucicfg.enable_experiment("crbug.com/1054172")
43
Oleh Prypin1a0593f2019-03-11 09:43:28 +010044lucicfg.config(
45 config_dir = ".",
46 tracked_files = [
47 "commit-queue.cfg",
48 "cr-buildbucket.cfg",
49 "luci-logdog.cfg",
50 "luci-milo.cfg",
Oleh Prypin71d17422019-03-28 08:43:16 +010051 "luci-notify.cfg",
Oleh Prypin705b6a62019-04-03 23:10:51 +020052 "luci-notify/**/*",
Oleh Prypin1a0593f2019-03-11 09:43:28 +010053 "luci-scheduler.cfg",
54 "project.cfg",
55 ],
Jeremy Leconteea2016b2020-07-01 09:47:22 +020056 lint_checks = ["default"],
Oleh Prypin1a0593f2019-03-11 09:43:28 +010057)
58
59luci.project(
60 name = "webrtc",
61 buildbucket = "cr-buildbucket.appspot.com",
62 logdog = "luci-logdog.appspot.com",
63 milo = "luci-milo.appspot.com",
Oleh Prypin71d17422019-03-28 08:43:16 +010064 notify = "luci-notify.appspot.com",
Oleh Prypin1a0593f2019-03-11 09:43:28 +010065 scheduler = "luci-scheduler.appspot.com",
66 swarming = "chromium-swarm.appspot.com",
67 acls = [
68 acl.entry(
69 [acl.BUILDBUCKET_READER, acl.LOGDOG_READER, acl.PROJECT_CONFIGS_READER, acl.SCHEDULER_READER],
70 groups = ["all"],
71 ),
72 acl.entry(acl.LOGDOG_WRITER, groups = ["luci-logdog-chromium-writers"]),
73 acl.entry(acl.SCHEDULER_OWNER, groups = ["project-webrtc-admins"]),
74 ],
75)
76
77luci.logdog(
78 gs_bucket = "chromium-luci-logdog",
79)
80
81luci.milo(
82 logo = "https://storage.googleapis.com/chrome-infra/webrtc-logo-vert-retro-255x305.png",
83)
84
Owen Rodleya55c38e2020-06-15 11:16:10 +100085luci.notify(tree_closing_enabled = True)
86
Oleh Prypin1a0593f2019-03-11 09:43:28 +010087luci.cq(
88 status_host = "chromium-cq-status.appspot.com",
89 submit_max_burst = 1,
90 submit_burst_delay = 1 * time.minute,
91)
92
93luci.gitiles_poller(
94 name = "webrtc-gitiles-trigger-master",
95 bucket = "ci",
96 repo = WEBRTC_GIT,
97)
98
99# Bucket definitions:
100
101luci.bucket(
102 name = "try",
103 acls = [
104 acl.entry(acl.BUILDBUCKET_TRIGGERER, groups = [
105 "service-account-cq",
106 "project-webrtc-tryjob-access",
107 ]),
108 ],
109)
110
111luci.bucket(
112 name = "ci",
113 acls = [
Oleh Prypinf35939d2019-05-03 20:42:38 +0200114 acl.entry(acl.BUILDBUCKET_TRIGGERER, groups = [
115 "project-webrtc-ci-schedulers",
116 ], users = [
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100117 "luci-scheduler@appspot.gserviceaccount.com",
118 ]),
119 ],
120)
121
122luci.bucket(
123 name = "perf",
124 acls = [
125 acl.entry(acl.BUILDBUCKET_TRIGGERER, users = [
126 "luci-scheduler@appspot.gserviceaccount.com",
127 "webrtc-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
128 ]),
129 ],
130)
131
132luci.bucket(
133 name = "cron",
134 acls = [
135 acl.entry(acl.BUILDBUCKET_TRIGGERER, users = [
136 "luci-scheduler@appspot.gserviceaccount.com",
137 ]),
138 ],
139)
140
141# Commit queue definitions:
142
143luci.cq_group(
144 name = "cq",
145 tree_status_host = "webrtc-status.appspot.com",
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100146 watch = [cq.refset(repo = WEBRTC_GERRIT, refs = ["refs/heads/master"])],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100147 acls = [
148 acl.entry(acl.CQ_COMMITTER, groups = ["project-webrtc-committers"]),
149 acl.entry(acl.CQ_DRY_RUNNER, groups = ["project-webrtc-tryjob-access"]),
150 ],
151 retry_config = cq.RETRY_ALL_FAILURES,
Oleh Prypin5416f2d2019-08-02 09:43:15 +0200152 cancel_stale_tryjobs = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100153)
154
155luci.cq_group(
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100156 name = "cq_branch",
157 watch = [cq.refset(repo = WEBRTC_GERRIT, refs = ["refs/branch-heads/.+"])],
158 acls = [
159 acl.entry(acl.CQ_COMMITTER, groups = ["project-webrtc-committers"]),
160 acl.entry(acl.CQ_DRY_RUNNER, groups = ["project-webrtc-tryjob-access"]),
161 ],
162 retry_config = cq.RETRY_ALL_FAILURES,
Oleh Prypin5416f2d2019-08-02 09:43:15 +0200163 cancel_stale_tryjobs = True,
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100164)
165
166luci.cq_group(
167 name = "cq_infra",
168 watch = [cq.refset(repo = WEBRTC_GERRIT, refs = ["refs/heads/infra/config"])],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100169 acls = [
170 acl.entry(acl.CQ_COMMITTER, groups = ["project-webrtc-admins"]),
171 acl.entry(acl.CQ_DRY_RUNNER, groups = ["project-webrtc-tryjob-access"]),
172 ],
173 retry_config = cq.RETRY_ALL_FAILURES,
Oleh Prypin5416f2d2019-08-02 09:43:15 +0200174 cancel_stale_tryjobs = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100175)
176
177luci.cq_tryjob_verifier(
178 builder = "presubmit",
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100179 cq_group = "cq_infra",
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100180)
181
Oleh Prypine56cd9b2019-05-29 21:14:31 +0200182luci.cq_tryjob_verifier(
183 builder = "webrtc-internal:g3.webrtc-internal.try/internal_compile_lite",
184 owner_whitelist = ["project-webrtc-internal-tryjob-access"],
185 cq_group = "cq",
186)
187
Oleh Prypin71d17422019-03-28 08:43:16 +0100188# Notifier definitions:
189
190luci.notifier(
191 name = "ci_notifier",
192 on_failure = True,
193 notify_emails = ["webrtc-sheriffs-robots@google.com"],
Oleh Prypin705b6a62019-04-03 23:10:51 +0200194 template = luci.notifier_template(
195 name = "ci",
196 body = io.read_file("luci-notify/email-templates/ci.template"),
197 ),
Oleh Prypin71d17422019-03-28 08:43:16 +0100198)
199
200luci.notifier(
201 name = "cron_notifier",
202 on_failure = True,
203 notify_emails = ["webrtc-troopers-robots@google.com"],
Oleh Prypin705b6a62019-04-03 23:10:51 +0200204 template = luci.notifier_template(
205 name = "cron",
206 body = io.read_file("luci-notify/email-templates/cron.template"),
207 ),
Oleh Prypin71d17422019-03-28 08:43:16 +0100208)
209
Owen Rodley6b721ba2020-05-26 10:04:20 +1000210# Tree closer definitions:
211
212luci.tree_closer(
213 name = "webrtc_tree_closer",
214 tree_status_host = "webrtc-status.appspot.com",
215 template = luci.notifier_template(
216 name = "status",
217 body = io.read_file("luci-notify/email-templates/status.template"),
218 ),
219 # TODO: These step filters are copied verbatim from Gatekeeper, for testing
220 # that LUCI-Notify would take the exact same actions. Once we've switched
221 # over, this should be updated - several of these steps don't exist in
222 # WebRTC recipes.
223 failed_step_regexp = [
224 "bot_update",
225 "compile",
226 "gclient runhooks",
227 "runhooks",
228 "update",
229 "extract build",
230 "cleanup_temp",
231 "taskkill",
232 "compile",
233 "gn",
234 ],
235 failed_step_regexp_exclude = ".*\\(experimental\\).*",
236)
237
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100238# Recipe definitions:
239
240def recipe(recipe, pkg = "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"):
241 return luci.recipe(
242 name = recipe.split("/")[-1],
243 cipd_package = pkg,
244 cipd_version = "refs/heads/master",
245 recipe = recipe,
246 )
247
248recipe("chromium_trybot")
249recipe("run_presubmit")
250recipe("webrtc/auto_roll_webrtc_deps")
251recipe("webrtc/ios")
252recipe("webrtc/ios_api_framework")
253recipe("webrtc/libfuzzer")
254recipe("webrtc/more_configs")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100255recipe("webrtc/standalone")
256recipe("lkgr_finder", pkg = "infra/recipe_bundles/chromium.googlesource.com/infra/infra")
257
258# Console definitions:
259
260luci.console_view(name = "ci", title = "Main", repo = WEBRTC_GIT, header = "console-header.textpb")
261luci.console_view(name = "perf", title = "Perf", repo = WEBRTC_GIT, header = "console-header.textpb")
262luci.list_view(name = "cron", title = "Cron")
263luci.console_view(name = "fyi", title = "FYI", repo = WEBRTC_GIT, header = "console-header.textpb")
264luci.list_view(name = "try", title = "Tryserver")
265
266def add_milo(builder, views):
267 """Add Milo console entries for the builder.
268
269 Args:
270 builder: builder name (str).
271 views: dict where keys are names of consoles and values are either a
272 category for the console (str, pipe-separated) or True, which means
273 adding to a list view rather than a console.
274 """
275 for view_name, category in views.items():
276 if category == None:
277 continue
278 elif type(category) == "string":
279 category, _, short_name = category.rpartition("|")
280 luci.console_view_entry(
281 console_view = view_name,
282 builder = builder,
283 category = category or None,
284 short_name = short_name or None,
285 )
286 elif category == True:
287 luci.list_view_entry(
288 list_view = view_name,
289 builder = builder,
290 )
291 else:
292 fail("Unexpected value for category: %r" % category)
293
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200294lkgr_builders = {}
295
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100296# Builder-defining functions:
297
Artem Titarenkof6588b72019-11-14 14:28:11 +0100298def webrtc_builder(
299 name,
300 recipe = "standalone",
301 dimensions = {},
Mirko Bonadei825f9c22020-07-01 11:52:39 +0200302 properties = {},
Artem Titarenkof6588b72019-11-14 14:28:11 +0100303 priority = 30,
304 execution_timeout = 2 * time.hour,
Mirko Bonadeib7ef4602020-07-01 13:28:26 +0200305 goma_jobs = None,
Artem Titarenkof6588b72019-11-14 14:28:11 +0100306 **kwargs):
Mirko Bonadeib7ef4602020-07-01 13:28:26 +0200307 """WebRTC specific wrapper around luci.builder.
308
309 Args:
310 name: builder name (str).
311 recipe: string with the name of the recipe to run.
312 dimensions: dict of Swarming dimensions (strings) to search machines by.
313 properties: dict of properties to pass to the recipe (on top of the default ones).
314 priority: int [1-255] or None, indicating swarming task priority, lower is
315 more important. If None, defer the decision to Buildbucket service.
316 execution_timeout: int or None, how long to wait for a running build to finish before
317 forcefully aborting it and marking the build as timed out. If None,
318 defer the decision to Buildbucket service.
319 goma_jobs: int or None, number of jobs to be used by the builder. If None, defer the
320 decition to the goma module.
321 **kwargs: Pass on to webrtc_builder / luci.builder.
322 Returns:
323 A luci.builder.
324 """
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100325 dimensions = merge_dicts({"cpu": "x86-64"}, dimensions)
326
Mirko Bonadeib7ef4602020-07-01 13:28:26 +0200327 if goma_jobs != None:
328 goma_additional_params = {"$build/goma": {"jobs": goma_jobs}}
329 properties = merge_dicts(properties, goma_additional_params)
Mirko Bonadei825f9c22020-07-01 11:52:39 +0200330
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100331 return luci.builder(
332 name = name,
Oleh Prypin705b6a62019-04-03 23:10:51 +0200333 executable = recipe,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100334 dimensions = {k: v for k, v in dimensions.items() if v != None},
Mirko Bonadeib7ef4602020-07-01 13:28:26 +0200335 properties = properties,
Artem Titarenkof6588b72019-11-14 14:28:11 +0100336 execution_timeout = execution_timeout,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100337 priority = priority,
338 build_numbers = True,
339 swarming_tags = ["vpython:native-python-wrapper"],
340 **kwargs
341 )
342
343def ci_builder(
344 name,
345 ci_cat,
346 perf_cat = None,
347 fyi_cat = None,
348 properties = {},
349 dimensions = {},
350 prioritized = False,
Oleh Prypinf5432c52019-03-19 15:10:37 +0100351 enabled = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100352 **kwargs):
Oleh Prypine5677672019-04-25 12:21:35 +0200353 """Add a post-submit builder.
354
355 Args:
356 name: builder name (str).
357 ci_cat: the category + name for the /ci/ console, or None to omit from the console.
358 perf_cat: the category + name for the /perf/ console, or None to omit from the console.
359 fyi_cat: the category + name for the /fyi/ console, or None to omit from the console.
360 properties: dict of properties to pass to the recipe (on top of the default ones).
361 dimensions: dict of Swarming dimensions (strings) to search machines by.
362 prioritized: True to make this builder have a higher priority and never batch builds.
363 enabled: False to exclude this builder from consoles and failure notifications.
364 **kwargs: Pass on to webrtc_builder / luci.builder.
365 Returns:
366 A luci.builder.
367
368 Notifications are also disabled if a builder is not on either of /ci/ or /perf/ consoles.
369 """
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100370 if prioritized:
371 kwargs["triggering_policy"] = scheduler.greedy_batching(
372 max_batch_size = 1,
373 max_concurrent_invocations = 3,
374 )
375 kwargs["priority"] = 29
376
Oleh Prypinf5432c52019-03-19 15:10:37 +0100377 if enabled:
378 add_milo(name, {"ci": ci_cat, "perf": perf_cat, "fyi": fyi_cat})
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200379 if ci_cat:
380 lkgr_builders[name] = True
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100381 return webrtc_builder(
382 name = name,
383 properties = merge_dicts({"mastername": "client.webrtc"}, properties),
384 dimensions = merge_dicts({"pool": "luci.webrtc.ci"}, dimensions),
385 bucket = "ci",
386 service_account = "webrtc-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
Oleh Prypinf5432c52019-03-19 15:10:37 +0100387 triggered_by = ["webrtc-gitiles-trigger-master"] if enabled else None,
Oleh Prypin71d17422019-03-28 08:43:16 +0100388 repo = WEBRTC_GIT,
Owen Rodley6b721ba2020-05-26 10:04:20 +1000389 notifies = ["ci_notifier", "webrtc_tree_closer"] if enabled and (ci_cat or perf_cat) else None,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100390 **kwargs
391 )
392
393def try_builder(
394 name,
395 try_cat = True,
396 fyi_cat = None,
397 properties = {},
398 dimensions = {},
399 cq = {},
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100400 branch_cq = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100401 **kwargs):
Oleh Prypine5677672019-04-25 12:21:35 +0200402 """Add a pre-submit builder.
403
404 Args:
405 name: builder name (str).
406 try_cat: boolean, whether to include this builder in the /try/ console. See also: `add_milo`.
407 fyi_cat: the category + name for the /fyi/ console, or None to omit from the console.
408 properties: dict of properties to pass to the recipe (on top of the default ones).
409 dimensions: dict of Swarming dimensions (strings) to search machines by.
410 cq: None to exclude this from all commit queues, or a dict of kwargs for cq_tryjob_verifier.
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100411 branch_cq: False to exclude this builder just from the release-branch CQ.
Oleh Prypine5677672019-04-25 12:21:35 +0200412 **kwargs: Pass on to webrtc_builder / luci.builder.
413 Returns:
414 A luci.builder.
415 """
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100416 add_milo(name, {"try": try_cat, "fyi": fyi_cat})
417 if cq != None:
418 luci.cq_tryjob_verifier(
419 builder = name,
420 cq_group = "cq",
421 **cq
422 )
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100423 if branch_cq:
424 luci.cq_tryjob_verifier(
425 builder = name,
426 cq_group = "cq_branch",
427 **cq
428 )
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100429
430 return webrtc_builder(
431 name = name,
432 properties = merge_dicts({"mastername": "tryserver.webrtc"}, properties),
433 dimensions = merge_dicts({"pool": "luci.webrtc.try"}, dimensions),
434 bucket = "try",
435 service_account = "webrtc-try-builder@chops-service-accounts.iam.gserviceaccount.com",
436 **kwargs
437 )
438
439def perf_builder(
440 name,
441 perf_cat,
442 recipe = "standalone",
443 properties = {},
444 dimensions = {},
445 **kwargs):
446 add_milo(name, {"perf": perf_cat})
447 return webrtc_builder(
448 name = name,
449 recipe = recipe,
450 properties = merge_dicts({"mastername": "client.webrtc.perf"}, properties),
451 dimensions = merge_dicts({"pool": "luci.webrtc.perf", "os": "Linux", "cpu": None}, dimensions),
452 bucket = "perf",
453 service_account = "webrtc-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
454 # log_base of 1.7 means:
455 # when there are P pending builds, LUCI will batch the first B builds.
456 # P: 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 ...
457 # B: 1 1 2 2 3 3 3 3 4 4 4 4 4 4 5 ...
Jeremy Leconteea2016b2020-07-01 09:47:22 +0200458 triggering_policy = scheduler.logarithmic_batching(log_base = 1.7),
Artem Titarenkof6588b72019-11-14 14:28:11 +0100459 execution_timeout = 3 * time.hour,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100460 **kwargs
461 )
462
Mirko Bonadei12643772020-03-27 13:16:35 +0100463def android_perf_builder(
464 name,
465 perf_cat,
466 recipe = "standalone",
467 properties = {},
468 dimensions = {},
469 **kwargs):
470 return perf_builder(
471 name = name,
472 perf_cat = perf_cat,
473 recipe = recipe,
474 properties = merge_dicts(GOMA_BACKEND_WEBRTC_RBE_PROD, properties),
475 dimensions = dimensions,
476 **kwargs
477 )
478
479def linux_perf_builder(
480 name,
481 perf_cat,
482 recipe = "standalone",
483 properties = {},
484 dimensions = {},
485 **kwargs):
486 return perf_builder(
487 name = name,
488 perf_cat = perf_cat,
489 recipe = recipe,
490 properties = merge_dicts(GOMA_BACKEND_WEBRTC_RBE_PROD, properties),
491 dimensions = dimensions,
492 **kwargs
493 )
494
495def mac_perf_builder(
496 name,
497 perf_cat,
498 recipe = "standalone",
499 properties = {},
500 dimensions = {},
501 **kwargs):
502 return perf_builder(
503 name = name,
504 perf_cat = perf_cat,
505 recipe = recipe,
506 properties = merge_dicts(GOMA_BACKEND_WEBRTC_RBE_PROD, properties),
507 dimensions = dimensions,
508 **kwargs
509 )
510
511def win_perf_builder(
512 name,
513 perf_cat,
514 recipe = "standalone",
515 properties = {},
516 dimensions = {},
517 **kwargs):
518 return perf_builder(
519 name = name,
520 perf_cat = perf_cat,
521 recipe = recipe,
522 properties = merge_dicts(GOMA_BACKEND_RBE_ATS_PROD, properties),
523 dimensions = dimensions,
524 **kwargs
525 )
526
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100527def cron_builder(name, dimensions = {}, **kwargs):
528 add_milo(name, {"cron": True})
529 return webrtc_builder(
530 name = name,
531 dimensions = merge_dicts({"pool": "luci.webrtc.cron", "os": "Linux"}, dimensions),
532 bucket = "cron",
533 service_account = "chromium-webrtc-autoroll@webrtc-ci.iam.gserviceaccount.com",
Oleh Prypin71d17422019-03-28 08:43:16 +0100534 repo = WEBRTC_GIT,
535 notifies = ["cron_notifier"],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100536 **kwargs
537 )
538
539# Helpers:
540
541def merge_dicts(a, b):
542 """Return the result of merging two dicts.
543
544 If matching values are both dicts or both lists, they will be merged (non-recursively).
545
546 Args:
547 a: first dict.
548 b: second dict (takes priority).
549 Returns:
550 Merged dict.
551 """
552 a = dict(a)
553 for k, bv in b.items():
554 av = a.get(k)
555 if type(av) == "dict" and type(bv) == "dict":
556 a[k] = dict(av)
557 a[k].update(bv)
558 elif type(av) == "list" and type(bv) == "list":
559 a[k] = av + bv
560 else:
561 a[k] = bv
562 return a
563
564def normal_builder_factory(**common_kwargs):
565 def builder(*args, **kwargs):
566 return ci_builder(*args, **merge_dicts(common_kwargs, kwargs))
567
568 def try_job(*args, **kwargs):
569 return try_builder(*args, **merge_dicts(common_kwargs, kwargs))
570
571 return builder, try_job
572
573# Mixins:
574
575linux_builder, linux_try_job = normal_builder_factory(
576 dimensions = {"os": "Linux", "inside_docker": "0"},
Mirko Bonadei32ca4862020-03-11 10:39:45 +0100577 properties = GOMA_BACKEND_WEBRTC_RBE_PROD,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100578)
579
580android_builder, android_try_job = normal_builder_factory(
581 dimensions = {"os": "Linux"},
Mirko Bonadei32ca4862020-03-11 10:39:45 +0100582 properties = GOMA_BACKEND_WEBRTC_RBE_PROD,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100583)
584
585win_builder, win_try_job = normal_builder_factory(
586 dimensions = {"os": "Windows"},
Patrik Höglund3916a972019-12-04 13:19:38 +0100587 properties = GOMA_BACKEND_RBE_ATS_PROD,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100588)
589
590mac_builder, mac_try_job = normal_builder_factory(
591 dimensions = {"os": "Mac"},
Mirko Bonadei32ca4862020-03-11 10:39:45 +0100592 properties = GOMA_BACKEND_WEBRTC_RBE_PROD,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100593)
594
595ios_builder, ios_try_job = normal_builder_factory(
Patrik Höglundd5dcd0e2020-02-19 21:50:39 +0100596 dimensions = {"os": "Mac-10.14"},
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100597 recipe = "ios",
Mirko Bonadei84360ca2020-03-12 08:35:48 +0100598 properties = GOMA_BACKEND_WEBRTC_IOS_RBE_PROD,
599 caches = [swarming.cache("osx_sdk")],
600)
601
602# Some iOS builders don't use goma because they build using
603# `use_xcode_clang=true` which is not supported by goma.
604# https://ci.chromium.org/p/webrtc/builders/ci/iOS%20API%20Framework%20Builder
605# https://ci.chromium.org/p/webrtc/builders/try/ios_api_framework
606ios_builder_no_goma, ios_try_job_no_goma = normal_builder_factory(
607 dimensions = {"os": "Mac-10.14"},
608 recipe = "ios",
Patrik Höglundb681c162020-02-21 07:34:53 +0100609 properties = {"$depot_tools/osx_sdk": {"sdk_version": "11c29"}},
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100610 caches = [swarming.cache("osx_sdk")],
611)
612
613# Actual builder configuration:
614
615android_builder("Android32 (M Nexus5X)(dbg)", "Android|arm|dbg")
616android_try_job("android_compile_arm_dbg", cq = None)
617android_try_job("android_arm_dbg")
618android_builder("Android32 (M Nexus5X)", "Android|arm|rel")
619android_try_job("android_arm_rel")
620android_builder("Android32 Builder arm", "Android|arm|size", perf_cat = "Android|arm|Builder|", prioritized = True)
621android_try_job("android_compile_arm_rel")
Mirko Bonadei12643772020-03-27 13:16:35 +0100622android_perf_builder("Perf Android32 (K Nexus5)", "Android|arm|Tester|K Nexus5", triggered_by = ["Android32 Builder arm"])
623android_perf_builder("Perf Android32 (M AOSP Nexus6)", "Android|arm|Tester|M AOSP Nexus6", triggered_by = ["Android32 Builder arm"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100624android_try_job("android_compile_arm64_dbg", cq = None)
625android_try_job("android_arm64_dbg", cq = None)
626android_builder("Android64 (M Nexus5X)", "Android|arm64|rel")
627android_try_job("android_arm64_rel")
628android_builder("Android64 Builder arm64", "Android|arm64|size", perf_cat = "Android|arm64|Builder|", prioritized = True)
Mirko Bonadei12643772020-03-27 13:16:35 +0100629android_perf_builder("Perf Android64 (M Nexus5X)", "Android|arm64|Tester|M Nexus5X", triggered_by = ["Android64 Builder arm64"])
630android_perf_builder("Perf Android64 (O Pixel2)", "Android|arm64|Tester|O Pixel2", triggered_by = ["Android64 Builder arm64"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100631android_try_job("android_compile_arm64_rel")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100632android_builder("Android64 Builder x64 (dbg)", "Android|x64|dbg")
633android_try_job("android_compile_x64_dbg")
634android_try_job("android_compile_x64_rel", cq = None)
635android_builder("Android32 Builder x86 (dbg)", "Android|x86|dbg")
636android_try_job("android_compile_x86_dbg")
637android_builder("Android32 Builder x86", "Android|x86|rel")
638android_try_job("android_compile_x86_rel")
639android_builder("Android32 (more configs)", "Android|arm|more", recipe = "more_configs")
640android_try_job("android_arm_more_configs", recipe = "more_configs")
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100641android_try_job("android_chromium_compile", recipe = "chromium_trybot", branch_cq = False)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100642
643ios_builder("iOS32 Debug", "iOS|arm|dbg")
644ios_try_job("ios_compile_arm_dbg")
645ios_builder("iOS32 Release", "iOS|arm|rel")
646ios_try_job("ios_compile_arm_rel")
647ios_builder("iOS64 Debug", "iOS|arm64|dbg")
648ios_try_job("ios_compile_arm64_dbg")
649ios_builder("iOS64 Release", "iOS|arm64|rel")
650ios_try_job("ios_compile_arm64_rel")
651ios_builder("iOS64 Sim Debug (iOS 10.0)", "iOS|x64|10")
Patrik Höglund17bed672020-02-20 13:31:36 +0000652ios_try_job("ios_sim_x64_dbg_ios10")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100653ios_builder("iOS64 Sim Debug (iOS 11)", "iOS|x64|11")
Patrik Höglund17bed672020-02-20 13:31:36 +0000654ios_try_job("ios_sim_x64_dbg_ios11")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100655ios_builder("iOS64 Sim Debug (iOS 12)", "iOS|x64|12")
Patrik Höglund17bed672020-02-20 13:31:36 +0000656ios_try_job("ios_sim_x64_dbg_ios12")
Mirko Bonadei84360ca2020-03-12 08:35:48 +0100657ios_builder_no_goma("iOS API Framework Builder", "iOS|fat|size", recipe = "ios_api_framework", prioritized = True)
Jeremy Leconteea2016b2020-07-01 09:47:22 +0200658ios_try_job_no_goma("ios_api_framework", recipe = "ios_api_framework", cq = None)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100659
660linux_builder("Linux32 Debug", "Linux|x86|dbg")
661linux_try_job("linux_x86_dbg")
662linux_builder("Linux32 Release", "Linux|x86|rel")
663linux_try_job("linux_x86_rel")
664linux_builder("Linux64 Debug", "Linux|x64|dbg")
665linux_try_job("linux_dbg", cq = None)
666linux_try_job("linux_compile_dbg")
667linux_builder("Linux64 Release", "Linux|x64|rel")
668linux_try_job("linux_rel")
669linux_builder("Linux64 Builder", "Linux|x64|size", perf_cat = "Linux|x64|Builder|", prioritized = True)
670linux_try_job("linux_compile_rel")
Mirko Bonadei12643772020-03-27 13:16:35 +0100671linux_perf_builder("Perf Linux Trusty", "Linux|x64|Tester|Trusty", triggered_by = ["Linux64 Builder"])
672linux_perf_builder("Perf Linux Xenial", None, triggered_by = ["Linux64 Builder"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100673linux_builder("Linux32 Debug (ARM)", "Linux|arm|dbg")
674linux_try_job("linux_compile_arm_dbg")
675linux_builder("Linux32 Release (ARM)", "Linux|arm|rel")
676linux_try_job("linux_compile_arm_rel")
677linux_builder("Linux64 Debug (ARM)", "Linux|arm64|dbg")
678linux_try_job("linux_compile_arm64_dbg")
679linux_builder("Linux64 Release (ARM)", "Linux|arm64|rel")
680linux_try_job("linux_compile_arm64_rel")
681linux_builder("Linux64 Release (GCC)", "Linux|x64|gcc")
682linux_try_job("linux_compile_gcc_rel")
683linux_builder("Linux Asan", "Linux|x64|asan")
684linux_try_job("linux_asan")
685linux_builder("Linux MSan", "Linux|x64|msan")
686linux_try_job("linux_msan")
687linux_builder("Linux Tsan v2", "Linux|x64|tsan")
688linux_try_job("linux_tsan2")
689linux_builder("Linux UBSan", "Linux|x64|ubsan")
690linux_try_job("linux_ubsan")
691linux_builder("Linux UBSan vptr", "Linux|x64|ubsan")
692linux_try_job("linux_ubsan_vptr")
693linux_builder("Linux64 Release (Libfuzzer)", "Linux|x64|fuzz", recipe = "libfuzzer")
694linux_try_job("linux_libfuzzer_rel", recipe = "libfuzzer")
695linux_builder("Linux (more configs)", "Linux|x64|more", recipe = "more_configs")
696linux_try_job("linux_more_configs", recipe = "more_configs")
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100697linux_try_job("linux_chromium_compile", recipe = "chromium_trybot", branch_cq = False)
Mirko Bonadei40de50f2019-11-07 11:23:47 +0100698linux_try_job("linux_chromium_compile_dbg", recipe = "chromium_trybot", branch_cq = False)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100699
700mac_builder("Mac64 Debug", "Mac|x64|dbg")
701mac_try_job("mac_dbg", cq = None)
702mac_try_job("mac_compile_dbg")
703mac_builder("Mac64 Release", "Mac|x64|rel")
704mac_try_job("mac_rel")
705mac_try_job("mac_compile_rel", cq = None)
706mac_builder("Mac64 Builder", ci_cat = None, perf_cat = "Mac|x64|Builder|")
Mirko Bonadei12643772020-03-27 13:16:35 +0100707mac_perf_builder("Perf Mac 10.11", "Mac|x64|Tester|10.11", triggered_by = ["Mac64 Builder"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100708mac_builder("Mac Asan", "Mac|x64|asan")
709mac_try_job("mac_asan")
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100710mac_try_job("mac_chromium_compile", recipe = "chromium_trybot", dimensions = {"cores": "8"}, branch_cq = False)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100711
712win_builder("Win32 Debug", "Win MSVC|x86|dbg")
713win_try_job("win_x86_msvc_dbg")
714win_try_job("win_compile_x86_msvc_dbg", cq = None)
Oleh Prypin63041b82019-03-19 22:56:01 +0000715win_builder("Win32 Release", "Win MSVC|x86|rel")
716win_try_job("win_x86_msvc_rel")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100717win_try_job("win_compile_x86_msvc_rel", cq = None)
718win_builder("Win64 Debug", "Win MSVC|x64|dbg")
719win_try_job("win_x64_msvc_dbg")
720win_try_job("win_compile_x64_msvc_dbg", cq = None)
721win_builder("Win64 Release", "Win MSVC|x64|rel")
722win_try_job("win_x64_msvc_rel")
723win_try_job("win_compile_x64_msvc_rel", cq = None)
724win_builder("Win32 Debug (Clang)", "Win Clang|x86|dbg")
725win_try_job("win_x86_clang_dbg", cq = None)
726win_try_job("win_compile_x86_clang_dbg")
727win_builder("Win32 Release (Clang)", "Win Clang|x86|rel")
728win_try_job("win_x86_clang_rel")
729win_try_job("win_compile_x86_clang_rel", cq = None)
730win_builder("Win32 Builder (Clang)", ci_cat = None, perf_cat = "Win|x86|Builder|")
Mirko Bonadei12643772020-03-27 13:16:35 +0100731win_perf_builder("Perf Win7", "Win|x86|Tester|7", triggered_by = ["Win32 Builder (Clang)"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100732win_builder("Win64 Debug (Clang)", "Win Clang|x64|dbg")
733win_try_job("win_x64_clang_dbg", cq = None)
734win_try_job("win_x64_clang_dbg_win8", cq = None)
735win_try_job("win_x64_clang_dbg_win10", cq = None)
736win_try_job("win_compile_x64_clang_dbg")
737win_builder("Win64 Release (Clang)", "Win Clang|x64|rel")
738win_try_job("win_x64_clang_rel", cq = None)
739win_try_job("win_compile_x64_clang_rel")
Mirko Bonadeiafbe33d2019-05-31 09:27:54 +0200740win_builder("Win64 ASan", "Win Clang|x64|asan")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100741win_try_job("win_asan")
742win_builder("Win64 UWP", ci_cat = None, fyi_cat = "")
743win_try_job("win_x64_uwp", cq = None, try_cat = None, fyi_cat = "")
744win_builder("Win (more configs)", "Win Clang|x86|more", recipe = "more_configs")
745win_try_job("win_x86_more_configs", recipe = "more_configs")
Mirko Bonadei825f9c22020-07-01 11:52:39 +0200746win_try_job("win_chromium_compile", recipe = "chromium_trybot", branch_cq = False, goma_jobs = 150)
747win_try_job("win_chromium_compile_dbg", recipe = "chromium_trybot", branch_cq = False, goma_jobs = 150)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100748
749linux_try_job(
750 "presubmit",
751 recipe = "run_presubmit",
752 properties = {"repo_name": "webrtc", "runhooks": True},
753 priority = 28,
754 cq = {"disable_reuse": True},
755)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100756
757cron_builder(
758 "Auto-roll - WebRTC DEPS",
759 recipe = "auto_roll_webrtc_deps",
Oleh Prypindc68a722019-06-25 10:43:13 +0200760 schedule = "0 */2 * * *", # Every 2 hours.
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100761)
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200762
763lkgr_config = {
764 "project": "webrtc",
765 "source_url": WEBRTC_GIT,
766 "status_url": "https://webrtc-status.appspot.com",
767 "allowed_lag": 300, # hours
768 "allowed_gap": 12, # commits behind
769 "error_recipients": "webrtc-troopers-robots@google.com",
Michael Achenbach09dd7dc2019-07-26 15:26:11 +0200770 "buckets": {
771 "webrtc/ci": {
772 # bucket alias: luci.webrtc.ci
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200773 "builders": sorted(lkgr_builders),
774 },
Michael Achenbach09dd7dc2019-07-26 15:26:11 +0200775 "chromium/webrtc.fyi": {
776 # bucket alias: luci.chromium.webrtc.fyi
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200777 "builders": [
778 "WebRTC Chromium FYI Android Builder (dbg)",
779 "WebRTC Chromium FYI Android Builder ARM64 (dbg)",
780 "WebRTC Chromium FYI Android Builder",
Mirko Bonadei04fe32f2020-05-01 00:45:58 +0200781 "WebRTC Chromium FYI Android Tests (dbg) (L Nexus5)",
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200782 "WebRTC Chromium FYI Android Tests (dbg) (M Nexus5X)",
783 "WebRTC Chromium FYI Linux Builder (dbg)",
784 "WebRTC Chromium FYI Linux Builder",
785 "WebRTC Chromium FYI Linux Tester",
786 "WebRTC Chromium FYI Mac Builder (dbg)",
787 "WebRTC Chromium FYI Mac Builder",
788 "WebRTC Chromium FYI Mac Tester",
789 "WebRTC Chromium FYI Win Builder (dbg)",
790 "WebRTC Chromium FYI Win Builder",
791 "WebRTC Chromium FYI Win10 Tester",
792 "WebRTC Chromium FYI Win7 Tester",
793 "WebRTC Chromium FYI Win8 Tester",
794 "WebRTC Chromium FYI ios-device",
Mirko Bonadei37071f62020-02-15 10:42:49 +0100795 "WebRTC Chromium FYI ios-simulator",
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200796 ],
797 },
798 },
799}
800
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100801cron_builder(
802 "WebRTC lkgr finder",
803 recipe = "lkgr_finder",
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200804 properties = {
805 "project": "webrtc",
806 "repo": WEBRTC_GIT,
807 "ref": "refs/heads/lkgr",
808 "lkgr_status_gs_path": "chromium-webrtc/lkgr-status",
809 "config": lkgr_config,
810 },
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100811 schedule = "*/10 * * * *", # Every 10 minutes.
812)