blob: 81d4d2371a60b34c274d9c29945d41a37f9ad42f [file] [log] [blame]
Oleh Prypin1a0593f2019-03-11 09:43:28 +01001#!/usr/bin/env lucicfg
2
3# Copyright (c) 2019 The WebRTC project authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7# https://chromium.googlesource.com/infra/luci/luci-go/+/master/lucicfg/doc/
8
Jeremy Leconteea2016b2020-07-01 09:47:22 +02009"""LUCI project configuration for WebRTC CQ and CI."""
10
11lucicfg.check_version("1.15.0")
Mirko Bonadei9189d9c2020-06-09 21:54:18 +020012
Oleh Prypin1a0593f2019-03-11 09:43:28 +010013WEBRTC_GIT = "https://webrtc.googlesource.com/src"
14WEBRTC_GERRIT = "https://webrtc-review.googlesource.com/src"
15
Mirko Bonadei32ca4862020-03-11 10:39:45 +010016GOMA_BACKEND_WEBRTC_RBE_PROD = {
Mirko Bonadei1e905342020-03-04 09:39:42 +010017 "$build/goma": {
18 "server_host": "goma.chromium.org",
Jeremy Leconteea2016b2020-07-01 09:47:22 +020019 "use_luci_auth": True,
20 },
Mirko Bonadei1e905342020-03-04 09:39:42 +010021}
22
Mirko Bonadei84360ca2020-03-12 08:35:48 +010023GOMA_BACKEND_WEBRTC_IOS_RBE_PROD = {
24 "$build/goma": {
25 "server_host": "goma.chromium.org",
Jeremy Leconteea2016b2020-07-01 09:47:22 +020026 "use_luci_auth": True,
Mirko Bonadei84360ca2020-03-12 08:35:48 +010027 },
28 "$depot_tools/osx_sdk": {"sdk_version": "11c29"},
29}
30
Patrik Höglund3916a972019-12-04 13:19:38 +010031GOMA_BACKEND_RBE_ATS_PROD = {
32 "$build/goma": {
33 "server_host": "goma.chromium.org",
Mirko Bonadei2688eca2020-03-25 09:15:54 +010034 "use_luci_auth": True,
Jeremy Leconteea2016b2020-07-01 09:47:22 +020035 "enable_ats": True,
36 },
Patrik Höglund3916a972019-12-04 13:19:38 +010037}
38
Oleh Prypin1a0593f2019-03-11 09:43:28 +010039# Top-level configs:
40
Owen Rodley6b721ba2020-05-26 10:04:20 +100041# Enable luci.tree_closer.
42lucicfg.enable_experiment("crbug.com/1054172")
43
Oleh Prypin1a0593f2019-03-11 09:43:28 +010044lucicfg.config(
45 config_dir = ".",
46 tracked_files = [
47 "commit-queue.cfg",
48 "cr-buildbucket.cfg",
49 "luci-logdog.cfg",
50 "luci-milo.cfg",
Oleh Prypin71d17422019-03-28 08:43:16 +010051 "luci-notify.cfg",
Oleh Prypin705b6a62019-04-03 23:10:51 +020052 "luci-notify/**/*",
Oleh Prypin1a0593f2019-03-11 09:43:28 +010053 "luci-scheduler.cfg",
54 "project.cfg",
55 ],
Jeremy Leconteea2016b2020-07-01 09:47:22 +020056 lint_checks = ["default"],
Oleh Prypin1a0593f2019-03-11 09:43:28 +010057)
58
59luci.project(
60 name = "webrtc",
61 buildbucket = "cr-buildbucket.appspot.com",
62 logdog = "luci-logdog.appspot.com",
63 milo = "luci-milo.appspot.com",
Oleh Prypin71d17422019-03-28 08:43:16 +010064 notify = "luci-notify.appspot.com",
Oleh Prypin1a0593f2019-03-11 09:43:28 +010065 scheduler = "luci-scheduler.appspot.com",
66 swarming = "chromium-swarm.appspot.com",
67 acls = [
68 acl.entry(
69 [acl.BUILDBUCKET_READER, acl.LOGDOG_READER, acl.PROJECT_CONFIGS_READER, acl.SCHEDULER_READER],
70 groups = ["all"],
71 ),
72 acl.entry(acl.LOGDOG_WRITER, groups = ["luci-logdog-chromium-writers"]),
73 acl.entry(acl.SCHEDULER_OWNER, groups = ["project-webrtc-admins"]),
74 ],
75)
76
77luci.logdog(
78 gs_bucket = "chromium-luci-logdog",
79)
80
81luci.milo(
82 logo = "https://storage.googleapis.com/chrome-infra/webrtc-logo-vert-retro-255x305.png",
83)
84
Owen Rodleya55c38e2020-06-15 11:16:10 +100085luci.notify(tree_closing_enabled = True)
86
Oleh Prypin1a0593f2019-03-11 09:43:28 +010087luci.cq(
88 status_host = "chromium-cq-status.appspot.com",
89 submit_max_burst = 1,
90 submit_burst_delay = 1 * time.minute,
91)
92
93luci.gitiles_poller(
94 name = "webrtc-gitiles-trigger-master",
95 bucket = "ci",
96 repo = WEBRTC_GIT,
97)
98
99# Bucket definitions:
100
101luci.bucket(
102 name = "try",
103 acls = [
104 acl.entry(acl.BUILDBUCKET_TRIGGERER, groups = [
105 "service-account-cq",
106 "project-webrtc-tryjob-access",
107 ]),
108 ],
109)
110
111luci.bucket(
112 name = "ci",
113 acls = [
Oleh Prypinf35939d2019-05-03 20:42:38 +0200114 acl.entry(acl.BUILDBUCKET_TRIGGERER, groups = [
115 "project-webrtc-ci-schedulers",
116 ], users = [
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100117 "luci-scheduler@appspot.gserviceaccount.com",
118 ]),
119 ],
120)
121
122luci.bucket(
123 name = "perf",
124 acls = [
Jeremy Leconte4e951372020-07-01 14:48:04 +0200125 acl.entry(acl.BUILDBUCKET_TRIGGERER, groups = [
126 # Allow Pinpoint to trigger builds for bisection
127 "service-account-chromeperf",
128 ]),
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100129 acl.entry(acl.BUILDBUCKET_TRIGGERER, users = [
130 "luci-scheduler@appspot.gserviceaccount.com",
131 "webrtc-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
132 ]),
133 ],
134)
135
136luci.bucket(
137 name = "cron",
138 acls = [
139 acl.entry(acl.BUILDBUCKET_TRIGGERER, users = [
140 "luci-scheduler@appspot.gserviceaccount.com",
141 ]),
142 ],
143)
144
145# Commit queue definitions:
146
147luci.cq_group(
148 name = "cq",
149 tree_status_host = "webrtc-status.appspot.com",
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100150 watch = [cq.refset(repo = WEBRTC_GERRIT, refs = ["refs/heads/master"])],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100151 acls = [
152 acl.entry(acl.CQ_COMMITTER, groups = ["project-webrtc-committers"]),
153 acl.entry(acl.CQ_DRY_RUNNER, groups = ["project-webrtc-tryjob-access"]),
154 ],
155 retry_config = cq.RETRY_ALL_FAILURES,
Oleh Prypin5416f2d2019-08-02 09:43:15 +0200156 cancel_stale_tryjobs = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100157)
158
159luci.cq_group(
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100160 name = "cq_branch",
161 watch = [cq.refset(repo = WEBRTC_GERRIT, refs = ["refs/branch-heads/.+"])],
162 acls = [
163 acl.entry(acl.CQ_COMMITTER, groups = ["project-webrtc-committers"]),
164 acl.entry(acl.CQ_DRY_RUNNER, groups = ["project-webrtc-tryjob-access"]),
165 ],
166 retry_config = cq.RETRY_ALL_FAILURES,
Oleh Prypin5416f2d2019-08-02 09:43:15 +0200167 cancel_stale_tryjobs = True,
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100168)
169
170luci.cq_group(
171 name = "cq_infra",
172 watch = [cq.refset(repo = WEBRTC_GERRIT, refs = ["refs/heads/infra/config"])],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100173 acls = [
174 acl.entry(acl.CQ_COMMITTER, groups = ["project-webrtc-admins"]),
175 acl.entry(acl.CQ_DRY_RUNNER, groups = ["project-webrtc-tryjob-access"]),
176 ],
177 retry_config = cq.RETRY_ALL_FAILURES,
Oleh Prypin5416f2d2019-08-02 09:43:15 +0200178 cancel_stale_tryjobs = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100179)
180
181luci.cq_tryjob_verifier(
182 builder = "presubmit",
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100183 cq_group = "cq_infra",
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100184)
185
Oleh Prypine56cd9b2019-05-29 21:14:31 +0200186luci.cq_tryjob_verifier(
187 builder = "webrtc-internal:g3.webrtc-internal.try/internal_compile_lite",
188 owner_whitelist = ["project-webrtc-internal-tryjob-access"],
189 cq_group = "cq",
190)
191
Oleh Prypin71d17422019-03-28 08:43:16 +0100192# Notifier definitions:
193
194luci.notifier(
195 name = "ci_notifier",
196 on_failure = True,
197 notify_emails = ["webrtc-sheriffs-robots@google.com"],
Oleh Prypin705b6a62019-04-03 23:10:51 +0200198 template = luci.notifier_template(
199 name = "ci",
200 body = io.read_file("luci-notify/email-templates/ci.template"),
201 ),
Oleh Prypin71d17422019-03-28 08:43:16 +0100202)
203
204luci.notifier(
205 name = "cron_notifier",
206 on_failure = True,
207 notify_emails = ["webrtc-troopers-robots@google.com"],
Oleh Prypin705b6a62019-04-03 23:10:51 +0200208 template = luci.notifier_template(
209 name = "cron",
210 body = io.read_file("luci-notify/email-templates/cron.template"),
211 ),
Oleh Prypin71d17422019-03-28 08:43:16 +0100212)
213
Owen Rodley6b721ba2020-05-26 10:04:20 +1000214# Tree closer definitions:
215
216luci.tree_closer(
217 name = "webrtc_tree_closer",
218 tree_status_host = "webrtc-status.appspot.com",
219 template = luci.notifier_template(
220 name = "status",
221 body = io.read_file("luci-notify/email-templates/status.template"),
222 ),
223 # TODO: These step filters are copied verbatim from Gatekeeper, for testing
224 # that LUCI-Notify would take the exact same actions. Once we've switched
225 # over, this should be updated - several of these steps don't exist in
226 # WebRTC recipes.
227 failed_step_regexp = [
228 "bot_update",
229 "compile",
230 "gclient runhooks",
231 "runhooks",
232 "update",
233 "extract build",
234 "cleanup_temp",
235 "taskkill",
236 "compile",
237 "gn",
238 ],
239 failed_step_regexp_exclude = ".*\\(experimental\\).*",
240)
241
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100242# Recipe definitions:
243
244def recipe(recipe, pkg = "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"):
245 return luci.recipe(
246 name = recipe.split("/")[-1],
247 cipd_package = pkg,
248 cipd_version = "refs/heads/master",
249 recipe = recipe,
250 )
251
252recipe("chromium_trybot")
253recipe("run_presubmit")
254recipe("webrtc/auto_roll_webrtc_deps")
255recipe("webrtc/ios")
256recipe("webrtc/ios_api_framework")
257recipe("webrtc/libfuzzer")
258recipe("webrtc/more_configs")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100259recipe("webrtc/standalone")
260recipe("lkgr_finder", pkg = "infra/recipe_bundles/chromium.googlesource.com/infra/infra")
261
262# Console definitions:
263
264luci.console_view(name = "ci", title = "Main", repo = WEBRTC_GIT, header = "console-header.textpb")
265luci.console_view(name = "perf", title = "Perf", repo = WEBRTC_GIT, header = "console-header.textpb")
266luci.list_view(name = "cron", title = "Cron")
267luci.console_view(name = "fyi", title = "FYI", repo = WEBRTC_GIT, header = "console-header.textpb")
268luci.list_view(name = "try", title = "Tryserver")
269
270def add_milo(builder, views):
271 """Add Milo console entries for the builder.
272
273 Args:
274 builder: builder name (str).
275 views: dict where keys are names of consoles and values are either a
276 category for the console (str, pipe-separated) or True, which means
277 adding to a list view rather than a console.
278 """
279 for view_name, category in views.items():
280 if category == None:
281 continue
282 elif type(category) == "string":
283 category, _, short_name = category.rpartition("|")
284 luci.console_view_entry(
285 console_view = view_name,
286 builder = builder,
287 category = category or None,
288 short_name = short_name or None,
289 )
290 elif category == True:
291 luci.list_view_entry(
292 list_view = view_name,
293 builder = builder,
294 )
295 else:
296 fail("Unexpected value for category: %r" % category)
297
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200298lkgr_builders = {}
299
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100300# Builder-defining functions:
301
Artem Titarenkof6588b72019-11-14 14:28:11 +0100302def webrtc_builder(
303 name,
304 recipe = "standalone",
305 dimensions = {},
Mirko Bonadei825f9c22020-07-01 11:52:39 +0200306 properties = {},
Artem Titarenkof6588b72019-11-14 14:28:11 +0100307 priority = 30,
308 execution_timeout = 2 * time.hour,
Mirko Bonadeib7ef4602020-07-01 13:28:26 +0200309 goma_jobs = None,
Artem Titarenkof6588b72019-11-14 14:28:11 +0100310 **kwargs):
Mirko Bonadeib7ef4602020-07-01 13:28:26 +0200311 """WebRTC specific wrapper around luci.builder.
312
313 Args:
314 name: builder name (str).
315 recipe: string with the name of the recipe to run.
316 dimensions: dict of Swarming dimensions (strings) to search machines by.
317 properties: dict of properties to pass to the recipe (on top of the default ones).
318 priority: int [1-255] or None, indicating swarming task priority, lower is
319 more important. If None, defer the decision to Buildbucket service.
320 execution_timeout: int or None, how long to wait for a running build to finish before
321 forcefully aborting it and marking the build as timed out. If None,
322 defer the decision to Buildbucket service.
323 goma_jobs: int or None, number of jobs to be used by the builder. If None, defer the
324 decition to the goma module.
325 **kwargs: Pass on to webrtc_builder / luci.builder.
326 Returns:
327 A luci.builder.
328 """
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100329 dimensions = merge_dicts({"cpu": "x86-64"}, dimensions)
330
Mirko Bonadeib7ef4602020-07-01 13:28:26 +0200331 if goma_jobs != None:
332 goma_additional_params = {"$build/goma": {"jobs": goma_jobs}}
333 properties = merge_dicts(properties, goma_additional_params)
Mirko Bonadei825f9c22020-07-01 11:52:39 +0200334
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100335 return luci.builder(
336 name = name,
Oleh Prypin705b6a62019-04-03 23:10:51 +0200337 executable = recipe,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100338 dimensions = {k: v for k, v in dimensions.items() if v != None},
Mirko Bonadeib7ef4602020-07-01 13:28:26 +0200339 properties = properties,
Artem Titarenkof6588b72019-11-14 14:28:11 +0100340 execution_timeout = execution_timeout,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100341 priority = priority,
342 build_numbers = True,
343 swarming_tags = ["vpython:native-python-wrapper"],
344 **kwargs
345 )
346
347def ci_builder(
348 name,
349 ci_cat,
350 perf_cat = None,
351 fyi_cat = None,
352 properties = {},
353 dimensions = {},
354 prioritized = False,
Oleh Prypinf5432c52019-03-19 15:10:37 +0100355 enabled = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100356 **kwargs):
Oleh Prypine5677672019-04-25 12:21:35 +0200357 """Add a post-submit builder.
358
359 Args:
360 name: builder name (str).
361 ci_cat: the category + name for the /ci/ console, or None to omit from the console.
362 perf_cat: the category + name for the /perf/ console, or None to omit from the console.
363 fyi_cat: the category + name for the /fyi/ console, or None to omit from the console.
364 properties: dict of properties to pass to the recipe (on top of the default ones).
365 dimensions: dict of Swarming dimensions (strings) to search machines by.
366 prioritized: True to make this builder have a higher priority and never batch builds.
367 enabled: False to exclude this builder from consoles and failure notifications.
368 **kwargs: Pass on to webrtc_builder / luci.builder.
369 Returns:
370 A luci.builder.
371
372 Notifications are also disabled if a builder is not on either of /ci/ or /perf/ consoles.
373 """
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100374 if prioritized:
375 kwargs["triggering_policy"] = scheduler.greedy_batching(
376 max_batch_size = 1,
377 max_concurrent_invocations = 3,
378 )
379 kwargs["priority"] = 29
380
Oleh Prypinf5432c52019-03-19 15:10:37 +0100381 if enabled:
382 add_milo(name, {"ci": ci_cat, "perf": perf_cat, "fyi": fyi_cat})
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200383 if ci_cat:
384 lkgr_builders[name] = True
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100385 return webrtc_builder(
386 name = name,
387 properties = merge_dicts({"mastername": "client.webrtc"}, properties),
388 dimensions = merge_dicts({"pool": "luci.webrtc.ci"}, dimensions),
389 bucket = "ci",
390 service_account = "webrtc-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
Oleh Prypinf5432c52019-03-19 15:10:37 +0100391 triggered_by = ["webrtc-gitiles-trigger-master"] if enabled else None,
Oleh Prypin71d17422019-03-28 08:43:16 +0100392 repo = WEBRTC_GIT,
Owen Rodley6b721ba2020-05-26 10:04:20 +1000393 notifies = ["ci_notifier", "webrtc_tree_closer"] if enabled and (ci_cat or perf_cat) else None,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100394 **kwargs
395 )
396
397def try_builder(
398 name,
399 try_cat = True,
400 fyi_cat = None,
401 properties = {},
402 dimensions = {},
403 cq = {},
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100404 branch_cq = True,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100405 **kwargs):
Oleh Prypine5677672019-04-25 12:21:35 +0200406 """Add a pre-submit builder.
407
408 Args:
409 name: builder name (str).
410 try_cat: boolean, whether to include this builder in the /try/ console. See also: `add_milo`.
411 fyi_cat: the category + name for the /fyi/ console, or None to omit from the console.
412 properties: dict of properties to pass to the recipe (on top of the default ones).
413 dimensions: dict of Swarming dimensions (strings) to search machines by.
414 cq: None to exclude this from all commit queues, or a dict of kwargs for cq_tryjob_verifier.
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100415 branch_cq: False to exclude this builder just from the release-branch CQ.
Oleh Prypine5677672019-04-25 12:21:35 +0200416 **kwargs: Pass on to webrtc_builder / luci.builder.
417 Returns:
418 A luci.builder.
419 """
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100420 add_milo(name, {"try": try_cat, "fyi": fyi_cat})
421 if cq != None:
422 luci.cq_tryjob_verifier(
423 builder = name,
424 cq_group = "cq",
425 **cq
426 )
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100427 if branch_cq:
428 luci.cq_tryjob_verifier(
429 builder = name,
430 cq_group = "cq_branch",
431 **cq
432 )
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100433
434 return webrtc_builder(
435 name = name,
436 properties = merge_dicts({"mastername": "tryserver.webrtc"}, properties),
437 dimensions = merge_dicts({"pool": "luci.webrtc.try"}, dimensions),
438 bucket = "try",
439 service_account = "webrtc-try-builder@chops-service-accounts.iam.gserviceaccount.com",
440 **kwargs
441 )
442
443def perf_builder(
444 name,
445 perf_cat,
446 recipe = "standalone",
447 properties = {},
448 dimensions = {},
449 **kwargs):
450 add_milo(name, {"perf": perf_cat})
451 return webrtc_builder(
452 name = name,
453 recipe = recipe,
454 properties = merge_dicts({"mastername": "client.webrtc.perf"}, properties),
455 dimensions = merge_dicts({"pool": "luci.webrtc.perf", "os": "Linux", "cpu": None}, dimensions),
456 bucket = "perf",
457 service_account = "webrtc-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
458 # log_base of 1.7 means:
459 # when there are P pending builds, LUCI will batch the first B builds.
460 # P: 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 ...
461 # B: 1 1 2 2 3 3 3 3 4 4 4 4 4 4 5 ...
Jeremy Leconteea2016b2020-07-01 09:47:22 +0200462 triggering_policy = scheduler.logarithmic_batching(log_base = 1.7),
Artem Titarenkof6588b72019-11-14 14:28:11 +0100463 execution_timeout = 3 * time.hour,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100464 **kwargs
465 )
466
Mirko Bonadei12643772020-03-27 13:16:35 +0100467def android_perf_builder(
468 name,
469 perf_cat,
470 recipe = "standalone",
471 properties = {},
472 dimensions = {},
473 **kwargs):
474 return perf_builder(
475 name = name,
476 perf_cat = perf_cat,
477 recipe = recipe,
478 properties = merge_dicts(GOMA_BACKEND_WEBRTC_RBE_PROD, properties),
479 dimensions = dimensions,
480 **kwargs
481 )
482
483def linux_perf_builder(
484 name,
485 perf_cat,
486 recipe = "standalone",
487 properties = {},
488 dimensions = {},
489 **kwargs):
490 return perf_builder(
491 name = name,
492 perf_cat = perf_cat,
493 recipe = recipe,
494 properties = merge_dicts(GOMA_BACKEND_WEBRTC_RBE_PROD, properties),
495 dimensions = dimensions,
496 **kwargs
497 )
498
499def mac_perf_builder(
500 name,
501 perf_cat,
502 recipe = "standalone",
503 properties = {},
504 dimensions = {},
505 **kwargs):
506 return perf_builder(
507 name = name,
508 perf_cat = perf_cat,
509 recipe = recipe,
510 properties = merge_dicts(GOMA_BACKEND_WEBRTC_RBE_PROD, properties),
511 dimensions = dimensions,
512 **kwargs
513 )
514
515def win_perf_builder(
516 name,
517 perf_cat,
518 recipe = "standalone",
519 properties = {},
520 dimensions = {},
521 **kwargs):
522 return perf_builder(
523 name = name,
524 perf_cat = perf_cat,
525 recipe = recipe,
526 properties = merge_dicts(GOMA_BACKEND_RBE_ATS_PROD, properties),
527 dimensions = dimensions,
528 **kwargs
529 )
530
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100531def cron_builder(name, dimensions = {}, **kwargs):
532 add_milo(name, {"cron": True})
533 return webrtc_builder(
534 name = name,
535 dimensions = merge_dicts({"pool": "luci.webrtc.cron", "os": "Linux"}, dimensions),
536 bucket = "cron",
537 service_account = "chromium-webrtc-autoroll@webrtc-ci.iam.gserviceaccount.com",
Oleh Prypin71d17422019-03-28 08:43:16 +0100538 repo = WEBRTC_GIT,
539 notifies = ["cron_notifier"],
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100540 **kwargs
541 )
542
543# Helpers:
544
545def merge_dicts(a, b):
546 """Return the result of merging two dicts.
547
548 If matching values are both dicts or both lists, they will be merged (non-recursively).
549
550 Args:
551 a: first dict.
552 b: second dict (takes priority).
553 Returns:
554 Merged dict.
555 """
556 a = dict(a)
557 for k, bv in b.items():
558 av = a.get(k)
559 if type(av) == "dict" and type(bv) == "dict":
560 a[k] = dict(av)
561 a[k].update(bv)
562 elif type(av) == "list" and type(bv) == "list":
563 a[k] = av + bv
564 else:
565 a[k] = bv
566 return a
567
568def normal_builder_factory(**common_kwargs):
569 def builder(*args, **kwargs):
570 return ci_builder(*args, **merge_dicts(common_kwargs, kwargs))
571
572 def try_job(*args, **kwargs):
573 return try_builder(*args, **merge_dicts(common_kwargs, kwargs))
574
575 return builder, try_job
576
577# Mixins:
578
579linux_builder, linux_try_job = normal_builder_factory(
580 dimensions = {"os": "Linux", "inside_docker": "0"},
Mirko Bonadei32ca4862020-03-11 10:39:45 +0100581 properties = GOMA_BACKEND_WEBRTC_RBE_PROD,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100582)
583
584android_builder, android_try_job = normal_builder_factory(
585 dimensions = {"os": "Linux"},
Mirko Bonadei32ca4862020-03-11 10:39:45 +0100586 properties = GOMA_BACKEND_WEBRTC_RBE_PROD,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100587)
588
589win_builder, win_try_job = normal_builder_factory(
590 dimensions = {"os": "Windows"},
Patrik Höglund3916a972019-12-04 13:19:38 +0100591 properties = GOMA_BACKEND_RBE_ATS_PROD,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100592)
593
594mac_builder, mac_try_job = normal_builder_factory(
595 dimensions = {"os": "Mac"},
Mirko Bonadei32ca4862020-03-11 10:39:45 +0100596 properties = GOMA_BACKEND_WEBRTC_RBE_PROD,
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100597)
598
599ios_builder, ios_try_job = normal_builder_factory(
Patrik Höglundd5dcd0e2020-02-19 21:50:39 +0100600 dimensions = {"os": "Mac-10.14"},
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100601 recipe = "ios",
Mirko Bonadei84360ca2020-03-12 08:35:48 +0100602 properties = GOMA_BACKEND_WEBRTC_IOS_RBE_PROD,
603 caches = [swarming.cache("osx_sdk")],
604)
605
606# Some iOS builders don't use goma because they build using
607# `use_xcode_clang=true` which is not supported by goma.
608# https://ci.chromium.org/p/webrtc/builders/ci/iOS%20API%20Framework%20Builder
609# https://ci.chromium.org/p/webrtc/builders/try/ios_api_framework
610ios_builder_no_goma, ios_try_job_no_goma = normal_builder_factory(
611 dimensions = {"os": "Mac-10.14"},
612 recipe = "ios",
Patrik Höglundb681c162020-02-21 07:34:53 +0100613 properties = {"$depot_tools/osx_sdk": {"sdk_version": "11c29"}},
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100614 caches = [swarming.cache("osx_sdk")],
615)
616
617# Actual builder configuration:
618
619android_builder("Android32 (M Nexus5X)(dbg)", "Android|arm|dbg")
620android_try_job("android_compile_arm_dbg", cq = None)
621android_try_job("android_arm_dbg")
622android_builder("Android32 (M Nexus5X)", "Android|arm|rel")
623android_try_job("android_arm_rel")
624android_builder("Android32 Builder arm", "Android|arm|size", perf_cat = "Android|arm|Builder|", prioritized = True)
625android_try_job("android_compile_arm_rel")
Mirko Bonadei12643772020-03-27 13:16:35 +0100626android_perf_builder("Perf Android32 (K Nexus5)", "Android|arm|Tester|K Nexus5", triggered_by = ["Android32 Builder arm"])
627android_perf_builder("Perf Android32 (M AOSP Nexus6)", "Android|arm|Tester|M AOSP Nexus6", triggered_by = ["Android32 Builder arm"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100628android_try_job("android_compile_arm64_dbg", cq = None)
629android_try_job("android_arm64_dbg", cq = None)
630android_builder("Android64 (M Nexus5X)", "Android|arm64|rel")
631android_try_job("android_arm64_rel")
632android_builder("Android64 Builder arm64", "Android|arm64|size", perf_cat = "Android|arm64|Builder|", prioritized = True)
Mirko Bonadei12643772020-03-27 13:16:35 +0100633android_perf_builder("Perf Android64 (M Nexus5X)", "Android|arm64|Tester|M Nexus5X", triggered_by = ["Android64 Builder arm64"])
634android_perf_builder("Perf Android64 (O Pixel2)", "Android|arm64|Tester|O Pixel2", triggered_by = ["Android64 Builder arm64"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100635android_try_job("android_compile_arm64_rel")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100636android_builder("Android64 Builder x64 (dbg)", "Android|x64|dbg")
637android_try_job("android_compile_x64_dbg")
638android_try_job("android_compile_x64_rel", cq = None)
639android_builder("Android32 Builder x86 (dbg)", "Android|x86|dbg")
640android_try_job("android_compile_x86_dbg")
641android_builder("Android32 Builder x86", "Android|x86|rel")
642android_try_job("android_compile_x86_rel")
643android_builder("Android32 (more configs)", "Android|arm|more", recipe = "more_configs")
644android_try_job("android_arm_more_configs", recipe = "more_configs")
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100645android_try_job("android_chromium_compile", recipe = "chromium_trybot", branch_cq = False)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100646
647ios_builder("iOS32 Debug", "iOS|arm|dbg")
648ios_try_job("ios_compile_arm_dbg")
649ios_builder("iOS32 Release", "iOS|arm|rel")
650ios_try_job("ios_compile_arm_rel")
651ios_builder("iOS64 Debug", "iOS|arm64|dbg")
652ios_try_job("ios_compile_arm64_dbg")
653ios_builder("iOS64 Release", "iOS|arm64|rel")
654ios_try_job("ios_compile_arm64_rel")
655ios_builder("iOS64 Sim Debug (iOS 10.0)", "iOS|x64|10")
Patrik Höglund17bed672020-02-20 13:31:36 +0000656ios_try_job("ios_sim_x64_dbg_ios10")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100657ios_builder("iOS64 Sim Debug (iOS 11)", "iOS|x64|11")
Patrik Höglund17bed672020-02-20 13:31:36 +0000658ios_try_job("ios_sim_x64_dbg_ios11")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100659ios_builder("iOS64 Sim Debug (iOS 12)", "iOS|x64|12")
Patrik Höglund17bed672020-02-20 13:31:36 +0000660ios_try_job("ios_sim_x64_dbg_ios12")
Mirko Bonadei84360ca2020-03-12 08:35:48 +0100661ios_builder_no_goma("iOS API Framework Builder", "iOS|fat|size", recipe = "ios_api_framework", prioritized = True)
Jeremy Leconteea2016b2020-07-01 09:47:22 +0200662ios_try_job_no_goma("ios_api_framework", recipe = "ios_api_framework", cq = None)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100663
664linux_builder("Linux32 Debug", "Linux|x86|dbg")
665linux_try_job("linux_x86_dbg")
666linux_builder("Linux32 Release", "Linux|x86|rel")
667linux_try_job("linux_x86_rel")
668linux_builder("Linux64 Debug", "Linux|x64|dbg")
669linux_try_job("linux_dbg", cq = None)
670linux_try_job("linux_compile_dbg")
671linux_builder("Linux64 Release", "Linux|x64|rel")
672linux_try_job("linux_rel")
673linux_builder("Linux64 Builder", "Linux|x64|size", perf_cat = "Linux|x64|Builder|", prioritized = True)
674linux_try_job("linux_compile_rel")
Mirko Bonadei12643772020-03-27 13:16:35 +0100675linux_perf_builder("Perf Linux Trusty", "Linux|x64|Tester|Trusty", triggered_by = ["Linux64 Builder"])
676linux_perf_builder("Perf Linux Xenial", None, triggered_by = ["Linux64 Builder"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100677linux_builder("Linux32 Debug (ARM)", "Linux|arm|dbg")
678linux_try_job("linux_compile_arm_dbg")
679linux_builder("Linux32 Release (ARM)", "Linux|arm|rel")
680linux_try_job("linux_compile_arm_rel")
681linux_builder("Linux64 Debug (ARM)", "Linux|arm64|dbg")
682linux_try_job("linux_compile_arm64_dbg")
683linux_builder("Linux64 Release (ARM)", "Linux|arm64|rel")
684linux_try_job("linux_compile_arm64_rel")
685linux_builder("Linux64 Release (GCC)", "Linux|x64|gcc")
686linux_try_job("linux_compile_gcc_rel")
687linux_builder("Linux Asan", "Linux|x64|asan")
688linux_try_job("linux_asan")
689linux_builder("Linux MSan", "Linux|x64|msan")
690linux_try_job("linux_msan")
691linux_builder("Linux Tsan v2", "Linux|x64|tsan")
692linux_try_job("linux_tsan2")
693linux_builder("Linux UBSan", "Linux|x64|ubsan")
694linux_try_job("linux_ubsan")
695linux_builder("Linux UBSan vptr", "Linux|x64|ubsan")
696linux_try_job("linux_ubsan_vptr")
697linux_builder("Linux64 Release (Libfuzzer)", "Linux|x64|fuzz", recipe = "libfuzzer")
698linux_try_job("linux_libfuzzer_rel", recipe = "libfuzzer")
699linux_builder("Linux (more configs)", "Linux|x64|more", recipe = "more_configs")
700linux_try_job("linux_more_configs", recipe = "more_configs")
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100701linux_try_job("linux_chromium_compile", recipe = "chromium_trybot", branch_cq = False)
Mirko Bonadei40de50f2019-11-07 11:23:47 +0100702linux_try_job("linux_chromium_compile_dbg", recipe = "chromium_trybot", branch_cq = False)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100703
704mac_builder("Mac64 Debug", "Mac|x64|dbg")
705mac_try_job("mac_dbg", cq = None)
706mac_try_job("mac_compile_dbg")
707mac_builder("Mac64 Release", "Mac|x64|rel")
708mac_try_job("mac_rel")
709mac_try_job("mac_compile_rel", cq = None)
710mac_builder("Mac64 Builder", ci_cat = None, perf_cat = "Mac|x64|Builder|")
Mirko Bonadei12643772020-03-27 13:16:35 +0100711mac_perf_builder("Perf Mac 10.11", "Mac|x64|Tester|10.11", triggered_by = ["Mac64 Builder"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100712mac_builder("Mac Asan", "Mac|x64|asan")
713mac_try_job("mac_asan")
Oleh Prypin7d1cabe2019-03-14 15:10:30 +0100714mac_try_job("mac_chromium_compile", recipe = "chromium_trybot", dimensions = {"cores": "8"}, branch_cq = False)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100715
716win_builder("Win32 Debug", "Win MSVC|x86|dbg")
717win_try_job("win_x86_msvc_dbg")
718win_try_job("win_compile_x86_msvc_dbg", cq = None)
Oleh Prypin63041b82019-03-19 22:56:01 +0000719win_builder("Win32 Release", "Win MSVC|x86|rel")
720win_try_job("win_x86_msvc_rel")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100721win_try_job("win_compile_x86_msvc_rel", cq = None)
722win_builder("Win64 Debug", "Win MSVC|x64|dbg")
723win_try_job("win_x64_msvc_dbg")
724win_try_job("win_compile_x64_msvc_dbg", cq = None)
725win_builder("Win64 Release", "Win MSVC|x64|rel")
726win_try_job("win_x64_msvc_rel")
727win_try_job("win_compile_x64_msvc_rel", cq = None)
728win_builder("Win32 Debug (Clang)", "Win Clang|x86|dbg")
729win_try_job("win_x86_clang_dbg", cq = None)
730win_try_job("win_compile_x86_clang_dbg")
731win_builder("Win32 Release (Clang)", "Win Clang|x86|rel")
732win_try_job("win_x86_clang_rel")
733win_try_job("win_compile_x86_clang_rel", cq = None)
734win_builder("Win32 Builder (Clang)", ci_cat = None, perf_cat = "Win|x86|Builder|")
Mirko Bonadei12643772020-03-27 13:16:35 +0100735win_perf_builder("Perf Win7", "Win|x86|Tester|7", triggered_by = ["Win32 Builder (Clang)"])
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100736win_builder("Win64 Debug (Clang)", "Win Clang|x64|dbg")
737win_try_job("win_x64_clang_dbg", cq = None)
738win_try_job("win_x64_clang_dbg_win8", cq = None)
739win_try_job("win_x64_clang_dbg_win10", cq = None)
740win_try_job("win_compile_x64_clang_dbg")
741win_builder("Win64 Release (Clang)", "Win Clang|x64|rel")
742win_try_job("win_x64_clang_rel", cq = None)
743win_try_job("win_compile_x64_clang_rel")
Mirko Bonadeiafbe33d2019-05-31 09:27:54 +0200744win_builder("Win64 ASan", "Win Clang|x64|asan")
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100745win_try_job("win_asan")
746win_builder("Win64 UWP", ci_cat = None, fyi_cat = "")
747win_try_job("win_x64_uwp", cq = None, try_cat = None, fyi_cat = "")
748win_builder("Win (more configs)", "Win Clang|x86|more", recipe = "more_configs")
749win_try_job("win_x86_more_configs", recipe = "more_configs")
Mirko Bonadei825f9c22020-07-01 11:52:39 +0200750win_try_job("win_chromium_compile", recipe = "chromium_trybot", branch_cq = False, goma_jobs = 150)
751win_try_job("win_chromium_compile_dbg", recipe = "chromium_trybot", branch_cq = False, goma_jobs = 150)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100752
753linux_try_job(
754 "presubmit",
755 recipe = "run_presubmit",
756 properties = {"repo_name": "webrtc", "runhooks": True},
757 priority = 28,
758 cq = {"disable_reuse": True},
759)
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100760
761cron_builder(
762 "Auto-roll - WebRTC DEPS",
763 recipe = "auto_roll_webrtc_deps",
Oleh Prypindc68a722019-06-25 10:43:13 +0200764 schedule = "0 */2 * * *", # Every 2 hours.
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100765)
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200766
767lkgr_config = {
768 "project": "webrtc",
769 "source_url": WEBRTC_GIT,
770 "status_url": "https://webrtc-status.appspot.com",
771 "allowed_lag": 300, # hours
772 "allowed_gap": 12, # commits behind
773 "error_recipients": "webrtc-troopers-robots@google.com",
Michael Achenbach09dd7dc2019-07-26 15:26:11 +0200774 "buckets": {
775 "webrtc/ci": {
776 # bucket alias: luci.webrtc.ci
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200777 "builders": sorted(lkgr_builders),
778 },
Michael Achenbach09dd7dc2019-07-26 15:26:11 +0200779 "chromium/webrtc.fyi": {
780 # bucket alias: luci.chromium.webrtc.fyi
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200781 "builders": [
782 "WebRTC Chromium FYI Android Builder (dbg)",
783 "WebRTC Chromium FYI Android Builder ARM64 (dbg)",
784 "WebRTC Chromium FYI Android Builder",
Mirko Bonadei04fe32f2020-05-01 00:45:58 +0200785 "WebRTC Chromium FYI Android Tests (dbg) (L Nexus5)",
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200786 "WebRTC Chromium FYI Android Tests (dbg) (M Nexus5X)",
787 "WebRTC Chromium FYI Linux Builder (dbg)",
788 "WebRTC Chromium FYI Linux Builder",
789 "WebRTC Chromium FYI Linux Tester",
790 "WebRTC Chromium FYI Mac Builder (dbg)",
791 "WebRTC Chromium FYI Mac Builder",
792 "WebRTC Chromium FYI Mac Tester",
793 "WebRTC Chromium FYI Win Builder (dbg)",
794 "WebRTC Chromium FYI Win Builder",
795 "WebRTC Chromium FYI Win10 Tester",
796 "WebRTC Chromium FYI Win7 Tester",
797 "WebRTC Chromium FYI Win8 Tester",
798 "WebRTC Chromium FYI ios-device",
Mirko Bonadei37071f62020-02-15 10:42:49 +0100799 "WebRTC Chromium FYI ios-simulator",
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200800 ],
801 },
802 },
803}
804
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100805cron_builder(
806 "WebRTC lkgr finder",
807 recipe = "lkgr_finder",
Oleh Prypin8f0dc312019-05-31 15:54:35 +0200808 properties = {
809 "project": "webrtc",
810 "repo": WEBRTC_GIT,
811 "ref": "refs/heads/lkgr",
812 "lkgr_status_gs_path": "chromium-webrtc/lkgr-status",
813 "config": lkgr_config,
814 },
Oleh Prypin1a0593f2019-03-11 09:43:28 +0100815 schedule = "*/10 * * * *", # Every 10 minutes.
816)