scripts/gen_luci_scheduler.py

# Copyright 2018 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# pylint: disable=line-too-long
"""Generate LUCI Scheduler config file.

This generates the LUCI Scheduler configuration file for ChromeOS builds based
on the chromeos_config contents.

Changes to chromite/config/luci-scheduler.cfg will be autodeployed:
  https://data.corp.google.com/sites/chromeos_ci_cros_ci_builds/utility/?f=board_name:in:luci-scheduler-updater

Notes:
  Normal builds are scheduled based on the builder values for
  'schedule' and 'triggered_gitiles' in config/chromeos_config.py.

  Branched builds are scheduled based on the function
  chromeos_config.BranchScheduleConfig()
"""
# pylint: enable=line-too-long

import sys

from chromite.config import chromeos_config
from chromite.lib import commandline
from chromite.lib import config_lib


_CONFIG_HEADER = """# Defines buckets on luci-scheduler.appspot.com.
#
# For schema of this file and documentation see ProjectConfig message in
# https://github.com/luci/luci-go/blob/HEAD/scheduler/appengine/messages/config.proto

# Generated with chromite/scripts/gen_luci_scheduler

# Autodeployed with:
# https://data.corp.google.com/sites/chromeos_ci_cros_ci_builds/utility/?f=board_name:in:luci-scheduler-updater

acl_sets {
  name: "default"
  acls {
    role: READER
    granted_to: "group:googlers"
  }
  acls {
    role: OWNER
    granted_to: "group:project-chromeos-admins"
  }
  acls {
    role: TRIGGERER
    granted_to: "group:mdb/chromeos-build-access"
  }
  acls {
    role: TRIGGERER
    granted_to: "group:project-chromeos-buildbucket-schedulers"
  }
}
"""

def buildJobName(build_config):
  if 'schedule_branch' in build_config:
    return '%s-%s' % (build_config.schedule_branch, build_config.name)
  else:
    return build_config.name


def genSchedulerJob(build_config):
  """Generate the luci scheduler job for a given build config.

  Args:
    build_config: config_lib.BuildConfig.

  Returns:
    Multiline string to include in the luci scheduler configuration.
  """
  job_name = buildJobName(build_config)
  if 'schedule_branch' in build_config:
    branch = build_config.schedule_branch
  else:
    branch = 'main'

  tags = {
      'cbb_branch': branch,
      'cbb_config': build_config.name,
      'cbb_display_label': build_config.display_label,
      'cbb_workspace_branch': build_config.workspace_branch,
      'cbb_goma_client_type': build_config.goma_client_type,
  }

  # Filter out tags with no value set.
  tags = {k: v for k, v in tags.items() if v}


  tag_lines = ['    tags: "%s:%s"' % (k, tags[k])
               for k in sorted(tags.keys())]
  prop_lines = ['    properties: "%s:%s"' % (k, tags[k])
                for k in sorted(tags.keys())]

  # TODO: Move --buildbot arg into recipe, and remove from here.
  template = """
job {
  id: "%(job_name)s"
  realm: "cbb-jobs"
  acl_sets: "default"
  schedule: "%(schedule)s"
  buildbucket: {
    server: "cr-buildbucket.appspot.com"
    bucket: "general"
    builder: "%(builder)s"
%(tag_lines)s
%(prop_lines)s
    properties: "cbb_extra_args:[\\"--buildbot\\"]"
  }
}
"""

  return template % {
      'job_name': job_name,
      'builder': build_config.luci_builder,
      'schedule': build_config.schedule,
      'tag_lines': '\n'.join(tag_lines),
      'prop_lines': '\n'.join(prop_lines),
  }


def genSchedulerTrigger(trigger_name, repo, refs, path_regexps, builds):
  """Generate the luci scheduler job for a given build config.

  Args:
    trigger_name: Name of the trigger as a string.
    repo: Gitiles URL git git repository.
    refs: Iterable of git refs to check. May use regular expressions.
    path_regexps: Iterable of path regular expressions of files to trigger on
        or falsy to trigger on everything.
    builds: Iterable of build config names to trigger.

  Returns:
    Multiline string to include in the luci scheduler configuration.
  """
  template = """
trigger {
  id: "%(trigger_name)s"
  realm: "cbb-jobs"
  acl_sets: "default"
  schedule: "with 5m interval"
  gitiles: {
    repo: "%(repo)s"
%(refs)s%(path_regexps)s
  }
%(triggers)s
}
"""
  if path_regexps:
    path_regexps = '\n' + '\n'.join('    path_regexps: "%s"' %
                                    r for r in path_regexps)
  else:
    path_regexps = ''
  return template % {
      'trigger_name': trigger_name,
      'repo': repo,
      'refs': '\n'.join('    refs: "%s"' % r for r in refs),
      'path_regexps': path_regexps,
      'triggers': '\n'.join('  triggers: "%s"' % b for b in builds),
  }


def genLuciSchedulerConfig(site_config, branch_config):
  """Generate a luciSchedulerConfig as a string.

  Args:
    site_config: A config_lib.SiteConfig instance.
    branch_config: A list of BuildConfig instances to schedule.

  Returns:
    The complete scheduler configuration contents as a string.
  """
  # Trigger collection is used to collect together trigger information, so
  # we can reuse the same trigger for multiple builds as needed.
  # It maps gitiles_key to a set of build_names.
  # A gitiles_key = (gitiles_url, tuple(ref_list))
  trigger_collection = {}

  jobs = []

  # Order the configs consistently.
  configs = [site_config[name] for name in sorted(site_config)] + branch_config

  for config in configs:
    # Populate jobs.
    if config.schedule:
      jobs.append(genSchedulerJob(config))

    # Populate trigger_collection.
    if config.triggered_gitiles:
      for trigger in config.triggered_gitiles:
        try:
          gitiles_url, ref_list, path_regexps = trigger
        except ValueError:
          gitiles_url, ref_list = trigger
          path_regexps = []
        gitiles_key = (gitiles_url, tuple(ref_list), tuple(path_regexps))
        trigger_collection.setdefault(gitiles_key, set())
        trigger_collection[gitiles_key].add(buildJobName(config))

  # Populate triggers.
  triggers = []
  trigger_counter = 0
  for gitiles_key in sorted(trigger_collection):
    builds = sorted(trigger_collection[gitiles_key])

    trigger_name = 'trigger_%s' % trigger_counter
    gitiles_url, refs, path_regexps = gitiles_key
    triggers.append(genSchedulerTrigger(
        trigger_name, gitiles_url, refs, path_regexps, builds))
    trigger_counter += 1

  return ''.join([_CONFIG_HEADER] + triggers + jobs)


def GetParser():
  """Creates the argparse parser."""
  parser = commandline.ArgumentParser(description=__doc__)

  parser.add_argument('-o', '--file_out', type='path',
                      help='Write output to specified file.')

  return parser


def main(argv):
  parser = GetParser()
  options = parser.parse_args(argv)
  options.Freeze()

  site_config = config_lib.GetConfig()
  branch_config = chromeos_config.BranchScheduleConfig()

  with (open(options.file_out, 'w') if options.file_out else sys.stdout) as fh:
    fh.write(genLuciSchedulerConfig(site_config, branch_config))