contrib/update_program_fw

#!/bin/bash
# Copyright 2020 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# Script to update base firmware in a program's config, and then
# regenerate the configs of projects that are part of the program.
# Also updates the firmware manifest, and uploads all of the changes
# for review.
#
# Usage:
#   ./update_program_fw --board=program --release=NNNNN [ --reviewer=reviewer ]
#                       [ --project=proj... ]
# E.g:
#   ./update_program_fw --board=puff --release=13291 --reviewer=amcrae@google.com
#
CONTRIB_DIR=$(dirname "$(readlink -f "$0")")
. "${CONTRIB_DIR}/common.sh" || exit 1

FLAGS_HELP="
Command to update the firmware version for a board.

Updates the master_version firmware configuration for a board's
master configuration (program.star) and for selected projects
that include the master configuration.

If no projects are specified, all projects for that board are selected.
An optional skip list can be specified to skip selected boards.

The configurations for the selected projects are regenerated,
and the firmware manifest are updated for the projects.

The necessary CLs for these changes are created and uploaded for review.
An optional reviewer can be specified to send all the CLs to.
"
# Flags
DEFINE_string board "${DEFAULT_BOARD}" "Which board (program) the firmware is for" b
DEFINE_integer release 0  "The firmware release to update to" r
DEFINE_string project "${DEFAULT_PROJECT}" "Which projects this release is for (defaults to all)" p
DEFINE_string skip "${DEFAULT_SKIP}" "Skip these projects" s
DEFINE_string reviewer "${DEFAULT_REVIEWER}" "The reviewer to send the CLs to (optional)"

# Parse command line
FLAGS "$@" || exit 1
eval set -- "${FLAGS_ARGV}"
set -e

# Script must be run inside the chroot.
assert_inside_chroot
#
# Variables
#
PATH="${PATH}:${GCLIENT_ROOT}/src/config/bin"
DIGITS="[1-9][0-9][0-9][0-9][0-9]"
BRANCH=""
PROGRAM_CL=""
PROGRAM="program.star"
#
# Common functions
#
cleanup() {
  if [[ -d "${TEMPDIR}" ]]; then
    rm -rf "${TEMPDIR}"
  fi
}
#
# Abort the update, and clean up branches and CLs
#
abort() {
  CLS=$(gerrit -i --raw search "owner:me status:open hashtag:${BRANCH}")
  if [[ -n "${CLS}" ]]; then
    echo "Abandoning uploaded CLs"
    for cl in ${CLS}; do
      gerrit -i abandon "${cl}"
    done
  fi
  "cros_workon-${FLAGS_board}" stop "chromeos-base/chromeos-firmware-${FLAGS_board}"
  "cros_workon-${FLAGS_board}" stop "chromeos-base/chromeos-config-bsp-${FLAGS_board}-private"
  repo abandon "${BRANCH}"
  die "$*"
}
#
# Extract a CL number from the file containing
# the output of repo upload
#
getcl() {
  CL=$(grep -o "https://chrome-internal-review.googlesource.com/c/chromeos/$1/+/[0-9][0-9]*" "$2")
  if [[ -z "${CL}" ]]; then
    cat "$2"
    abort CL number not found in repo upload output
  fi
  echo "${CL}" | grep -o "[0-9][0-9]*"
}
#
# If not on this branch, start a branch
#
branch() {
  if ! (git branch --show-current | grep -q "${BRANCH}"); then
     repo start "${BRANCH}"
  else
     echo "${BRANCH} already exists, skipping repo start"
  fi
}
#
# Return true if repo has changes.
changed() {
  [[ -n $(git status -s) ]]
}
#
# Add a Cq-Depend line to a commit.
#
amend() {
  git log -1 --pretty=%B > "${TEMPDIR}/amend-msg"
  sed -i "/^Change-Id/ i ${1}" "${TEMPDIR}/amend-msg"
  git commit -q --amend -F "${TEMPDIR}/amend-msg"
}
#
# Return true if $1 is in list $2
#
in_list() {
  for S in ${2}; do
     if [[ "$1" == "${S}" ]]; then
       return 0
     fi
  done
  return 1
}
#
# Validate arguments
#
if [[ -z "${FLAGS_board}" ]]; then
  die "-b or --board required."
fi
if [[ -z "${FLAGS_release}" ]]; then
  die "-r or --release required."
fi
#
# Program must exist as a directory
#
PROGDIR="${GCLIENT_ROOT}/src/program/${FLAGS_board}"
if [[ ! -d "${PROGDIR}" ]]; then
  die "${FLAGS_board} is not a valid program (${PROGDIR} missing)"
fi
# Release must be a 5 digit number
if [[ ! "${FLAGS_release}" =~ ^${DIGITS}$ ]]; then
  die "release must be a 5 digit number"
fi
# Use a common git branch name.
BRANCH="update_${FLAGS_board}_fw_${FLAGS_release}"
#
# Build the project list.
# If no projects are specified, use all in the programs directory.
#
if [[ -z "${FLAGS_project}" ]]; then
  BDIR="${GCLIENT_ROOT}/src/project/${FLAGS_board}"
  cd "${BDIR}"
  mapfile -t PROJLIST < <(ls)
else
  IFS=',' read -r -a PROJLIST <<< "${FLAGS_project}"
fi
#
# Filter out the projects that are to be skipped.
#
if [[ -n "${FLAGS_skip}" ]]; then
  PROJECTS=()
  IFS=',' read -r -a SKIP_ARRAY <<< "${FLAGS_skip}"
  SKIPPED="${SKIP_ARRAY[*]}"
  for P in "${PROJLIST[@]}"; do
    if ! (in_list "${P}" "${SKIPPED}"); then
      PROJECTS+=("${P}")
    fi
  done
else
  PROJECTS=("${PROJLIST[@]}")
fi
#
# Validate project list and file locations.
#
for P in "${PROJECTS[@]}"; do
  PDIR="${GCLIENT_ROOT}/src/project/${FLAGS_board}/${P}"
  if [[ ! -d "${PDIR}" ]]; then
    die "${P} is not a valid project (${PDIR} missing)"
  fi
done
# Validate project overlay location
OVERLAY="${GCLIENT_ROOT}/src/private-overlays/overlay-${FLAGS_board}-private/chromeos-base/chromeos-firmware-${FLAGS_board}"
if [[ ! -d "${OVERLAY}" ]]; then
  die "${OVERLAY}: invalid directory"
fi
# Validate ebuild file
EB9999="chromeos-firmware-${FLAGS_board}-9999.ebuild"
if [[ ! -f "${OVERLAY}/${EB9999}" ]]; then
  die "${OVERLAY}/${EB9999}: missing file"
fi
# Make sure dev/contrib is accessible
DEVCONTRIB="${GCLIENT_ROOT}/src/platform/dev/contrib"
if [[ ! -d "${DEVCONTRIB}" ]]; then
  die "${DEVCONTRIB}: invalid directory"
fi
#
# Create a temp directory.
TEMPDIR=$(mktemp -d -t fw-XXXXXXXXXX)

trap "exit 1"           HUP INT PIPE QUIT TERM
trap 'cleanup' EXIT

#
# Update the firmware version in the program config
# From now on, all errors should invoke 'abort'
# so that the branches and CLs are cleaned up on exit.
#
cd "${PROGDIR}"
echo "Updating ${PROGRAM} for board ${FLAGS_board}"
branch
sed "/^  *major_version = ${DIGITS}$/s/${DIGITS}/${FLAGS_release}/" "${PROGRAM}" > "${TEMPDIR}/new-${PROGRAM}"
#
# Verify that only 1-5 characters have changed.
#
DIFF=$(cmp -l "${PROGRAM}" "${TEMPDIR}/new-${PROGRAM}" | wc -l)
if [[ "${DIFF}" -gt 5 ]]; then
  diff "${PROGRAM}" "new-${TEMPDIR}/${PROGRAM}"
  abort "${PROGDIR}/${PROGRAM} update error"
fi
#
# If program config has changed, create a CL.
#
if [[ "${DIFF}" -ne 0 ]]; then
  cp "${TEMPDIR}/new-${PROGRAM}" "${PROGRAM}"
  git add .
  git commit -q -F - <<EOF
${FLAGS_board}: Update firmware to ${FLAGS_release}

BUG=none
TEST=FAFT tests on ${FLAGS_board}
EOF
  if ! repo upload -y "--ht=${BRANCH}" --cbr . > "${TEMPDIR}/upload.output" 2>&1 ;then
    cat  "${TEMPDIR}/upload.output"
    abort "repo upload failed"
  fi
  PROGRAM_CL=$(getcl "program/${FLAGS_board}" "${TEMPDIR}/upload.output")
fi
#
# Now walk through the projects and regenerate the configs.
# Create and upload a CL and capture the CL number and project directory
# if the project has changed.
#
PROJ_CLS=()
PROJ_DIRS=()
for PROJ in "${PROJECTS[@]}"; do
  echo "Updating configs for project ${PROJ}"
  PDIR="${GCLIENT_ROOT}/src/project/${FLAGS_board}/${PROJ}"
  cd "${PDIR}"
  branch
  ./config.star || abort "Generate config failed for ${PROJ}"
  check_config > "${TEMPDIR}/check_config-${PROJ}.output" || abort "check_config failed for ${PROJ}"
  #
  # Check if any files changed.
  #
  if changed; then
    echo "Creating CL for changes to project ${PROJ}"
    git add .
    git commit -q -F - <<EOF
${PROJ}: Update firmware to ${FLAGS_release}

BUG=none
TEST=FAFT tests on ${FLAGS_board}
EOF
    if ! repo upload -y "--ht=${BRANCH}" --cbr . > "${TEMPDIR}/upload.${PROJ}.output" 2>&1 ;then
      cat  "${TEMPDIR}/upload.${PROJ}.output"
      abort "repo upload failed"
    fi
    P_CL=$(getcl "project/${FLAGS_board}/${PROJ}" "${TEMPDIR}/upload.${PROJ}.output")
    PROJ_CLS+=("${P_CL}")
    PROJ_DIRS+=("${PDIR}")
  fi
done
#
# Create a Cq-Depend line with all the project CLs
#
if [[ -n "${PROJ_CLS[*]}" ]];then
  SEP=" "
  PROG_CQD="Cq-Depend:"
  for CL in "${PROJ_CLS[@]}"; do
    PROG_CQD="${PROG_CQD}${SEP}chrome-internal:${CL}"
    SEP=", "
  done
  #
  # If a program CL exists, add the Cq-Depend line to it.
  #
  if [[ -n "${PROGRAM_CL}" ]]; then
    cd "${PROGDIR}"
    amend "${PROG_CQD}"
    if ! repo upload -y "--ht=${BRANCH}" --cbr . > "${TEMPDIR}/upload.amend.output" 2>&1 ;then
      cat  "${TEMPDIR}/upload.amend.output"
      abort "repo upload failed"
    fi
  fi
fi
#
# All the boxster configs have been uploaded.
# Now run the update script and update the firmware manifest.
#
# Build base coreboot files
# TODO: Should be selective here.
#
echo "Running emerge for coreboot. This may take a while..."
if ! ("emerge-${FLAGS_board}" --quiet-build chromeos-ec coreboot depthcharge vboot_reference \
                 libpayload chromeos-bootimage coreboot-private-files \
                 "coreboot-private-files-${FLAGS_board}"); then
  abort "emerge for coreboot failed!"
fi
echo "emerge of coreboot successful"
EB9999="chromeos-firmware-${FLAGS_board}-9999.ebuild"
#
# Remove any previous attempts to build the firmware.
#
"cros_workon-${FLAGS_board}" stop "chromeos-base/chromeos-firmware-${FLAGS_board}"
"cros_workon-${FLAGS_board}" stop "chromeos-base/chromeos-config-bsp-${FLAGS_board}-private"
cd "${OVERLAY}"
branch
cd "${DEVCONTRIB}"
if ! (./cros_update_firmware -q "--board=${FLAGS_board}"); then
  abort "cros_update_firmware failed for ${FLAGS_board}"
fi
cd "${OVERLAY}"
#
# If files have been updated, then create a CL for the changes.
#
OVERLAY_CL=""
if changed; then
  #
  # Bump the version in the ebuild file. Relies on the format
  # of the version so that the last number is at the end of the line.
  #
  CURVERS=$(grep "VERSION=REVBUMP" "${EB9999}" | grep -o "[0-9][0-9]*$")
  NEXTVERS=$((CURVERS + 1))
  sed -i "/VERSION=REVBUMP/s/${CURVERS}$/${NEXTVERS}/" "${EB9999}"
  git add .
  git commit -q -F - <<EOF
${FLAGS_board}: Update firmware to ${FLAGS_release}

BUG=none
TEST=FAFT tests on ${FLAGS_board}

${PROG_CQD}
EOF
  #
  # Upload with no-verify since the file lines are too long.
  #
  if ! repo upload "--ht=${BRANCH}" -y --no-verify --cbr . > "${TEMPDIR}/overlay.output" 2>&1 ;then
    cat  "${TEMPDIR}/overlay.output"
    abort "repo upload failed"
  fi
  OVERLAY_CL=$(getcl "overlays/overlay-${FLAGS_board}-private" "${TEMPDIR}/overlay.output")
  #
  # Go back and amend all the project commit messages with a Cq-Depend on
  # the program and overlay CLs.
  #
  CQD="Cq-Depend: chrome-internal:${OVERLAY_CL}"
  if [[ -n "${PROGRAM_CL}" ]]; then
    CQD="${CQD}, chrome-internal:${PROGRAM_CL}"
  fi
  for DIR in "${PROJ_DIRS[@]}"; do
    cd "${DIR}"
    amend "${CQD}"
    if ! repo upload -y --cbr . > "${TEMPDIR}/cqd.output" 2>&1 ;then
      cat  "${TEMPDIR}/cqd.output"
      abort "repo upload failed"
    fi
  done
fi
#
# Send all of the CLs to the CQ for a dry run.
#
ALL_CLS=$(gerrit -i --raw search "owner:me status:open hashtag:${BRANCH}")
if [[ -z "${ALL_CLS}" ]]; then
  echo "No changes required for program ${FLAGS_board}"
  repo abandon "${BRANCH}"
  exit 0
fi
for cl in ${ALL_CLS}; do
  gerrit -i label-cq "${cl}" 1
  gerrit -i label-v "${cl}" 1
  gerrit -i label-as "${cl}" 1
done
#
# If reviewer is set, then add them to the CLs
#
if [[ -n "${FLAGS_reviewer}" ]]; then
  echo "Sending CLs ${ALL_CLS} to ${FLAGS_reviewer} for review"
  for cl in ${ALL_CLS}; do
    gerrit -i reviewers "${cl}" "${FLAGS_reviewer}"
  done
else
  echo "Send CLs for review by running:"
  echo "   for cl in ${ALL_CLS}; do gerrit -i reviewers \$cl <reviewer>; done"
fi
#
# Final instructions.
#
echo "Run:"
echo "  /build/${FLAGS_board}/usr/sbin/chromeos-firmwareupdate --manifest"
echo "to verify firmware update"
echo "When submitted, cleanup by running:"
echo "repo abandon ${BRANCH}"