build_packages

#!/bin/bash

# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

. "$(dirname "$0")/common.sh" || exit 1

# Script must run inside the chroot
restart_in_chroot_if_needed "$@"

assert_not_root_user

# Developer-visible flags.
DEFINE_string board "${DEFAULT_BOARD}" \
  "The board to build packages for."
DEFINE_boolean usepkg "${FLAGS_TRUE}" \
  "Use binary packages to bootstrap when possible."
DEFINE_boolean usepkgonly "${FLAGS_FALSE}" \
  "Only use binary packages to bootstrap; abort if any are missing."
DEFINE_boolean noworkon "${FLAGS_FALSE}" \
  "Don't force-build workon packages."
DEFINE_boolean showoutput "${FLAGS_FALSE}" \
  "Show all output from parallel_emerge."
DEFINE_boolean withautotest "${FLAGS_TRUE}" \
  "Build autotest client code."
DEFINE_boolean fetchonly "${FLAGS_FALSE}" \
  "Don't build anything, instead only fetch what is needed."
DEFINE_boolean unpackonly "${FLAGS_FALSE}" \
  "Don't build anything; instead only fetch and unpack what is needed."
DEFINE_boolean withdebugsymbols "${FLAGS_FALSE}" \
  "Install the debug symbols for all packages"
DEFINE_boolean withevents "${FLAGS_FALSE}" \
  "Generate events during parallel_emerge step"
DEFINE_string eventfile "${DEFAULT_EVENT_FILE}" \
  "Define the file that event logs will be written."
DEFINE_boolean withrevdeps "${FLAGS_TRUE}" \
  "Generate install plan filename if not provided to calc reverse deps."
DEFINE_string save_install_plan "" \
  "Save install plan filename for correct reverse dependency calculation."

# The --board_root flag specifies the environment variables ROOT and PKGDIR.
# This allows fetching and emerging of all packages to specified board_root.
# Note that --board_root will setup the board normally in /build/$BOARD, if it's
# not setup yet. It also expects the toolchain to already be installed in the
# board_root. --usepkgonly and --norebuild are required, because building is not
# supported when board_root is set.
# enforce this)."
DEFINE_string board_root "" \
  "Emerge packages to board_root."

FLAGS_HELP="usage: $(basename $0) [flags] [packages]

build_packages updates the set of binary packages needed by Chrome OS. It will
cross compile all packages that have been updated into the given target's root
and build binary packages as a side-effect. The output packages will be picked
up by the build_image script to put together a bootable Chrome OS image.

If [packages] are specified, only build those specific packages (and any
dependencies they might need).

For the fastest builds, use --nowithautotest --noworkon.
"
show_help_if_requested "$@"

# The following options are advanced options, only available to those willing
# to read the source code. They are not shown in help output, since they are
# not needed for the typical developer workflow.
DEFINE_string accept_licenses "" \
  "Licenses to append to the accept list."
DEFINE_integer jobs -1 \
  "How many packages to build in parallel at maximum."
DEFINE_boolean norebuild "${FLAGS_FALSE}" \
  "Don't automatically rebuild dependencies."
DEFINE_boolean skip_chroot_upgrade "${FLAGS_FALSE}" \
  "Don't run the chroot upgrade automatically; use with care."
DEFINE_boolean skip_toolchain_update "${FLAGS_FALSE}" \
  "Don't update toolchain automatically."
DEFINE_boolean withdev "${FLAGS_TRUE}" \
  "Build useful developer friendly utilities."
DEFINE_boolean withdebug "${FLAGS_TRUE}" \
  "Build debug versions of Chromium-OS-specific packages."
DEFINE_boolean withfactory "${FLAGS_TRUE}" \
  "Build factory installer."
DEFINE_boolean withtest "${FLAGS_TRUE}" \
  "Build packages required for testing."
DEFINE_boolean chrome_binhost_only "${FLAGS_FALSE}" \
  "Only fetch packages from the Chrome binhost."
DEFINE_boolean buildretry "${FLAGS_TRUE}" \
  "Retry failed packages."

# The --reuse_pkgs_from_local_boards flag tells Portage to share binary
# packages between boards that are built locally, so that the total time
# required to build several boards is reduced. This flag is only useful
# when you are not able to use remote binary packages, since remote binary
# packages are usually more up to date than anything you have locally.
DEFINE_boolean reuse_pkgs_from_local_boards "${FLAGS_FALSE}" \
  "Bootstrap from local packages instead of remote packages."

# --run_goma option is designed to be used on bots.
# If you're trying to build pacakges with goma in your local dev env, this is
# *not* the option you're looking for. Please see comments below.
# This option; 1) starts goma, 2) builds packages (expecting that goma is
# used), then 3) stops goma explicitly.
# 3) is a request from the goma team, so that stats/logs can be taken.
# Note: GOMA_DIR and GOMA_SERVICE_ACCOUNT_JSON_FILE are expected to be passed
# via env var.
#
# In local dev env cases, compiler_proxy is expected to keep running.
# In such a case;
#   $ python ${GOMA_DIR}/goma_ctl.py ensure_start
#   $ ./build_packages (... and options without --run_goma ...)
# is an expected commandline sequence. If you set --run_goma flag while
# compiler_proxy is already running, the existing compiler_proxy will be
# stopped.
DEFINE_boolean run_goma "${FLAGS_FALSE}" \
  "If set to true, (re)starts goma, builds packages, and then stops goma."

# Parse command line
FLAGS "$@" || exit 1
eval set -- "${FLAGS_ARGV}"

# Die on any errors.
switch_to_strict_mode

# Right now build_packages has to be run from scripts/
. ${SRC_ROOT}/third_party/chromiumos-overlay/chromeos/config/chromeos_version.sh

if [[ -z "${FLAGS_board}" ]]; then
  echo "Error: --board is required."
  exit 1
fi

# Before we can run any tools, we need to update chroot or setup_board.
UPDATE_ARGS=()
if [[ -n ${FLAGS_accept_licenses} ]]; then
  UPDATE_ARGS+=( --accept_licenses "${FLAGS_accept_licenses}" )
fi
if [ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]; then
  UPDATE_ARGS+=( --usepkg )
else
  UPDATE_ARGS+=( --nousepkg )
fi
if [[ "${FLAGS_jobs}" -ne -1 ]]; then
  UPDATE_ARGS+=( --jobs=${FLAGS_jobs} )
fi
if [ "${FLAGS_chrome_binhost_only}" -eq "${FLAGS_TRUE}" ]; then
  UPDATE_ARGS+=( --chrome_binhost_only )
fi
if [ "${FLAGS_reuse_pkgs_from_local_boards}" -eq "${FLAGS_TRUE}" ]; then
  UPDATE_ARGS+=( --reuse_pkgs_from_local_boards )
fi
if [ "${FLAGS_skip_toolchain_update}" -eq "${FLAGS_TRUE}" ]; then
  UPDATE_ARGS+=( --skip_toolchain_update )
fi
if [ "${FLAGS_skip_chroot_upgrade}" -eq "${FLAGS_TRUE}" ]; then
  UPDATE_ARGS+=( --skip_chroot_upgrade )
fi
if [[ ${FLAGS_unpackonly} -eq ${FLAGS_TRUE} ]]; then
  UPDATE_ARGS+=( --skip_board_pkg_init )
fi
if [[ -n ${FLAGS_board_root} ]]; then
  UPDATE_ARGS+=( --board_root "${FLAGS_board_root}" )
fi

if [[ -z "${FLAGS_save_install_plan}" ]]; then
  if [[ "${FLAGS_withrevdeps}" -eq "${FLAGS_TRUE}" ]]; then
    # No explicit install plan filename (cbuildbot generated)
    # so generate a temporary filename and reset the flag.
    FLAGS_save_install_plan=$(mktemp)
  fi
fi
info "saving install plan to ${FLAGS_save_install_plan}"
if [[ -n "${FLAGS_save_install_plan}" ]]; then
  UPDATE_ARGS+=( --save_install_plan="${FLAGS_save_install_plan}" )
fi

"${SCRIPTS_DIR}"/setup_board --quiet --board=${FLAGS_board} "${UPDATE_ARGS[@]}"

sudo_clear_shadow_locks "/build/${FLAGS_board}"

# Setup all the emerge command/flags.
EMERGE_FLAGS=( -uDNv --backtrack=30 --select --newrepo )

EMERGE_CMD=(
  "${CHROMITE_BIN}/parallel_emerge"
  --board=${FLAGS_board}
)

if [[ "${FLAGS_fetchonly}" -eq "${FLAGS_TRUE}" ]]; then
  EMERGE_CMD+=( --fetchonly )
fi
if [[ "${FLAGS_unpackonly}" -eq "${FLAGS_TRUE}" ]]; then
  EMERGE_CMD+=( --unpackonly )
fi
if [[ "${FLAGS_buildretry}" -eq "${FLAGS_FALSE}" ]]; then
  EMERGE_CMD+=( --retries=0 )
fi

EMERGE_CMD+=( ${EXTRA_BOARD_FLAGS} )

if [[ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ||
      "${FLAGS_reuse_pkgs_from_local_boards}" -eq "${FLAGS_TRUE}" ||
      "${FLAGS_usepkgonly}" -eq "${FLAGS_TRUE}" ]]; then
  # Use binary packages. Include all build-time dependencies,
  # so as to avoid unnecessary differences between source
  # and binary builds.
  EMERGE_FLAGS+=( --getbinpkg --with-bdeps y )
  if [[ ${FLAGS_usepkgonly} -eq ${FLAGS_TRUE} ]]; then
    EMERGE_FLAGS+=( --usepkgonly )
  else
    EMERGE_FLAGS+=( --usepkg )
  fi
fi

if [[ "${FLAGS_jobs}" -ne -1 ]]; then
  EMERGE_FLAGS+=( --jobs=${FLAGS_jobs} )
fi

if [[ "${FLAGS_norebuild}" -eq "${FLAGS_FALSE}" ]]; then
  EMERGE_FLAGS+=( --rebuild-if-unbuilt )
fi
if [[ "${FLAGS_showoutput}" -eq "${FLAGS_TRUE}" ]]; then
  EMERGE_FLAGS+=( --show-output )
fi

if [[ "${FLAGS_withdebug}" -eq "${FLAGS_FALSE}" ]]; then
  export USE="${USE} -cros-debug"
fi

# TODO Handle case where passed default value, but events not enabled
if [[ "${FLAGS_eventfile}" != "${DEFAULT_EVENT_FILE}" ]]; then
  FLAGS_withevents="${FLAGS_TRUE}"
fi

if [[ "${FLAGS_withevents}" -eq "${FLAGS_TRUE}" ]]; then
  mkdir -p "$(dirname $FLAGS_eventfile)"
  EMERGE_FLAGS+=( "--eventlogfile=${FLAGS_eventfile}" )
fi

# Figure out which packages we should be building.
PACKAGES=( "$@" )
FORCE_LOCAL_BUILD_PKGS=()
if [[ $# -eq 0 ]]; then
  PACKAGES=( virtual/target-os )
  if [[ "${FLAGS_withdev}" -eq "${FLAGS_TRUE}" ]]; then
    PACKAGES+=( virtual/target-os-dev )
  fi
  if [[ "${FLAGS_withfactory}" -eq "${FLAGS_TRUE}" ]]; then
    PACKAGES+=( virtual/target-os-factory )
    PACKAGES+=( virtual/target-os-factory-shim )
  fi
  if [[ "${FLAGS_withtest}" -eq "${FLAGS_TRUE}" ]]; then
    PACKAGES+=( virtual/target-os-test )
    # chromeos-ssh-testkeys may generate ssh keys if the right USE flag is set.
    # We force rebuilding this package from source every time, so that
    # consecutive builds don't share ssh keys.
    FORCE_LOCAL_BUILD_PKGS+=( chromeos-base/chromeos-ssh-testkeys )
  fi
  if [[ "${FLAGS_withautotest}" -eq "${FLAGS_TRUE}" ]]; then
    PACKAGES+=( chromeos-base/autotest-all )
  fi
fi

# Verify that all packages can be emerged from scratch, without any
# backtracking. Only print the output if this step fails.
info "Checking package dependencies are correct: ${PACKAGES[*]}"
if ! OUTPUT=$(emerge-${FLAGS_board} -pe --backtrack=0 \
              "${PACKAGES[@]}" 2>&1); then
  printf "%s\n" "${OUTPUT}"
  die_notrace "emerge detected broken ebuilds. See error message above."
fi

# Build cros_workon packages when they are changed.
CROS_WORKON_PKGS=()
if [ "${FLAGS_noworkon}" -eq "${FLAGS_FALSE}" ]; then
  LIST_MODIFIED_PACKAGES="${CHROMITE_BIN}/cros_list_modified_packages"
  MODIFIED_PACKAGES=( $("${LIST_MODIFIED_PACKAGES}" --board=${FLAGS_board}) )
  info "cros_workon modified packages '${MODIFIED_PACKAGES[*]}' detected"
  CROS_WORKON_PKGS+=( "${MODIFIED_PACKAGES[@]}" )

  # TODO(anush): Make chrome a fake cros-workon package.
  if [[ -n "${CHROME_ORIGIN}" ]]; then
    CROS_WORKON_PKGS+=( chromeos-base/chromeos-chrome )
  fi
fi

# cros_workon packages always have to be rebuilt.
FORCE_LOCAL_BUILD_PKGS+=( "${CROS_WORKON_PKGS[@]}" )

if [[ -n "${FLAGS_board_root}" ]]; then
  export ROOT="${FLAGS_board_root}"
  export PORTAGE_CONFIGROOT="${ROOT}"
  export SYSROOT="${ROOT}"
  export PKGDIR="${ROOT}"/packages
fi

# Temporarily modify the emerge flags so we can calculate the revdeps
# on the modified packages.
SIM_EMERGE_FLAGS=( "${EMERGE_FLAGS[@]}" )
if [[ ${#PACKAGES[@]} -gt 0 ]]; then
  SIM_EMERGE_FLAGS+=(
    --reinstall-atoms="${PACKAGES[*]}"
    --usepkg-exclude="${PACKAGES[*]}"
  )
fi

sudo -E "${EMERGE_CMD[@]}" "${SIM_EMERGE_FLAGS[@]}" --pretend "${PACKAGES[@]}"

MOD_PKGS=()
if [[ -n "${FLAGS_save_install_plan}" ]]; then
  [ -f ${FLAGS_save_install_plan} ] \
    || die "${FLAGS_save_install_plan} does not exist"
  readarray NEW_DEPS < ${FLAGS_save_install_plan}
  if [[ ${#NEW_DEPS[@]} -gt 0 ]]; then
    # Convert specific versions into base package names
    MOD_PKGS+=( $(\
      equery-${FLAGS_board} list -p -o --format='$category/$name' \
          ${NEW_DEPS[*]}) )
  fi
  FORCE_LOCAL_BUILD_PKGS+=( ${MOD_PKGS[@]} )
fi

if [[ ${#MOD_PKGS[@]} -gt 0 ]]; then
  info "calculating reverse dependencies on packages: ${MOD_PKGS[*]}"
  REV_DEPS=( $(\
        equery-${FLAGS_board} -q depends --indirect ${MOD_PKGS[@]} |\
        awk '{print $1}' | grep -v ^virtual/ | sort -u) )
  if [[ ${#REV_DEPS[@]} -gt 0 ]]; then
    # Convert specific versions into base package names
    RMOD_PKGS=( $(\
        equery-${FLAGS_board} -q list -p -o --format='$category/$name' \
        ${REV_DEPS[*]}) )
    # Remove Chrome as rebuilding it is expensive and almost never makes sense
    RMOD_PKGS=( $( echo "${RMOD_PKGS[*]}" |\
        sed -e 's/chromeos-base\/chromeos-chrome[^ ]*//' ) )
    info "final reverse dependencies that will be rebuilt: ${RMOD_PKGS[*]}"
    FORCE_LOCAL_BUILD_PKGS+=( ${RMOD_PKGS[@]} )
  fi
fi

if [[ ${#FORCE_LOCAL_BUILD_PKGS[@]} -gt 0 ]]; then
  EMERGE_FLAGS+=(
    --reinstall-atoms="${FORCE_LOCAL_BUILD_PKGS[*]}"
    --usepkg-exclude="${FORCE_LOCAL_BUILD_PKGS[*]}"
  )
fi

# Prepare tmp file to capture emerge output from tee.
tmpfile=$(mktemp --tmpdir tmp.build_packages-emerge.XXXXXX)
trap "rm -f '${tmpfile}'" EXIT

info "Merging board packages now"
(
  # Support goma on bots. This has to run in subshell, otherwise EXIT trap
  # handler is overwritten.
  if [[ "${FLAGS_run_goma}" -eq "${FLAGS_TRUE}" ]]; then
    info "Starting goma compiler_proxy."
    goma_ctl="${GOMA_DIR:-${HOME}/goma}/goma_ctl.py"
    "${goma_ctl}" restart
    trap "'${goma_ctl}' stop" EXIT
  fi

  set -o pipefail
  sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" "${PACKAGES[@]}" | \
    tee "${tmpfile}"
)

# Extract total package count from emerge output.
package_count=$(awk '$0 ~ /^Total: [0-9]+ packages/ { print $2 }' "${tmpfile}")
rm "${tmpfile}"
trap - EXIT

if [[ -n "${FLAGS_save_install_plan}" ]]; then
  sudo -E rm ${FLAGS_save_install_plan}
fi

echo "Builds complete"

if [[ ${FLAGS_withdebugsymbols} -eq ${FLAGS_TRUE} ]]; then
  info "fetching the debug symbols"
  sudo -E "${CHROMITE_BIN}/cros_install_debug_syms" \
    "--board=${FLAGS_board}" "--all"
fi

EXTRA_COMMAND_STATS[package_count]=${package_count}
command_completed
echo "Done"