[Kokoro/Bazel] Send Bazel build results to BES. (#10008)
This change adds bazel_wrapper.sh, which uses values set by Kokoro to log remotely.
This commit is contained in:
parent
e5e8378a62
commit
bbcc459aa0
71
kokoro/common/bazel_wrapper.sh
Executable file
71
kokoro/common/bazel_wrapper.sh
Executable file
@ -0,0 +1,71 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Wrapper for invoking bazel on Kokoro.
|
||||
#
|
||||
# This script adds extra flags to a bazel invocation when it is run from Kokoro.
|
||||
# When the special environment variables are not present (e.g., if you run
|
||||
# Kokoro build scripts locally), this script is equivalent to the "bazel"
|
||||
# command.
|
||||
#
|
||||
# Example of running directly:
|
||||
# path/to/bazel_wrapper.sh build //...
|
||||
#
|
||||
# Example of `source`ing:
|
||||
# source path/to/bazel_wrapper.sh
|
||||
# bazel_wrapper build //...
|
||||
|
||||
function bazel_wrapper::gen_invocation_id() {
|
||||
# Create a new invocation ID and store in the artifacts dir.
|
||||
local _invocation_id=$(uuidgen | tr A-Z a-z)
|
||||
|
||||
# Put the new invocation ID at the start of the output IDs file. Some
|
||||
# Google-internal tools only look at the first entry, so this ensures the most
|
||||
# recent entry is first.
|
||||
local _ids_file=${KOKORO_ARTIFACTS_DIR}/bazel_invocation_ids
|
||||
local _temp_ids=$(mktemp)
|
||||
echo ${_invocation_id} > ${_temp_ids}
|
||||
[[ -e ${_ids_file} ]] && cat ${_ids_file} >> ${_temp_ids}
|
||||
mv -f ${_temp_ids} ${_ids_file}
|
||||
|
||||
echo -n ${_invocation_id}
|
||||
}
|
||||
|
||||
# Prints flags to use on Kokoro.
|
||||
function bazel_wrapper::kokoro_flags() {
|
||||
[[ -n ${KOKORO_BES_PROJECT_ID:-} ]] || return
|
||||
|
||||
local -a _flags
|
||||
_flags+=(
|
||||
--bes_backend=${KOKORO_BES_BACKEND_ADDRESS:-buildeventservice.googleapis.com}
|
||||
--bes_results_url=https://source.cloud.google.com/results/invocations/
|
||||
--invocation_id=$(bazel_wrapper::gen_invocation_id)
|
||||
--project_id=${KOKORO_BES_PROJECT_ID} # --bes_instance_name in Bazel 5+
|
||||
--remote_cache=https://storage.googleapis.com/protobuf-bazel-cache
|
||||
)
|
||||
if [[ -n ${KOKORO_BAZEL_AUTH_CREDENTIAL:-} ]]; then
|
||||
_flags+=( --google_credentials=${KOKORO_BAZEL_AUTH_CREDENTIAL} )
|
||||
else
|
||||
_flags+=( --google_default_credentials=true )
|
||||
fi
|
||||
|
||||
echo "${_flags[@]}"
|
||||
}
|
||||
|
||||
# Runs bazel with Kokoro flags, if appropriate.
|
||||
function bazel_wrapper() {
|
||||
local -a _flags
|
||||
|
||||
# We might need to add flags. They need to come after any startup flags and
|
||||
# the command, but before any terminating "--", so copy them into the _flags
|
||||
# variable.
|
||||
until (( ${#@} == 0 )) || [[ $1 == "--" ]]; do
|
||||
_flags+=( "${1}" ); shift
|
||||
done
|
||||
|
||||
# Set the `BAZEL` env variable to override the actual bazel binary to use:
|
||||
${BAZEL:=bazel} "${_flags[@]}" $(bazel_wrapper::kokoro_flags) "${@}"
|
||||
}
|
||||
|
||||
# If this script was called directly, run bazel. Otherwise (i.e., this script
|
||||
# was `source`d), the sourcing script will call bazel_wrapper themselves.
|
||||
(( ${#BASH_SOURCE[@]} == 1 )) && bazel_wrapper "${@}"
|
@ -1,7 +1,7 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Build file to set up and run tests
|
||||
set -ex
|
||||
set -eu
|
||||
|
||||
# Install Bazel 4.0.0.
|
||||
use_bazel.sh 4.0.0
|
||||
@ -9,28 +9,36 @@ bazel version
|
||||
|
||||
# Change to repo root
|
||||
cd $(dirname $0)/../../..
|
||||
source kokoro/common/pyenv.sh
|
||||
|
||||
git submodule update --init --recursive
|
||||
# Get kokoro scripts from repo root by default.
|
||||
: ${SCRIPT_ROOT:=$(pwd)}
|
||||
source ${SCRIPT_ROOT}/kokoro/common/pyenv.sh
|
||||
|
||||
# Disabled for now, re-enable if appropriate.
|
||||
# //:build_files_updated_unittest \
|
||||
|
||||
bazel test \
|
||||
-k --copt=-Werror --host_copt=-Werror --test_output=errors \
|
||||
//build_defs:all \
|
||||
//java:tests \
|
||||
//src/... \
|
||||
//:protobuf_python \
|
||||
bazel_args=(
|
||||
test
|
||||
--keep_going
|
||||
--copt=-Werror
|
||||
--host_copt=-Werror
|
||||
--test_output=errors
|
||||
--
|
||||
//...
|
||||
-//objectivec/... # only works on macOS
|
||||
@com_google_protobuf_examples//...
|
||||
)
|
||||
|
||||
${SCRIPT_ROOT}/kokoro/common/bazel_wrapper.sh "${bazel_args[@]}"
|
||||
|
||||
# Verify that we can build successfully from generated tar files.
|
||||
(
|
||||
pyenv versions
|
||||
pyenv shell 2.7.9 # python2 required for old googletest autotools support
|
||||
git submodule update --init --recursive
|
||||
./autogen.sh && ./configure && make -j$(nproc) dist
|
||||
)
|
||||
DIST=`ls *.tar.gz`
|
||||
tar -xf $DIST
|
||||
cd ${DIST//.tar.gz}
|
||||
bazel build //:protobuf //:protobuf_java
|
||||
${SCRIPT_ROOT}/kokoro/common/bazel_wrapper.sh build //:protobuf //:protobuf_java
|
||||
|
9
kokoro/linux/bazel/common.cfg
Normal file
9
kokoro/linux/bazel/common.cfg
Normal file
@ -0,0 +1,9 @@
|
||||
# Common config shared by presubmit and continuous.
|
||||
|
||||
bazel_setting: {
|
||||
project_id: "protobuf-build"
|
||||
bes_backend_address: "buildeventservice.googleapis.com"
|
||||
foundry_backend_address: "remotebuildexecution.googleapis.com"
|
||||
upsalite_frontend_address: "https://source.cloud.google.com"
|
||||
local_execution: true
|
||||
}
|
@ -5,64 +5,56 @@
|
||||
# Note that the builds use WORKSPACE to fetch external sources, not
|
||||
# git submodules.
|
||||
|
||||
set -eux
|
||||
|
||||
BUILD_ONLY_TARGETS=(
|
||||
//pkg:all
|
||||
//:protoc
|
||||
//:protobuf
|
||||
//:protobuf_python
|
||||
)
|
||||
|
||||
TEST_TARGETS=(
|
||||
//build_defs:all
|
||||
//conformance:all
|
||||
//java:tests
|
||||
//python:all
|
||||
//src/...
|
||||
@com_google_protobuf_examples//...
|
||||
)
|
||||
set -eu
|
||||
|
||||
use_bazel.sh 5.0.0 || true
|
||||
bazel version
|
||||
|
||||
# Change to repo root
|
||||
cd $(dirname $0)/../../..
|
||||
source kokoro/common/pyenv.sh
|
||||
|
||||
# Get kokoro scripts from repo root by default.
|
||||
: ${SCRIPT_ROOT:=$(pwd)}
|
||||
source ${SCRIPT_ROOT}/kokoro/common/pyenv.sh
|
||||
|
||||
# Build distribution archive
|
||||
date
|
||||
bazel build //pkg:dist_all_tar
|
||||
echo "============================================================"
|
||||
echo -e "[[ $(date) ]] Building distribution archive...\n"
|
||||
${SCRIPT_ROOT}/kokoro/common/bazel_wrapper.sh build //pkg:dist_all_tar
|
||||
DIST_ARCHIVE=$(readlink $(bazel info bazel-bin)/pkg/dist_all_tar.tar.gz)
|
||||
bazel shutdown
|
||||
|
||||
# Extract the dist archive.
|
||||
echo "============================================================"
|
||||
echo -e "[[ $(date) ]] Extracting distribution archive...\n"
|
||||
|
||||
# Construct temp directory for running the dist build.
|
||||
# If you want to run locally and keep the build dir, create a directory
|
||||
# and pass it in the DIST_WORK_ROOT env var.
|
||||
if [[ -z ${DIST_WORK_ROOT:-} ]]; then
|
||||
: ${DIST_WORK_ROOT:=$(mktemp -d)}
|
||||
function dist_cleanup() {
|
||||
rm -rf ${DIST_WORK_ROOT}
|
||||
(( $BASH_SUBSHELL == 0 )) && rm -rf ${DIST_WORK_ROOT}
|
||||
}
|
||||
trap dist_cleanup EXIT
|
||||
fi
|
||||
|
||||
# Extract the dist archive.
|
||||
date
|
||||
DIST_WORKSPACE=${DIST_WORK_ROOT}/protobuf
|
||||
mkdir -p ${DIST_WORKSPACE}
|
||||
tar -C ${DIST_WORKSPACE} --strip-components=1 -axf bazel-bin/pkg/dist_all_tar.tar.gz
|
||||
|
||||
# Perform build steps in the extracted dist sources.
|
||||
echo "============================================================"
|
||||
echo -e "[[ $(date) ]] Building extracted archive...\n"
|
||||
|
||||
cd ${DIST_WORKSPACE}
|
||||
FAILED=false
|
||||
|
||||
date
|
||||
bazel test --test_output=errors -k \
|
||||
"${BUILD_ONLY_TARGETS[@]}" "${TEST_TARGETS[@]}" || FAILED=true
|
||||
|
||||
if ${FAILED}; then
|
||||
echo FAILED
|
||||
exit 1
|
||||
fi
|
||||
echo PASS
|
||||
bazel_args=(
|
||||
test
|
||||
--keep_going
|
||||
--test_output=errors
|
||||
--
|
||||
//...
|
||||
-//objectivec/... # only works on macOS
|
||||
@com_google_protobuf_examples//...
|
||||
)
|
||||
${SCRIPT_ROOT}/kokoro/common/bazel_wrapper.sh "${bazel_args[@]}"
|
||||
|
9
kokoro/linux/bazel_distcheck/common.cfg
Normal file
9
kokoro/linux/bazel_distcheck/common.cfg
Normal file
@ -0,0 +1,9 @@
|
||||
# Common config shared by presubmit and continuous.
|
||||
|
||||
bazel_setting: {
|
||||
project_id: "protobuf-build"
|
||||
bes_backend_address: "buildeventservice.googleapis.com"
|
||||
foundry_backend_address: "remotebuildexecution.googleapis.com"
|
||||
upsalite_frontend_address: "https://source.cloud.google.com"
|
||||
local_execution: true
|
||||
}
|
Loading…
Reference in New Issue
Block a user