diff --git a/kokoro/caplog.sh b/kokoro/common/caplog.sh similarity index 98% rename from kokoro/caplog.sh rename to kokoro/common/caplog.sh index fe81949ea..33ee1abdf 100644 --- a/kokoro/caplog.sh +++ b/kokoro/common/caplog.sh @@ -19,7 +19,7 @@ # # Use the provided functions below as build/test fixtures, e.g.: # -# source kokoro/capture_logs.sh +# source kokoro/common/capture_logs.sh # caplog build/step1 # caplog tests/step2 # diff --git a/kokoro/common/check_missing_dist_files.sh b/kokoro/common/check_missing_dist_files.sh new file mode 100755 index 000000000..0bb26ad34 --- /dev/null +++ b/kokoro/common/check_missing_dist_files.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# +# Script to compare a distribution archive for expected files based on git. +# +# Usage: +# check_missing_dist_files.sh path/to/dist_archive.tar.gz + +set -eux +set -o pipefail + +# By default, look for a git repo based on this script's path. +: ${SOURCE_DIR:=$(cd $(dirname $0)/../.. ; pwd)} + +# Use a temporary directory for intermediate files. +# Note that pipelines below use subshells to avoid multiple trap executions. +_workdir=$(mktemp -d) +function cleanup_workdir() { rm -r ${_workdir}; } +trap cleanup_workdir EXIT + +# List all the files in the archive. +( + tar -atf $1 | \ + cut -d/ -f2- | \ + sort +) > ${_workdir}/archive.lst + +# List all files in the git repo that should be in the archive. +( + git -C ${SOURCE_DIR} ls-files | \ + grep "^\(java\|python\|objectivec\|csharp\|ruby\|php\|cmake\|examples\|src/google/protobuf/.*\.proto\)" |\ + grep -v ".gitignore" | \ + grep -v "java/lite/proguard.pgcfg" | \ + grep -v "python/compatibility_tests" | \ + grep -v "python/docs" | \ + grep -v "python/.repo-metadata.json" | \ + grep -v "python/protobuf_distutils" | \ + grep -v "csharp/compatibility_tests" | \ + sort +) > ${_workdir}/expected.lst + +# Check for missing files. +MISSING_FILES=( $(cd ${_workdir} && comm -13 archive.lst expected.lst) ) +if (( ${#MISSING_FILES[@]} == 0 )); then + exit 0 +fi + +( + set +x + echo -e "\n\nMissing files from archive:" + for (( i=0 ; i < ${#MISSING_FILES[@]} ; i++ )); do + echo " ${MISSING_FILES[i]}" + done + echo -e "\nAdd them to the `pkg_files` rule in corresponding BUILD.bazel.\n" +) >&2 +exit 1 diff --git a/kokoro/common/cmake.sh b/kokoro/common/cmake.sh new file mode 100755 index 000000000..99be3bef2 --- /dev/null +++ b/kokoro/common/cmake.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# +# Build tests under CMake. +# +# This script is used from macos and linux builds. It runs cmake and ctest in +# the current directory. Any additional setup should be done before running this +# script. +# +# This script uses `caplog` to save logfiles. See caplog.sh for details. + +set -eu -o pipefail +: ${SCRIPT_ROOT:=$(cd $(dirname $0)/../..; pwd)} + +################################################################################ +# If you are using this script to run tests, you can set some environment +# variables to control behavior: +# +# By default, find the sources based on this script's path. +: ${SOURCE_DIR:=${SCRIPT_ROOT}} +# +# By default, put outputs under /cmake/build. +: ${BUILD_DIR:=${SOURCE_DIR}/cmake/build} +# +# CMAKE_BUILD_TYPE is supported in cmake 3.22+. If set, we pass the value of this +# variable explicitly for compatibility with older versions of cmake. See: +# https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html +# (N.B.: not to be confused with CMAKE_CONFIG_TYPE.) +if [[ -n ${CMAKE_BUILD_TYPE:-} ]]; then + CMAKE_BUILD_TYPE_FLAG="-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}" +else + CMAKE_BUILD_TYPE_FLAG= +fi +# +# For several other CMake options, see docs here: +# https://cmake.org/cmake/help/latest/manual/cmake-env-variables.7.html +# +# Some variables you may want to override (see cmake docs for details): +# CMAKE_BUILD_PARALLEL_LEVEL +# CMAKE_CONFIG_TYPE (N.B.: not to be confused with CMAKE_BUILD_TYPE) +# CMAKE_GENERATOR +# CTEST_PARALLEL_LEVEL +################################################################################ + +echo "Building using..." +echo " Sources: ${SOURCE_DIR}" +echo " Build output: ${BUILD_DIR}" +if [[ ${SOURCE_DIR} != ${SCRIPT_ROOT} ]]; then + echo " Build scripts: ${SCRIPT_ROOT}" +fi +set -x +source ${SCRIPT_ROOT}/kokoro/common/caplog.sh + +# +# Configure under $BUILD_DIR: +# +mkdir -p "${BUILD_DIR}" + +( + cd "${BUILD_DIR}" + caplog 01_configure \ + cmake -S "${SOURCE_DIR}" \ + ${CMAKE_BUILD_TYPE_FLAG} \ + ${CAPLOG_CMAKE_ARGS:-} +) +if [[ -n ${CAPLOG_DIR:-} ]]; then + # Save configuration logs. + mkdir -p "${CAPLOG_DIR}/CMakeFiles" + cp "${BUILD_DIR}"/CMakeFiles/CMake*.log "${CAPLOG_DIR}/CMakeFiles" +fi + +# +# Build: +# +caplog 02_build \ + cmake --build "${BUILD_DIR}" + +# +# Run tests +# +( + cd "${BUILD_DIR}" + caplog 03_combined_testlog \ + ctest ${CAPLOG_CTEST_ARGS:-} +) diff --git a/kokoro/linux/cmake_distcheck/build.sh b/kokoro/linux/cmake_distcheck/build.sh new file mode 100755 index 000000000..5e957f8a5 --- /dev/null +++ b/kokoro/linux/cmake_distcheck/build.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# +# Build file to set up and run tests based on distribution archive + +set -eux + +# Change to repo root +cd $(dirname $0)/../../.. + +# +# Update git submodules +# +git submodule update --init --recursive + +# +# Build distribution archive +# +# TODO: this should use Bazel-built dist archives. +date ; ./autogen.sh +date ; ./configure +date ; make dist +date + +DIST_ARCHIVE=( $(ls protobuf-*.tar.gz) ) +if (( ${#DIST_ARCHIVE[@]} != 1 )); then + echo >&2 "Distribution archive not found. ${#DIST_ARCHIVE[@]} matches:" + echo >&2 "${DIST_ARCHIVE[@]}" + exit 1 +fi + +# +# Check for all expected files +# +kokoro/common/check_missing_dist_files.sh ${DIST_ARCHIVE} + +# +# Extract to a temporary directory +# +if [[ -z ${DIST_WORK_ROOT:-} ]]; then + # If you want to preserve the extracted sources, set the DIST_WORK_ROOT + # environment variable to an existing directory that should be used. + DIST_WORK_ROOT=$(mktemp -d) + function cleanup_work_root() { + echo "Cleaning up temporary directory ${DIST_WORK_ROOT}..." + rm -rf ${DIST_WORK_ROOT} + } + trap cleanup_work_root EXIT +fi + +tar -C ${DIST_WORK_ROOT} --strip-components=1 -axf ${DIST_ARCHIVE} + +# +# Run tests using extracted sources +# +if SOURCE_DIR=${DIST_WORK_ROOT} \ + CMAKE_GENERATOR=Ninja \ + CTEST_PARALLEL_LEVEL=$(nproc) \ + kokoro/common/cmake.sh; then + # TODO: remove this conditional. + # The cmake build is expected to fail due to missing abseil sources. + echo "$0: Expected failure, but build passed." >&2 + echo "Please update $(basename $0) to remove failure expectation." >&2 + echo "FAIL" >&2 + exit 1 +fi +echo "PASS" diff --git a/kokoro/linux/cmake_distcheck/continuous.cfg b/kokoro/linux/cmake_distcheck/continuous.cfg new file mode 100644 index 000000000..6ef4c890b --- /dev/null +++ b/kokoro/linux/cmake_distcheck/continuous.cfg @@ -0,0 +1,5 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/cmake_distcheck/build.sh" +timeout_mins: 1440 diff --git a/kokoro/linux/cmake_distcheck/presubmit.cfg b/kokoro/linux/cmake_distcheck/presubmit.cfg new file mode 100644 index 000000000..6ef4c890b --- /dev/null +++ b/kokoro/linux/cmake_distcheck/presubmit.cfg @@ -0,0 +1,5 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/cmake_distcheck/build.sh" +timeout_mins: 1440 diff --git a/kokoro/macos-next/cpp/build.sh b/kokoro/macos-next/cpp/build.sh index 490e99016..451c8508d 100755 --- a/kokoro/macos-next/cpp/build.sh +++ b/kokoro/macos-next/cpp/build.sh @@ -9,47 +9,21 @@ if [[ -h /tmpfs ]] && [[ ${PWD} == /tmpfs/src ]]; then cd /Volumes/BuildData/tmpfs/src fi -# These vars can be changed when running manually, e.g.: -# -# % BUILD_CONFIG=RelWithDebInfo path/to/build.sh +# Default environment variables used by cmake build: +: ${CMAKE_CONFIG_TYPE:=Debug} +export CMAKE_CONFIG_TYPE +: ${CTEST_PARALLEL_LEVEL:=4} +export CTEST_PARALLEL_LEVEL -# By default, build using Debug config. -: ${BUILD_CONFIG:=Debug} - -# By default, find the sources based on this script path. -: ${SOURCE_DIR:=$(cd $(dirname $0)/../../..; pwd)} - -# By default, put outputs under /cmake/build. -: ${BUILD_DIR:=${SOURCE_DIR}/cmake/build} - -source ${SOURCE_DIR}/kokoro/caplog.sh +# Run from the project root directory. +cd $(dirname $0)/../../.. # # Update submodules # -git -C "${SOURCE_DIR}" submodule update --init --recursive +git submodule update --init --recursive # -# Configure and build in a separate directory +# Run build # -mkdir -p "${BUILD_DIR}" - -caplog 01_configure \ - cmake -S "${SOURCE_DIR}" -B "${BUILD_DIR}" ${CAPLOG_CMAKE_ARGS:-} - -if [[ -n ${CAPLOG_DIR:-} ]]; then - mkdir -p "${CAPLOG_DIR}/CMakeFiles" - cp "${BUILD_DIR}"/CMakeFiles/CMake*.log "${CAPLOG_DIR}/CMakeFiles" -fi - -caplog 02_build \ - cmake --build "${BUILD_DIR}" --config "${BUILD_CONFIG}" - -# -# Run tests -# -( - cd "${BUILD_DIR}" - caplog 03_combined_testlog \ - ctest -C "${BUILD_CONFIG}" -j4 ${CAPLOG_CTEST_ARGS:-} -) +kokoro/common/cmake.sh