[Kokoro] Add dist check build for linux+cmake (#9939)

The cmake tests are expected to fail for now due to Abseil sources missing from the distribution artifact. The tests are structured as expected failures.

kokoro/common/caplog.sh

@@ -19,7 +19,7 @@
 #
 # Use the provided functions below as build/test fixtures, e.g.:
 #
-#   source kokoro/capture_logs.sh
+#   source kokoro/common/capture_logs.sh
 #   caplog build/step1 <build command>
 #   caplog tests/step2 <test command>
 #

kokoro/common/check_missing_dist_files.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+#
+# Script to compare a distribution archive for expected files based on git.
+#
+# Usage:
+#   check_missing_dist_files.sh path/to/dist_archive.tar.gz
+
+set -eux
+set -o pipefail
+
+# By default, look for a git repo based on this script's path.
+: ${SOURCE_DIR:=$(cd $(dirname $0)/../.. ; pwd)}
+
+# Use a temporary directory for intermediate files.
+# Note that pipelines below use subshells to avoid multiple trap executions.
+_workdir=$(mktemp -d)
+function cleanup_workdir() { rm -r ${_workdir}; }
+trap cleanup_workdir EXIT
+
+# List all the files in the archive.
+(
+  tar -atf $1 | \
+    cut -d/ -f2- | \
+    sort
+) > ${_workdir}/archive.lst
+
+# List all files in the git repo that should be in the archive.
+(
+  git -C ${SOURCE_DIR} ls-files | \
+  grep "^\(java\|python\|objectivec\|csharp\|ruby\|php\|cmake\|examples\|src/google/protobuf/.*\.proto\)" |\
+  grep -v ".gitignore" | \
+  grep -v "java/lite/proguard.pgcfg" | \
+  grep -v "python/compatibility_tests" | \
+  grep -v "python/docs" | \
+  grep -v "python/.repo-metadata.json" | \
+  grep -v "python/protobuf_distutils" | \
+  grep -v "csharp/compatibility_tests" | \
+  sort
+) > ${_workdir}/expected.lst
+
+# Check for missing files.
+MISSING_FILES=( $(cd ${_workdir} && comm -13 archive.lst expected.lst) )
+if (( ${#MISSING_FILES[@]} == 0 )); then
+  exit 0
+fi
+
+(
+  set +x
+  echo -e "\n\nMissing files from archive:"
+  for (( i=0 ; i < ${#MISSING_FILES[@]} ; i++ )); do
+    echo "  ${MISSING_FILES[i]}"
+  done
+  echo -e "\nAdd them to the `pkg_files` rule in corresponding BUILD.bazel.\n"
+) >&2
+exit 1

kokoro/common/cmake.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+#
+# Build tests under CMake.
+#
+# This script is used from macos and linux builds. It runs cmake and ctest in
+# the current directory. Any additional setup should be done before running this
+# script.
+#
+# This script uses `caplog` to save logfiles. See caplog.sh for details.
+
+set -eu -o pipefail
+: ${SCRIPT_ROOT:=$(cd $(dirname $0)/../..; pwd)}
+
+################################################################################
+# If you are using this script to run tests, you can set some environment
+# variables to control behavior:
+#
+# By default, find the sources based on this script's path.
+: ${SOURCE_DIR:=${SCRIPT_ROOT}}
+#
+# By default, put outputs under <git root>/cmake/build.
+: ${BUILD_DIR:=${SOURCE_DIR}/cmake/build}
+#
+# CMAKE_BUILD_TYPE is supported in cmake 3.22+. If set, we pass the value of this
+# variable explicitly for compatibility with older versions of cmake. See:
+#   https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html
+# (N.B.: not to be confused with CMAKE_CONFIG_TYPE.)
+if [[ -n ${CMAKE_BUILD_TYPE:-} ]]; then
+  CMAKE_BUILD_TYPE_FLAG="-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
+else
+  CMAKE_BUILD_TYPE_FLAG=
+fi
+#
+# For several other CMake options, see docs here:
+#   https://cmake.org/cmake/help/latest/manual/cmake-env-variables.7.html
+#
+# Some variables you may want to override (see cmake docs for details):
+#   CMAKE_BUILD_PARALLEL_LEVEL
+#   CMAKE_CONFIG_TYPE (N.B.: not to be confused with CMAKE_BUILD_TYPE)
+#   CMAKE_GENERATOR
+#   CTEST_PARALLEL_LEVEL
+################################################################################
+
+echo "Building using..."
+echo "       Sources: ${SOURCE_DIR}"
+echo "  Build output: ${BUILD_DIR}"
+if [[ ${SOURCE_DIR} != ${SCRIPT_ROOT} ]]; then
+  echo " Build scripts: ${SCRIPT_ROOT}"
+fi
+set -x
+source ${SCRIPT_ROOT}/kokoro/common/caplog.sh
+
+#
+# Configure under $BUILD_DIR:
+#
+mkdir -p "${BUILD_DIR}"
+
+(
+  cd "${BUILD_DIR}"
+  caplog 01_configure \
+    cmake -S "${SOURCE_DIR}" \
+    ${CMAKE_BUILD_TYPE_FLAG} \
+    ${CAPLOG_CMAKE_ARGS:-}
+)
+if [[ -n ${CAPLOG_DIR:-} ]]; then
+  # Save configuration logs.
+  mkdir -p "${CAPLOG_DIR}/CMakeFiles"
+  cp "${BUILD_DIR}"/CMakeFiles/CMake*.log "${CAPLOG_DIR}/CMakeFiles"
+fi
+
+#
+# Build:
+#
+caplog 02_build \
+  cmake --build "${BUILD_DIR}"
+
+#
+# Run tests
+#
+(
+  cd "${BUILD_DIR}"
+  caplog 03_combined_testlog \
+    ctest ${CAPLOG_CTEST_ARGS:-}
+)

kokoro/linux/cmake_distcheck/build.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+#
+# Build file to set up and run tests based on distribution archive
+
+set -eux
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+#
+# Update git submodules
+#
+git submodule update --init --recursive
+
+#
+# Build distribution archive
+#
+# TODO: this should use Bazel-built dist archives.
+date ; ./autogen.sh
+date ; ./configure
+date ; make dist
+date
+
+DIST_ARCHIVE=( $(ls protobuf-*.tar.gz) )
+if (( ${#DIST_ARCHIVE[@]} != 1 )); then
+  echo >&2 "Distribution archive not found. ${#DIST_ARCHIVE[@]} matches:"
+  echo >&2 "${DIST_ARCHIVE[@]}"
+  exit 1
+fi
+
+#
+# Check for all expected files
+#
+kokoro/common/check_missing_dist_files.sh ${DIST_ARCHIVE}
+
+#
+# Extract to a temporary directory
+#
+if [[ -z ${DIST_WORK_ROOT:-} ]]; then
+  # If you want to preserve the extracted sources, set the DIST_WORK_ROOT
+  # environment variable to an existing directory that should be used.
+  DIST_WORK_ROOT=$(mktemp -d)
+  function cleanup_work_root() {
+    echo "Cleaning up temporary directory ${DIST_WORK_ROOT}..."
+    rm -rf ${DIST_WORK_ROOT}
+  }
+  trap cleanup_work_root EXIT
+fi
+
+tar -C ${DIST_WORK_ROOT} --strip-components=1 -axf ${DIST_ARCHIVE}
+
+#
+# Run tests using extracted sources
+#
+if SOURCE_DIR=${DIST_WORK_ROOT} \
+  CMAKE_GENERATOR=Ninja \
+  CTEST_PARALLEL_LEVEL=$(nproc) \
+  kokoro/common/cmake.sh; then
+  # TODO: remove this conditional.
+  # The cmake build is expected to fail due to missing abseil sources.
+  echo "$0: Expected failure, but build passed." >&2
+  echo "Please update $(basename $0) to remove failure expectation." >&2
+  echo "FAIL" >&2
+  exit 1
+fi
+echo "PASS"

kokoro/linux/cmake_distcheck/continuous.cfg

@@ -0,0 +1,5 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/cmake_distcheck/build.sh"
+timeout_mins: 1440

kokoro/linux/cmake_distcheck/presubmit.cfg

@@ -0,0 +1,5 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/cmake_distcheck/build.sh"
+timeout_mins: 1440

kokoro/macos-next/cpp/build.sh

@@ -9,47 +9,21 @@ if [[ -h /tmpfs ]] && [[ ${PWD} == /tmpfs/src ]]; then
   cd /Volumes/BuildData/tmpfs/src
 fi
 
-# These vars can be changed when running manually, e.g.:
-#
-#   % BUILD_CONFIG=RelWithDebInfo path/to/build.sh
+# Default environment variables used by cmake build:
+: ${CMAKE_CONFIG_TYPE:=Debug}
+export CMAKE_CONFIG_TYPE
+: ${CTEST_PARALLEL_LEVEL:=4}
+export CTEST_PARALLEL_LEVEL
 
-# By default, build using Debug config.
-: ${BUILD_CONFIG:=Debug}
-
-# By default, find the sources based on this script path.
-: ${SOURCE_DIR:=$(cd $(dirname $0)/../../..; pwd)}
-
-# By default, put outputs under <git root>/cmake/build.
-: ${BUILD_DIR:=${SOURCE_DIR}/cmake/build}
-
-source ${SOURCE_DIR}/kokoro/caplog.sh
+# Run from the project root directory.
+cd $(dirname $0)/../../..
 
 #
 # Update submodules
 #
-git -C "${SOURCE_DIR}" submodule update --init --recursive
+git submodule update --init --recursive
 
 #
-# Configure and build in a separate directory
+# Run build
 #
-mkdir -p "${BUILD_DIR}"
-
-caplog 01_configure \
-  cmake -S "${SOURCE_DIR}" -B "${BUILD_DIR}" ${CAPLOG_CMAKE_ARGS:-}
-
-if [[ -n ${CAPLOG_DIR:-} ]]; then
-  mkdir -p "${CAPLOG_DIR}/CMakeFiles"
-  cp "${BUILD_DIR}"/CMakeFiles/CMake*.log "${CAPLOG_DIR}/CMakeFiles"
-fi
-
-caplog 02_build \
-  cmake --build "${BUILD_DIR}" --config "${BUILD_CONFIG}"
-
-#
-# Run tests
-#
-(
-  cd "${BUILD_DIR}"
-  caplog 03_combined_testlog \
-    ctest -C "${BUILD_CONFIG}" -j4 ${CAPLOG_CTEST_ARGS:-}
-)
+kokoro/common/cmake.sh