Migrate all Kokoro build scripts over to use the docker VM image (#4114)

* Work around GCC-9 warning treated as error

```
../source/opt/instruction.h:101:23: error: '*((void*)& operand +32)' may be used uninitialized in this function [-Werror=maybe-uninitialized]
  101 |     uint64_t result = uint64_t(words[0]);
```

* Migrate all Kokoro build scripts over to use the docker VM image

Required updating the NDK SDK and build scripts, as well as the check_copyright for handling 2021.
This commit is contained in:
Ben Clayton 2021-01-18 18:36:26 +00:00 committed by GitHub
parent ee39b5db5f
commit 8383bd5d6f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 246 additions and 226 deletions

View File

@ -1,5 +1,5 @@
APP_ABI := all
APP_BUILD_SCRIPT := Android.mk
APP_STL := gnustl_static
APP_STL := c++_static
APP_PLATFORM := android-9
NDK_TOOLCHAIN_VERSION := 4.9

32
kokoro/android/build.sh Normal file → Executable file
View File

@ -20,33 +20,5 @@ set -e
# Display commands being run.
set -x
BUILD_ROOT=$PWD
SRC=$PWD/github/SPIRV-Tools
TARGET_ARCH="armeabi-v7a with NEON"
export ANDROID_NDK=/opt/android-ndk-r15c
# Get NINJA.
wget -q https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip
unzip -q ninja-linux.zip
export PATH="$PWD:$PATH"
git clone --depth=1 https://github.com/taka-no-me/android-cmake.git android-cmake
export TOOLCHAIN_PATH=$PWD/android-cmake/android.toolchain.cmake
cd $SRC
git clone --depth=1 https://github.com/KhronosGroup/SPIRV-Headers external/spirv-headers
git clone https://github.com/google/googletest external/googletest
cd external && cd googletest && git reset --hard 1fb1bb23bb8418dc73a5a9a82bbed31dc610fec7 && cd .. && cd ..
git clone --depth=1 https://github.com/google/effcee external/effcee
git clone --depth=1 https://github.com/google/re2 external/re2
mkdir build && cd $SRC/build
# Invoke the build.
BUILD_SHA=${KOKORO_GITHUB_COMMIT:-$KOKORO_GITHUB_PULL_REQUEST_COMMIT}
echo $(date): Starting build...
cmake -DPYTHON_EXECUTABLE:FILEPATH=/usr/bin/python3 -DCMAKE_BUILD_TYPE=Release -DANDROID_NATIVE_API_LEVEL=android-14 -DANDROID_ABI="armeabi-v7a with NEON" -DSPIRV_SKIP_TESTS=ON -DCMAKE_TOOLCHAIN_FILE=$TOOLCHAIN_PATH -GNinja -DANDROID_NDK=$ANDROID_NDK ..
echo $(date): Build everything...
ninja
echo $(date): Build completed.
SCRIPT_DIR=`dirname "$BASH_SOURCE"`
source $SCRIPT_DIR/../scripts/linux/build.sh ASAN clang cmake-android-ndk

2
kokoro/linux-clang-asan/build.sh Normal file → Executable file
View File

@ -21,4 +21,4 @@ set -e
set -x
SCRIPT_DIR=`dirname "$BASH_SOURCE"`
source $SCRIPT_DIR/../scripts/linux/build.sh ASAN clang
source $SCRIPT_DIR/../scripts/linux/build.sh ASAN clang cmake

2
kokoro/linux-clang-debug/build.sh Normal file → Executable file
View File

@ -21,4 +21,4 @@ set -e
set -x
SCRIPT_DIR=`dirname "$BASH_SOURCE"`
source $SCRIPT_DIR/../scripts/linux/build.sh DEBUG clang
source $SCRIPT_DIR/../scripts/linux/build.sh DEBUG clang cmake

23
kokoro/linux-clang-release-bazel/build.sh Normal file → Executable file
View File

@ -20,24 +20,5 @@ set -e
# Display commands being run.
set -x
CC=clang
CXX=clang++
SRC=$PWD/github/SPIRV-Tools
cd $SRC
git clone --depth=1 https://github.com/KhronosGroup/SPIRV-Headers external/spirv-headers
git clone https://github.com/google/googletest external/googletest
cd external && cd googletest && git reset --hard 1fb1bb23bb8418dc73a5a9a82bbed31dc610fec7 && cd .. && cd ..
git clone --depth=1 https://github.com/google/effcee external/effcee
git clone --depth=1 https://github.com/google/re2 external/re2
gsutil cp gs://bazel/0.29.1/release/bazel-0.29.1-linux-x86_64 .
chmod +x bazel-0.29.1-linux-x86_64
echo $(date): Build everything...
./bazel-0.29.1-linux-x86_64 build :all
echo $(date): Build completed.
echo $(date): Starting bazel test...
./bazel-0.29.1-linux-x86_64 test :all
echo $(date): Bazel test completed.
SCRIPT_DIR=`dirname "$BASH_SOURCE"`
source $SCRIPT_DIR/../scripts/linux/build.sh RELEASE clang bazel

2
kokoro/linux-clang-release/build.sh Normal file → Executable file
View File

@ -21,4 +21,4 @@ set -e
set -x
SCRIPT_DIR=`dirname "$BASH_SOURCE"`
source $SCRIPT_DIR/../scripts/linux/build.sh RELEASE clang
source $SCRIPT_DIR/../scripts/linux/build.sh RELEASE clang cmake

2
kokoro/linux-gcc-debug/build.sh Normal file → Executable file
View File

@ -21,4 +21,4 @@ set -e
set -x
SCRIPT_DIR=`dirname "$BASH_SOURCE"`
source $SCRIPT_DIR/../scripts/linux/build.sh DEBUG gcc
source $SCRIPT_DIR/../scripts/linux/build.sh DEBUG gcc cmake

2
kokoro/linux-gcc-release/build.sh Normal file → Executable file
View File

@ -21,4 +21,4 @@ set -e
set -x
SCRIPT_DIR=`dirname "$BASH_SOURCE"`
source $SCRIPT_DIR/../scripts/linux/build.sh RELEASE gcc
source $SCRIPT_DIR/../scripts/linux/build.sh RELEASE gcc cmake

37
kokoro/ndk-build/build.sh Normal file → Executable file
View File

@ -20,38 +20,5 @@ set -e
# Display commands being run.
set -x
BUILD_ROOT=$PWD
SRC=$PWD/github/SPIRV-Tools
# Get NINJA.
wget -q https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip
unzip -q ninja-linux.zip
export PATH="$PWD:$PATH"
# NDK Path
export ANDROID_NDK=/opt/android-ndk-r15c
# Get the dependencies.
cd $SRC
git clone --depth=1 https://github.com/KhronosGroup/SPIRV-Headers external/spirv-headers
git clone https://github.com/google/googletest external/googletest
cd external && cd googletest && git reset --hard 1fb1bb23bb8418dc73a5a9a82bbed31dc610fec7 && cd .. && cd ..
git clone --depth=1 https://github.com/google/effcee external/effcee
git clone --depth=1 https://github.com/google/re2 external/re2
mkdir build && cd $SRC/build
mkdir libs
mkdir app
# Invoke the build.
BUILD_SHA=${KOKORO_GITHUB_COMMIT:-$KOKORO_GITHUB_PULL_REQUEST_COMMIT}
echo $(date): Starting ndk-build ...
$ANDROID_NDK/ndk-build \
-C $SRC/android_test \
NDK_PROJECT_PATH=. \
NDK_LIBS_OUT=./libs \
NDK_APP_OUT=./app \
-j8
echo $(date): ndk-build completed.
SCRIPT_DIR=`dirname "$BASH_SOURCE"`
source $SCRIPT_DIR/../scripts/linux/build.sh ASAN clang android-ndk-build

View File

@ -0,0 +1,196 @@
#!/bin/bash
# Copyright (c) 2018 Google LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Linux Build Script.
# Fail on any error.
set -e
# Display commands being run.
set -x
. /bin/using.sh # Declare the bash `using` function for configuring toolchains.
if [ $COMPILER = "clang" ]; then
using clang-10.0.0
elif [ $COMPILER = "gcc" ]; then
using gcc-9
fi
cd $ROOT_DIR
function clone_if_missing() {
url=$1
dir=$2
if [[ ! -d "$dir" ]]; then
git clone ${@:3} "$url" "$dir"
fi
}
function clean_dir() {
dir=$1
if [[ -d "$dir" ]]; then
rm -fr "$dir"
fi
mkdir "$dir"
}
clone_if_missing https://github.com/KhronosGroup/SPIRV-Headers external/spirv-headers --depth=1
clone_if_missing https://github.com/google/googletest external/googletest
pushd external/googletest; git reset --hard 1fb1bb23bb8418dc73a5a9a82bbed31dc610fec7; popd
clone_if_missing https://github.com/google/effcee external/effcee --depth=1
clone_if_missing https://github.com/google/re2 external/re2 --depth=1
clone_if_missing https://github.com/protocolbuffers/protobuf external/protobuf --branch v3.13.0
if [ $TOOL = "cmake" ]; then
using cmake-3.17.2
using ninja-1.10.0
# Possible configurations are:
# ASAN, COVERAGE, RELEASE, DEBUG, DEBUG_EXCEPTION, RELEASE_MINGW
BUILD_TYPE="Debug"
if [ $CONFIG = "RELEASE" ] || [ $CONFIG = "RELEASE_MINGW" ]; then
BUILD_TYPE="RelWithDebInfo"
fi
SKIP_TESTS="False"
ADDITIONAL_CMAKE_FLAGS=""
if [ $CONFIG = "ASAN" ]; then
ADDITIONAL_CMAKE_FLAGS="-DSPIRV_USE_SANITIZER=address,bounds,null"
[ $COMPILER = "clang" ] || { echo "$CONFIG requires clang"; exit 1; }
elif [ $CONFIG = "COVERAGE" ]; then
ADDITIONAL_CMAKE_FLAGS="-DENABLE_CODE_COVERAGE=ON"
SKIP_TESTS="True"
elif [ $CONFIG = "DEBUG_EXCEPTION" ]; then
ADDITIONAL_CMAKE_FLAGS="-DDISABLE_EXCEPTIONS=ON -DDISABLE_RTTI=ON"
elif [ $CONFIG = "RELEASE_MINGW" ]; then
ADDITIONAL_CMAKE_FLAGS="-Dgtest_disable_pthreads=ON -DCMAKE_TOOLCHAIN_FILE=$SRC/cmake/linux-mingw-toolchain.cmake"
SKIP_TESTS="True"
fi
clean_dir "$ROOT_DIR/build"
cd "$ROOT_DIR/build"
# Invoke the build.
BUILD_SHA=${KOKORO_GITHUB_COMMIT:-$KOKORO_GITHUB_PULL_REQUEST_COMMIT}
echo $(date): Starting build...
cmake -DPYTHON_EXECUTABLE:FILEPATH=/usr/bin/python3 -GNinja -DCMAKE_INSTALL_PREFIX=$KOKORO_ARTIFACTS_DIR/install -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DRE2_BUILD_TESTING=OFF -DSPIRV_BUILD_FUZZER=ON $ADDITIONAL_CMAKE_FLAGS ..
echo $(date): Build everything...
ninja
echo $(date): Build completed.
if [ $CONFIG = "COVERAGE" ]; then
echo $(date): Check coverage...
ninja report-coverage
echo $(date): Check coverage completed.
fi
echo $(date): Starting ctest...
if [ $SKIP_TESTS = "False" ]; then
ctest -j4 --output-on-failure --timeout 300
fi
echo $(date): ctest completed.
# Package the build.
ninja install
cd $KOKORO_ARTIFACTS_DIR
tar czf install.tgz install
elif [ $TOOL = "cmake-smoketest" ]; then
using cmake-3.17.2
using ninja-1.10.0
# Get shaderc.
SHADERC_DIR=/tmp/shaderc
clean_dir "$SHADERC_DIR"
cd $SHADERC_DIR
git clone https://github.com/google/shaderc.git .
cd $SHADERC_DIR/third_party
# Get shaderc dependencies. Link the appropriate SPIRV-Tools.
git clone https://github.com/google/googletest.git
git clone https://github.com/KhronosGroup/glslang.git
ln -s $ROOT_DIR spirv-tools
git clone https://github.com/KhronosGroup/SPIRV-Headers.git spirv-headers
git clone https://github.com/google/re2
git clone https://github.com/google/effcee
cd $SHADERC_DIR
mkdir build
cd $SHADERC_DIR/build
# Invoke the build.
echo $(date): Starting build...
cmake -GNinja -DRE2_BUILD_TESTING=OFF -DCMAKE_BUILD_TYPE="Release" ..
echo $(date): Build glslang...
ninja glslangValidator
echo $(date): Build everything...
ninja
echo $(date): Build completed.
echo $(date): Check Shaderc for copyright notices...
ninja check-copyright
echo $(date): Starting ctest...
ctest --output-on-failure -j4
echo $(date): ctest completed.
elif [ $TOOL = "cmake-android-ndk" ]; then
using cmake-3.17.2
using ndk-r21d
using ninja-1.10.0
clean_dir "$ROOT_DIR/build"
cd "$ROOT_DIR/build"
echo $(date): Starting build...
cmake -DCMAKE_BUILD_TYPE=Release \
-DANDROID_NATIVE_API_LEVEL=android-16 \
-DANDROID_ABI="armeabi-v7a with NEON" \
-DSPIRV_SKIP_TESTS=ON \
-DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK_HOME/build/cmake/android.toolchain.cmake" \
-GNinja \
-DANDROID_NDK=$ANDROID_NDK \
..
echo $(date): Build everything...
ninja
echo $(date): Build completed.
elif [ $TOOL = "android-ndk-build" ]; then
using ndk-r21d
clean_dir "$ROOT_DIR/build"
cd "$ROOT_DIR/build"
echo $(date): Starting ndk-build ...
$ANDROID_NDK_HOME/ndk-build \
-C $ROOT_DIR/android_test \
NDK_PROJECT_PATH=. \
NDK_LIBS_OUT=./libs \
NDK_APP_OUT=./app \
-j4
echo $(date): ndk-build completed.
elif [ $TOOL = "bazel" ]; then
using bazel-3.1.0
echo $(date): Build everything...
bazel build :all
echo $(date): Build completed.
echo $(date): Starting bazel test...
bazel test :all
echo $(date): Bazel test completed.
fi

View File

@ -1,5 +1,5 @@
#!/bin/bash
# Copyright (c) 2018 Google LLC.
# Copyright (c) 2021 Google LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -17,89 +17,38 @@
# Fail on any error.
set -e
# Display commands being run.
set -x
BUILD_ROOT=$PWD
SRC=$PWD/github/SPIRV-Tools
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )"
ROOT_DIR="$( cd "${SCRIPT_DIR}/../../.." >/dev/null 2>&1 && pwd )"
CONFIG=$1
COMPILER=$2
SKIP_TESTS="False"
BUILD_TYPE="Debug"
CMAKE_C_CXX_COMPILER=""
if [ $COMPILER = "clang" ]
then
PATH=/usr/lib/llvm-3.8/bin:$PATH
CMAKE_C_CXX_COMPILER="-DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++"
fi
# Possible configurations are:
# ASAN, COVERAGE, RELEASE, DEBUG, DEBUG_EXCEPTION, RELEASE_MINGW
if [ $CONFIG = "RELEASE" ] || [ $CONFIG = "RELEASE_MINGW" ]
then
BUILD_TYPE="RelWithDebInfo"
fi
ADDITIONAL_CMAKE_FLAGS=""
if [ $CONFIG = "ASAN" ]
then
ADDITIONAL_CMAKE_FLAGS="-DSPIRV_USE_SANITIZER=address,bounds,null"
[ $COMPILER = "clang" ] || { echo "$CONFIG requires clang"; exit 1; }
elif [ $CONFIG = "COVERAGE" ]
then
ADDITIONAL_CMAKE_FLAGS="-DENABLE_CODE_COVERAGE=ON"
SKIP_TESTS="True"
elif [ $CONFIG = "DEBUG_EXCEPTION" ]
then
ADDITIONAL_CMAKE_FLAGS="-DDISABLE_EXCEPTIONS=ON -DDISABLE_RTTI=ON"
elif [ $CONFIG = "RELEASE_MINGW" ]
then
ADDITIONAL_CMAKE_FLAGS="-Dgtest_disable_pthreads=ON -DCMAKE_TOOLCHAIN_FILE=$SRC/cmake/linux-mingw-toolchain.cmake"
SKIP_TESTS="True"
fi
# Get NINJA.
wget -q https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip
unzip -q ninja-linux.zip
export PATH="$PWD:$PATH"
cd $SRC
git clone --depth=1 https://github.com/KhronosGroup/SPIRV-Headers external/spirv-headers
git clone https://github.com/google/googletest external/googletest
cd external && cd googletest && git reset --hard 1fb1bb23bb8418dc73a5a9a82bbed31dc610fec7 && cd .. && cd ..
git clone --depth=1 https://github.com/google/effcee external/effcee
git clone --depth=1 https://github.com/google/re2 external/re2
git clone --depth=1 --branch v3.13.0 https://github.com/protocolbuffers/protobuf external/protobuf
mkdir build && cd $SRC/build
# Invoke the build.
TOOL=$3
BUILD_SHA=${KOKORO_GITHUB_COMMIT:-$KOKORO_GITHUB_PULL_REQUEST_COMMIT}
echo $(date): Starting build...
cmake -DPYTHON_EXECUTABLE:FILEPATH=/usr/bin/python3 -GNinja -DCMAKE_INSTALL_PREFIX=$KOKORO_ARTIFACTS_DIR/install -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DRE2_BUILD_TESTING=OFF -DSPIRV_BUILD_FUZZER=ON $ADDITIONAL_CMAKE_FLAGS $CMAKE_C_CXX_COMPILER ..
echo $(date): Build everything...
ninja
echo $(date): Build completed.
docker run --rm -i \
--volume "${ROOT_DIR}:${ROOT_DIR}" \
--volume "${KOKORO_ARTIFACTS_DIR}:${KOKORO_ARTIFACTS_DIR}" \
--workdir "${ROOT_DIR}" \
--env SCRIPT_DIR=${SCRIPT_DIR} \
--env ROOT_DIR=${ROOT_DIR} \
--env CONFIG=${CONFIG} \
--env COMPILER=${COMPILER} \
--env TOOL=${TOOL} \
--env KOKORO_ARTIFACTS_DIR="${KOKORO_ARTIFACTS_DIR}" \
--env BUILD_SHA="${BUILD_SHA}" \
--entrypoint "${SCRIPT_DIR}/build-docker.sh" \
"gcr.io/shaderc-build/radial-build:latest"
if [ $CONFIG = "COVERAGE" ]
then
echo $(date): Check coverage...
ninja report-coverage
echo $(date): Check coverage completed.
fi
echo $(date): Starting ctest...
if [ $SKIP_TESTS = "False" ]
then
ctest -j4 --output-on-failure --timeout 300
fi
echo $(date): ctest completed.
# chown the given directory to the current user, if it exists.
# Docker creates files with the root user - this can upset the Kokoro artifact copier.
function chown_dir() {
dir=$1
if [[ -d "$dir" ]]; then
sudo chown -R "$(id -u):$(id -g)" "$dir"
fi
}
# Package the build.
ninja install
cd $KOKORO_ARTIFACTS_DIR
tar czf install.tgz install
chown_dir "${ROOT_DIR}/build"
chown_dir "${ROOT_DIR}/external"

53
kokoro/shaderc-smoketest/build.sh Normal file → Executable file
View File

@ -18,54 +18,5 @@ set -e
# Display commands being run.
set -x
BUILD_ROOT=$PWD
GITHUB_DIR=$BUILD_ROOT/github
SKIP_TESTS="False"
BUILD_TYPE="Release"
# Get NINJA.
wget -q https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip
unzip -q ninja-linux.zip
export PATH="$PWD:$PATH"
# Get shaderc.
cd $GITHUB_DIR
git clone https://github.com/google/shaderc.git
SHADERC_DIR=$GITHUB_DIR/shaderc
cd $SHADERC_DIR/third_party
# Get shaderc dependencies. Link the appropriate SPIRV-Tools.
git clone https://github.com/google/googletest.git
git clone https://github.com/KhronosGroup/glslang.git
ln -s $GITHUB_DIR/SPIRV-Tools spirv-tools
git clone https://github.com/KhronosGroup/SPIRV-Headers.git spirv-headers
git clone https://github.com/google/re2
git clone https://github.com/google/effcee
cd $SHADERC_DIR
mkdir build
cd $SHADERC_DIR/build
# Invoke the build.
BUILD_SHA=${KOKORO_GITHUB_COMMIT:-$KOKORO_GITHUB_PULL_REQUEST_COMMIT}
echo $(date): Starting build...
cmake -GNinja -DRE2_BUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
echo $(date): Build glslang...
ninja glslangValidator
echo $(date): Build everything...
ninja
echo $(date): Build completed.
echo $(date): Check Shaderc for copyright notices...
ninja check-copyright
echo $(date): Starting ctest...
if [ $SKIP_TESTS = "False" ]
then
ctest --output-on-failure -j4
fi
echo $(date): ctest completed.
SCRIPT_DIR=`dirname "$BASH_SOURCE"`
source $SCRIPT_DIR/../scripts/linux/build.sh RELEASE gcc cmake-smoketest

View File

@ -97,10 +97,14 @@ struct Operand {
assert(type == SPV_OPERAND_TYPE_TYPED_LITERAL_NUMBER);
assert(1 <= words.size());
assert(words.size() <= 2);
// Load the low word.
uint64_t result = uint64_t(words[0]);
uint64_t result = 0;
if (words.size() > 0) { // Needed to avoid maybe-uninitialized GCC warning
uint32_t low = words[0];
result = uint64_t(low);
}
if (words.size() > 1) {
result = result | (uint64_t(words[1]) << 32);
uint32_t high = words[1];
result = result | (uint64_t(high) << 32);
}
return result;
}

View File

@ -39,7 +39,7 @@ AUTHORS = ['The Khronos Group Inc.',
'Stefano Milizia']
CURRENT_YEAR='2020'
YEARS = '(2014-2016|2015-2016|2015-2020|2016|2016-2017|2017|2017-2019|2018|2019|2020)'
YEARS = '(2014-2016|2015-2016|2015-2020|2016|2016-2017|2017|2017-2019|2018|2019|2020|2021)'
COPYRIGHT_RE = re.compile(
'Copyright \(c\) {} ({})'.format(YEARS, '|'.join(AUTHORS)))