Deleting python release kokoro setup. This has been replaced by upb's python release (#10055)

This commit is contained in:
mkruskal-google 2022-06-03 14:50:53 -07:00 committed by GitHub
parent 0663080c5a
commit 3380463ff8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 0 additions and 668 deletions

View File

@ -1,90 +0,0 @@
#!/bin/bash
set -ex
# change to repo root
pushd $(dirname $0)/../../../..
# Create stage dir
ORIGINAL_DIR=`pwd`
pushd ..
cp -R $ORIGINAL_DIR stage
export STAGE_DIR="`pwd`/stage"
popd
export REPO_DIR=protobuf
export BUILD_VERSION=`grep -i "version" python/google/protobuf/__init__.py | grep -o "'.*'" | tr -d "'"`
export BUILD_COMMIT=`git rev-parse HEAD`
export PLAT=x86_64
export UNICODE_WIDTH=32
export MACOSX_DEPLOYMENT_TARGET=10.9
rm -rf artifacts/
rm -rf multibuild/
mkdir artifacts
export ARTIFACT_DIR=$(pwd)/artifacts
git clone https://github.com/matthew-brett/multibuild.git
# Pin multibuild scripts at a known commit to avoid potentially unwanted future changes from
# silently creeping in (see https://github.com/protocolbuffers/protobuf/issues/9180).
# IMPORTANT: always pin multibuild at the same commit for:
# - linux/build_artifacts.sh
# - linux/build_artifacts.sh
# - windows/build_artifacts.bat
(cd multibuild; git checkout b89bb903e94308be79abefa4f436bf123ebb1313)
cp kokoro/release/python/linux/config.sh config.sh
build_artifact_version() {
MB_PYTHON_VERSION=$1
cp -R $STAGE_DIR $REPO_DIR
source multibuild/common_utils.sh
source multibuild/travis_steps.sh
before_install
clean_code $REPO_DIR $BUILD_COMMIT
build_wheel $REPO_DIR/python $PLAT
mv wheelhouse/* $ARTIFACT_DIR
# Clean up env
rm -rf venv
sudo rm -rf $REPO_DIR
}
build_x86_64_manylinux1_artifact_version() {
# Explicitly request building manylinux1 wheels, which is no longer the default.
# https://github.com/protocolbuffers/protobuf/issues/9180
MB_ML_VER=1
build_artifact_version $@
}
build_x86_64_manylinux2010_artifact_version() {
# Explicitly request building manylinux2010 wheels
MB_ML_VER=2010
build_artifact_version $@
}
build_crosscompiled_aarch64_manylinux2014_artifact_version() {
# crosscompilation is only supported with the dockcross manylinux2014 image
DOCKER_IMAGE=dockcross/manylinux2014-aarch64:20210706-65bf2dd
MB_ML_VER=2014
PLAT=aarch64
# TODO(jtatermusch): currently when crosscompiling, "auditwheel repair" will be disabled
# since auditwheel doesn't work for crosscomiled wheels.
build_artifact_version $@
}
build_x86_64_manylinux1_artifact_version 3.6
build_x86_64_manylinux1_artifact_version 3.7
build_x86_64_manylinux1_artifact_version 3.8
build_x86_64_manylinux1_artifact_version 3.9
build_x86_64_manylinux2010_artifact_version 3.10
build_crosscompiled_aarch64_manylinux2014_artifact_version 3.7
build_crosscompiled_aarch64_manylinux2014_artifact_version 3.8
build_crosscompiled_aarch64_manylinux2014_artifact_version 3.9
build_crosscompiled_aarch64_manylinux2014_artifact_version 3.10

View File

@ -1,93 +0,0 @@
# Define custom utilities
# Test for OSX with [ -n "$IS_OSX" ]
function pre_build {
# Any stuff that you need to do before you start building the wheels
# Runs in the root directory of this repository.
pushd protobuf
if [ "$PLAT" == "aarch64" ]
then
local configure_host_flag="--host=aarch64"
fi
# Build protoc and libprotobuf
./autogen.sh
CXXFLAGS="-fPIC -g -O2" ./configure $configure_host_flag
make -j8
if [ "$PLAT" == "aarch64" ]
then
# we are crosscompiling for aarch64 while running on x64
# the simplest way for build_py command to be able to generate
# the protos is by running the protoc process under
# an emulator. That way we don't have to build a x64 version
# of protoc. The qemu-arm emulator is already included
# in the dockcross docker image.
# Running protoc under an emulator is fast as protoc doesn't
# really do much.
# create a simple shell wrapper that runs crosscompiled protoc under qemu
echo '#!/bin/bash' >protoc_qemu_wrapper.sh
echo 'exec qemu-aarch64 "../src/protoc" "$@"' >>protoc_qemu_wrapper.sh
chmod ugo+x protoc_qemu_wrapper.sh
# PROTOC variable is by build_py step that runs under ./python directory
export PROTOC=../protoc_qemu_wrapper.sh
fi
# Generate python dependencies.
pushd python
python setup.py build_py
popd
popd
}
function bdist_wheel_cmd {
# Builds wheel with bdist_wheel, puts into wheelhouse
#
# It may sometimes be useful to use bdist_wheel for the wheel building
# process. For example, versioneer has problems with versions which are
# fixed with bdist_wheel:
# https://github.com/warner/python-versioneer/issues/121
local abs_wheelhouse=$1
# Modify build version
pwd
ls
if [ "$PLAT" == "aarch64" ]
then
# when crosscompiling for aarch64, --plat-name needs to be set explicitly
# to end up with correctly named wheel file
# the value should be manylinuxABC_ARCH and dockcross docker image
# conveniently provides the value in the AUDITWHEEL_PLAT env
local plat_name_flag="--plat-name=$AUDITWHEEL_PLAT"
# override the value of EXT_SUFFIX to make sure the crosscompiled .so files in the wheel have the correct filename suffix
export PROTOCOL_BUFFERS_OVERRIDE_EXT_SUFFIX="$(python -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX").replace("-x86_64-linux-gnu.so", "-aarch64-linux-gnu.so"))')"
fi
python setup.py bdist_wheel --cpp_implementation --compile_static_extension $plat_name_flag
cp dist/*.whl $abs_wheelhouse
}
function build_wheel {
build_wheel_cmd "bdist_wheel_cmd" $@
}
function run_tests {
# Runs tests on installed distribution from an empty directory
python --version
python -c "from google.protobuf.pyext import _message;"
}
if [ "$PLAT" == "aarch64" ]
then
# when crosscompiling for aarch64, override the default multibuild's repair_wheelhouse logic
# since "auditwheel repair" doesn't work for crosscompiled wheels
function repair_wheelhouse {
echo "Skipping repair_wheelhouse since auditwheel requires build architecture to match wheel architecture."
}
fi

View File

@ -1,8 +0,0 @@
# Config file for running tests in Kokoro
build_file: "protobuf/kokoro/release/python/linux/build_artifacts.sh"
action {
define_artifacts {
regex: "github/protobuf/artifacts/**"
}
}

View File

@ -1,8 +0,0 @@
# Config file for running tests in Kokoro
build_file: "protobuf/kokoro/release/python/linux/build_artifacts.sh"
action {
define_artifacts {
regex: "github/protobuf/artifacts/**"
}
}

View File

@ -1,8 +0,0 @@
# Configuration for Linux release builds
build_file: "protobuf/kokoro/release/python/linux/build_artifacts.sh"
action {
define_artifacts {
regex: "github/protobuf/artifacts/**"
}
}

View File

@ -1,65 +0,0 @@
#!/bin/bash
set -ex
# change to repo root
pushd $(dirname $0)/../../../..
# Create stage dir
ORIGINAL_DIR=`pwd`
pushd ..
cp -R $ORIGINAL_DIR stage
export STAGE_DIR="`pwd`/stage"
popd
export REPO_DIR=protobuf
export BUILD_VERSION=`grep -i "version" python/google/protobuf/__init__.py | grep -o "'.*'" | tr -d "'"`
export BUILD_COMMIT=`git rev-parse HEAD`
export PLAT=x86_64
export UNICODE_WIDTH=32
export MACOSX_DEPLOYMENT_TARGET=10.9
export TRAVIS_OS_NAME="osx"
rm -rf artifacts/
rm -rf multibuild/
mkdir artifacts
export ARTIFACT_DIR=$(pwd)/artifacts
git clone https://github.com/matthew-brett/multibuild.git
# Pin multibuild scripts at a known commit to avoid potentially unwanted future changes from
# silently creeping in (see https://github.com/protocolbuffers/protobuf/issues/9180).
# IMPORTANT: always pin multibuild at the same commit for:
# - linux/build_artifacts.sh
# - linux/build_artifacts.sh
# - windows/build_artifacts.bat
(cd multibuild; git checkout b89bb903e94308be79abefa4f436bf123ebb1313)
cp kokoro/release/python/macos/config.sh config.sh
OLD_PATH=$PATH
build_artifact_version() {
MB_PYTHON_VERSION=$1
# Clean up env
rm -rf venv
sudo rm -rf $REPO_DIR
cp -R $STAGE_DIR $REPO_DIR
export PATH=$OLD_PATH
source multibuild/common_utils.sh
source multibuild/travis_steps.sh
before_install
clean_code $REPO_DIR $BUILD_COMMIT
build_wheel $REPO_DIR/python $PLAT
mv wheelhouse/* $ARTIFACT_DIR
}
export MB_PYTHON_OSX_VER=10.9
build_artifact_version 3.6
build_artifact_version 3.7
build_artifact_version 3.8
build_artifact_version 3.9
build_artifact_version 3.10

View File

@ -1,68 +0,0 @@
# Define custom utilities
# Test for OSX with [ -n "$IS_OSX" ]
function remove_travis_ve_pip {
# Removing the system virtualenv or pip can be very problematic for
# macOS on Kokoro, so just leave them be.
:;
}
function install_pip {
check_python
PIP_CMD="sudo $PYTHON_EXE -m pip${pip_args:+ $pip_args}"
$PIP_CMD install --upgrade pip
}
function install_virtualenv {
check_python
check_pip
$PIP_CMD install --upgrade virtualenv
VIRTUALENV_CMD="$PYTHON_EXE -m virtualenv"
}
function pre_build {
# Any stuff that you need to do before you start building the wheels
# Runs in the root directory of this repository.
pushd protobuf
# Build protoc and protobuf libraries
use_bazel.sh 5.1.1
bazel build -c opt //:protoc //pkg:protobuf //pkg:protobuf_lite
local _bazel_bin=$(bazel info -c opt bazel-bin)
export PROTOC=${_bazel_bin}/protoc
export LIBPROTOBUF=${_bazel_bin}/pkg/libprotobuf.a
# Generate python dependencies.
pushd python
python setup.py build_py
popd
popd
}
function bdist_wheel_cmd {
# Builds wheel with bdist_wheel, puts into wheelhouse
#
# It may sometimes be useful to use bdist_wheel for the wheel building
# process. For example, versioneer has problems with versions which are
# fixed with bdist_wheel:
# https://github.com/warner/python-versioneer/issues/121
local abs_wheelhouse=$1
# Modify build version
pwd
ls
python setup.py build_ext --cpp_implementation -O${LIBPROTOBUF}
python setup.py bdist_wheel --cpp_implementation
cp dist/*.whl $abs_wheelhouse
}
function build_wheel {
build_wheel_cmd "bdist_wheel_cmd" $@
}
function run_tests {
# Runs tests on installed distribution from an empty directory
python --version
python -c "from google.protobuf.pyext import _message;"
}

View File

@ -1,8 +0,0 @@
# Configuration for Mac OSX release builds
build_file: "protobuf/kokoro/release/python/macos/build_artifacts.sh"
action {
define_artifacts {
regex: "github/protobuf/artifacts/**"
}
}

View File

@ -1,8 +0,0 @@
# Configuration for Mac OSX release builds
build_file: "protobuf/kokoro/release/python/macos/build_artifacts.sh"
action {
define_artifacts {
regex: "github/protobuf/artifacts/**"
}
}

View File

@ -1,8 +0,0 @@
# Configuration for Mac OSX release builds
build_file: "protobuf/kokoro/release/python/macos/build_artifacts.sh"
action {
define_artifacts {
regex: "github/protobuf/artifacts/**"
}
}

View File

@ -1,88 +0,0 @@
REM Move scripts to root
set REPO_DIR_STAGE=%cd%\github\protobuf-stage
xcopy /S github\protobuf "%REPO_DIR_STAGE%\"
cd github\protobuf
copy kokoro\release\python\windows\build_single_artifact.bat build_single_artifact.bat
REM Set environment variables
set PACKAGE_NAME=protobuf
set REPO_DIR=protobuf
set BUILD_DLL=OFF
set UNICODE=ON
set OTHER_TEST_DEP="setuptools==38.5.1"
set OLD_PATH=C:\Program Files (x86)\MSBuild\14.0\bin\;%PATH%
REM Fetch multibuild
git clone https://github.com/matthew-brett/multibuild.git
REM Pin multibuild scripts at a known commit to avoid potentially unwanted future changes from
REM silently creeping in (see https://github.com/protocolbuffers/protobuf/issues/9180).
REM IMPORTANT: always pin multibuild at the same commit for:
REM - linux/build_artifacts.sh
REM - linux/build_artifacts.sh
REM - windows/build_artifacts.bat
cd multibuild
git checkout b89bb903e94308be79abefa4f436bf123ebb1313
cd ..
REM Install zlib
mkdir zlib
curl -L -o zlib.zip http://www.winimage.com/zLibDll/zlib123dll.zip
curl -L -o zlib-src.zip http://www.winimage.com/zLibDll/zlib123.zip
7z x zlib.zip -ozlib
7z x zlib-src.zip -ozlib\include
SET ZLIB_ROOT=%cd%\zlib
del /Q zlib.zip
del /Q zlib-src.zip
REM Create directory for artifacts
SET ARTIFACT_DIR=%cd%\artifacts
mkdir %ARTIFACT_DIR%
REM Build wheel
SET PYTHON=C:\python37_32bit
SET PYTHON_VERSION=3.7
SET PYTHON_ARCH=32
CALL build_single_artifact.bat || goto :error
SET PYTHON=C:\python37
SET PYTHON_VERSION=3.7
SET PYTHON_ARCH=64
CALL build_single_artifact.bat || goto :error
powershell -File kokoro/release/python/windows/install_python_interpreters.ps1
SET PYTHON=C:\python38_32bit
SET PYTHON_VERSION=3.8
SET PYTHON_ARCH=32
CALL build_single_artifact.bat || goto :error
SET PYTHON=C:\python38
SET PYTHON_VERSION=3.8
SET PYTHON_ARCH=64
CALL build_single_artifact.bat || goto :error
SET PYTHON=C:\python39_32bit
SET PYTHON_VERSION=3.9
SET PYTHON_ARCH=32
CALL build_single_artifact.bat || goto :error
SET PYTHON=C:\python39
SET PYTHON_VERSION=3.9
SET PYTHON_ARCH=64
CALL build_single_artifact.bat || goto :error
SET PYTHON=C:\python310_32bit
SET PYTHON_VERSION=3.10
SET PYTHON_ARCH=32
CALL build_single_artifact.bat || goto :error
SET PYTHON=C:\python310
SET PYTHON_VERSION=3.10
SET PYTHON_ARCH=64
CALL build_single_artifact.bat || goto :error
goto :EOF
:error
exit /b %errorlevel%

View File

@ -1,78 +0,0 @@
setlocal
if %PYTHON%==C:\python37_32bit set generator=Visual Studio 14
if %PYTHON%==C:\python37_32bit set vcplatform=Win32
if %PYTHON%==C:\python37 set generator=Visual Studio 14 Win64
if %PYTHON%==C:\python37 set vcplatform=x64
if %PYTHON%==C:\python38_32bit set generator=Visual Studio 14
if %PYTHON%==C:\python38_32bit set vcplatform=Win32
if %PYTHON%==C:\python38 set generator=Visual Studio 14 Win64
if %PYTHON%==C:\python38 set vcplatform=x64
if %PYTHON%==C:\python39_32bit set generator=Visual Studio 14
if %PYTHON%==C:\python39_32bit set vcplatform=Win32
if %PYTHON%==C:\python39 set generator=Visual Studio 14 Win64
if %PYTHON%==C:\python39 set vcplatform=x64
if %PYTHON%==C:\python310_32bit set generator=Visual Studio 14
if %PYTHON%==C:\python310_32bit set vcplatform=Win32
if %PYTHON%==C:\python310 set generator=Visual Studio 14 Win64
if %PYTHON%==C:\python310 set vcplatform=x64
REM Prepend newly installed Python to the PATH of this build (this cannot be
REM done from inside the powershell script as it would require to restart
REM the parent CMD process).
SET PATH=C:\Program Files\CMake\bin;%PYTHON%;%PYTHON%\Scripts;%OLD_PATH%
python -m pip install -U pip
pip install wheel
REM Check that we have the expected version and architecture for Python
python --version
python -c "import struct; print(struct.calcsize('P') * 8)"
rmdir /s/q %REPO_DIR%
xcopy /s %REPO_DIR_STAGE% "%REPO_DIR%\"
REM Checkout release commit
cd %REPO_DIR%
REM ======================
REM Build Protobuf Library
REM ======================
mkdir src\.libs
mkdir vcprojects
pushd vcprojects
cmake -G "%generator%" -Dprotobuf_BUILD_SHARED_LIBS=%BUILD_DLL% -Dprotobuf_UNICODE=%UNICODE% -Dprotobuf_BUILD_TESTS=OFF ../cmake || goto :error
msbuild protobuf.sln /p:Platform=%vcplatform% /p:Configuration=Release || goto :error
dir /s /b
popd
copy vcprojects\Release\libprotobuf.lib src\.libs\libprotobuf.a
copy vcprojects\Release\libprotobuf-lite.lib src\.libs\libprotobuf-lite.a
SET PATH=%cd%\vcprojects\Release;%PATH%
dir vcprojects\Release
REM ======================
REM Build python library
REM ======================
cd python
REM sed -i 's/\ extra_compile_args\ =\ \[\]/\ extra_compile_args\ =\ \[\'\/MT\'\]/g' setup.py
python setup.py bdist_wheel --cpp_implementation --compile_static_extension
dir dist
copy dist\* %ARTIFACT_DIR%
dir %ARTIFACT_DIR%
cd ..\..
goto :EOF
:error
exit /b %errorlevel%

View File

@ -1,8 +0,0 @@
# Configuration for Windows protoc release builds
build_file: "protobuf/kokoro/release/python/windows/build_artifacts.bat"
action {
define_artifacts {
regex: "github/protobuf/artifacts/**"
}
}

View File

@ -1,114 +0,0 @@
#!/usr/bin/env powershell
# Install Python 3.8 for x64 and x86 in order to build wheels on Windows.
# Originally from grpc/tools/internal_ci/helper_scripts/install_python_interpreters.ps1
Set-StrictMode -Version 2
$ErrorActionPreference = 'Stop'
trap {
$ErrorActionPreference = "Continue"
Write-Error $_
exit 1
}
# Avoid "Could not create SSL/TLS secure channel"
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
function Install-Python {
Param(
[string]$PythonVersion,
[string]$PythonInstaller,
[string]$PythonInstallPath,
[string]$PythonInstallerHash
)
$PythonInstallerUrl = "https://www.python.org/ftp/python/$PythonVersion/$PythonInstaller.exe"
$PythonInstallerPath = "C:\tools\$PythonInstaller.exe"
# Downloads installer
Write-Host "Downloading the Python installer: $PythonInstallerUrl => $PythonInstallerPath"
Invoke-WebRequest -Uri $PythonInstallerUrl -OutFile $PythonInstallerPath
# Validates checksum
$HashFromDownload = Get-FileHash -Path $PythonInstallerPath -Algorithm MD5
if ($HashFromDownload.Hash -ne $PythonInstallerHash) {
throw "Invalid Python installer: failed checksum!"
}
Write-Host "Python installer $PythonInstallerPath validated."
# Installs Python
& $PythonInstallerPath /passive InstallAllUsers=1 PrependPath=1 Include_test=0 TargetDir=$PythonInstallPath
if (-Not $?) {
throw "The Python installation exited with error!"
}
# NOTE(lidiz) Even if the install command finishes in the script, that
# doesn't mean the Python installation is finished. If using "ps" to check
# for running processes, you might see ongoing installers at this point.
# So, we needs this "hack" to reliably validate that the Python binary is
# functioning properly.
# Wait for the installer process
Wait-Process -Name $PythonInstaller -Timeout 300
Write-Host "Installation process exits normally."
# Validate Python binary
$PythonBinary = "$PythonInstallPath\python.exe"
& $PythonBinary -c 'print(42)'
Write-Host "Python binary works properly."
# Installs pip
& $PythonBinary -m ensurepip --user
Write-Host "Python $PythonVersion installed by $PythonInstaller at $PythonInstallPath."
}
# Python 3.8
$Python38x86Config = @{
PythonVersion = "3.8.0"
PythonInstaller = "python-3.8.0"
PythonInstallPath = "C:\python38_32bit"
PythonInstallerHash = "412a649d36626d33b8ca5593cf18318c"
}
Install-Python @Python38x86Config
$Python38x64Config = @{
PythonVersion = "3.8.0"
PythonInstaller = "python-3.8.0-amd64"
PythonInstallPath = "C:\python38"
PythonInstallerHash = "29ea87f24c32f5e924b7d63f8a08ee8d"
}
Install-Python @Python38x64Config
# Python 3.9
$Python39x86Config = @{
PythonVersion = "3.9.0"
PythonInstaller = "python-3.9.0"
PythonInstallPath = "C:\python39_32bit"
PythonInstallerHash = "4a2812db8ab9f2e522c96c7728cfcccb"
}
Install-Python @Python39x86Config
$Python39x64Config = @{
PythonVersion = "3.9.0"
PythonInstaller = "python-3.9.0-amd64"
PythonInstallPath = "C:\python39"
PythonInstallerHash = "b61a33dc28f13b561452f3089c87eb63"
}
Install-Python @Python39x64Config
# Python 3.10
$Python310x86Config = @{
PythonVersion = "3.10.0"
PythonInstaller = "python-3.10.0"
PythonInstallPath = "C:\python310_32bit"
PythonInstallerHash = "133aa48145032e341ad2a000cd3bff50"
}
Install-Python @Python310x86Config
$Python310x64Config = @{
PythonVersion = "3.10.0"
PythonInstaller = "python-3.10.0-amd64"
PythonInstallPath = "C:\python310"
PythonInstallerHash = "c3917c08a7fe85db7203da6dcaa99a70"
}
Install-Python @Python310x64Config

View File

@ -1,8 +0,0 @@
# Configuration for Windows protoc release builds
build_file: "protobuf/kokoro/release/python/windows/build_artifacts.bat"
action {
define_artifacts {
regex: "github/protobuf/artifacts/**"
}
}

View File

@ -1,8 +0,0 @@
# Configuration for Windows protoc release builds
build_file: "protobuf/kokoro/release/python/windows/build_artifacts.bat"
action {
define_artifacts {
regex: "github/protobuf/artifacts/**"
}
}