Fix benchmark to use docker image on kokoro (#5481)
* Fix benchmark to use docker image on kokoro * Deleted pure php benchmark and fix long time benchmark
This commit is contained in:
Yilun Chong
GitHub
parent
cbf89a647a
commit
6adc3d74fb
@ -62,7 +62,7 @@ class Benchmark
|
||||
$t = $this->runBenchmarkWithTimes(1);
|
||||
$times = ceil($this->benchmark_time / $t);
|
||||
return $this->total_bytes * $times /
|
||||
$this->runBenchmarkWithTimes($times) *
|
||||
($times == 1 ? $t : $this->runBenchmarkWithTimes($times)) *
|
||||
$this->coefficient;
|
||||
}
|
||||
|
||||
|
||||
@ -134,9 +134,10 @@ class Benchmark:
|
||||
t = self.dry_run(test_method_args, setup_method_args);
|
||||
if t < 3 :
|
||||
reps = int(math.ceil(3 / t)) * self.full_iteration
|
||||
t = timeit.timeit(stmt="%s(%s)" % (self.test_method, test_method_args),
|
||||
setup=self.full_setup_code(setup_method_args),
|
||||
number=reps);
|
||||
if reps != self.full_iteration:
|
||||
t = timeit.timeit(stmt="%s(%s)" % (self.test_method, test_method_args),
|
||||
setup=self.full_setup_code(setup_method_args),
|
||||
number=reps);
|
||||
return self.total_bytes * 1.0 / 2 ** 20 / (1.0 * t / reps)
|
||||
|
||||
|
||||
|
||||
@ -61,12 +61,12 @@ def upload_result(result_list, metadata):
|
||||
new_result["timestamp"] = _INITIAL_TIME
|
||||
print(labels_string)
|
||||
|
||||
bq = big_query_utils.create_big_query()
|
||||
row = big_query_utils.make_row(str(uuid.uuid4()), new_result)
|
||||
if not big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET,
|
||||
bq = big_query_utils.create_big_query()
|
||||
row = big_query_utils.make_row(str(uuid.uuid4()), new_result)
|
||||
if not big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET,
|
||||
_TABLE + "$" + _NOW,
|
||||
[row]):
|
||||
print('Error when uploading result', new_result)
|
||||
print('Error when uploading result', new_result)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
109
kokoro/linux/benchmark/build.sh
Executable file → Normal file
109
kokoro/linux/benchmark/build.sh
Executable file → Normal file
@ -1,108 +1,9 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Change to repo root
|
||||
|
||||
cd $(dirname $0)/../../..
|
||||
|
||||
export DOCKERFILE_DIR=kokoro/linux/64-bit
|
||||
export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
|
||||
export OUTPUT_DIR=testoutput
|
||||
oldpwd=`pwd`
|
||||
|
||||
# tcmalloc
|
||||
if [ ! -f gperftools/.libs/libtcmalloc.so ]; then
|
||||
git clone https://github.com/gperftools/gperftools.git
|
||||
cd gperftools
|
||||
./autogen.sh
|
||||
./configure
|
||||
make -j8
|
||||
cd ..
|
||||
fi
|
||||
|
||||
# download datasets for benchmark
|
||||
cd benchmarks
|
||||
./download_data.sh
|
||||
datasets=$(for file in $(find . -type f -name "dataset.*.pb" -not -path "./tmp/*"); do echo "$(pwd)/$file"; done | xargs)
|
||||
echo $datasets
|
||||
cd $oldpwd
|
||||
|
||||
# build Python protobuf
|
||||
./autogen.sh
|
||||
./configure CXXFLAGS="-fPIC -O2"
|
||||
make -j8
|
||||
cd python
|
||||
python setup.py build --cpp_implementation
|
||||
pip install . --user
|
||||
|
||||
|
||||
# build and run Python benchmark
|
||||
cd ../benchmarks
|
||||
make python-pure-python-benchmark
|
||||
make python-cpp-reflection-benchmark
|
||||
make -j8 python-cpp-generated-code-benchmark
|
||||
echo "[" > tmp/python_result.json
|
||||
echo "benchmarking pure python..."
|
||||
./python-pure-python-benchmark --json --behavior_prefix="pure-python-benchmark" $datasets >> tmp/python_result.json
|
||||
echo "," >> "tmp/python_result.json"
|
||||
echo "benchmarking python cpp reflection..."
|
||||
env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" LD_LIBRARY_PATH="$oldpwd/src/.libs" ./python-cpp-reflection-benchmark --json --behavior_prefix="cpp-reflection-benchmark" $datasets >> tmp/python_result.json
|
||||
echo "," >> "tmp/python_result.json"
|
||||
echo "benchmarking python cpp generated code..."
|
||||
env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" LD_LIBRARY_PATH="$oldpwd/src/.libs" ./python-cpp-generated-code-benchmark --json --behavior_prefix="cpp-generated-code-benchmark" $datasets >> tmp/python_result.json
|
||||
echo "]" >> "tmp/python_result.json"
|
||||
cd $oldpwd
|
||||
|
||||
# build CPP protobuf
|
||||
./configure
|
||||
make clean && make -j8
|
||||
|
||||
# build Java protobuf
|
||||
cd java
|
||||
mvn package
|
||||
cd ..
|
||||
|
||||
# build CPP benchmark
|
||||
cd benchmarks
|
||||
mv tmp/python_result.json . && make clean && make -j8 cpp-benchmark && mv python_result.json tmp
|
||||
echo "benchmarking cpp..."
|
||||
env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" ./cpp-benchmark --benchmark_min_time=5.0 --benchmark_out_format=json --benchmark_out="tmp/cpp_result.json" $datasets
|
||||
cd $oldpwd
|
||||
|
||||
# build go protobuf
|
||||
export PATH="`pwd`/src:$PATH"
|
||||
export GOPATH="$HOME/gocode"
|
||||
mkdir -p "$GOPATH/src/github.com/google"
|
||||
rm -f "$GOPATH/src/github.com/protocolbuffers/protobuf"
|
||||
ln -s "`pwd`" "$GOPATH/src/github.com/protocolbuffers/protobuf"
|
||||
export PATH="$GOPATH/bin:$PATH"
|
||||
go get github.com/golang/protobuf/protoc-gen-go
|
||||
|
||||
# build go benchmark
|
||||
cd benchmarks
|
||||
make go-benchmark
|
||||
echo "benchmarking go..."
|
||||
./go-benchmark $datasets > tmp/go_result.txt
|
||||
|
||||
# build java benchmark
|
||||
make java-benchmark
|
||||
echo "benchmarking java..."
|
||||
./java-benchmark -Cresults.file.options.file="tmp/java_result.json" $datasets
|
||||
|
||||
make js-benchmark
|
||||
echo "benchmarking js..."
|
||||
./js-benchmark $datasets --json_output=$(pwd)/tmp/node_result.json
|
||||
|
||||
make -j8 generate_proto3_data
|
||||
proto3_datasets=$(for file in $datasets; do echo $(pwd)/tmp/proto3_data/${file#$(pwd)}; done | xargs)
|
||||
echo $proto3_datasets
|
||||
|
||||
# build php benchmark
|
||||
make -j8 php-benchmark
|
||||
echo "benchmarking php..."
|
||||
./php-benchmark $proto3_datasets --json --behavior_prefix="php" > tmp/php_result.json
|
||||
make -j8 php-c-benchmark
|
||||
echo "benchmarking php_c..."
|
||||
./php-c-benchmark $proto3_datasets --json --behavior_prefix="php_c" > tmp/php_c_result.json
|
||||
|
||||
# upload result to bq
|
||||
make python_add_init
|
||||
env LD_LIBRARY_PATH="$oldpwd/src/.libs" python -m util.result_uploader -php="../tmp/php_result.json" -php_c="../tmp/php_c_result.json" \
|
||||
-cpp="../tmp/cpp_result.json" -java="../tmp/java_result.json" -go="../tmp/go_result.txt" -python="../tmp/python_result.json" -node="../tmp/node_result.json"
|
||||
cd $oldpwd
|
||||
export TEST_SET="benchmark"
|
||||
./kokoro/linux/build_and_run_docker.sh
|
||||
|
||||
105
kokoro/linux/benchmark/run.sh
Executable file
105
kokoro/linux/benchmark/run.sh
Executable file
@ -0,0 +1,105 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Change to repo root
|
||||
cd $(dirname $0)/../../..
|
||||
|
||||
export OUTPUT_DIR=testoutput
|
||||
oldpwd=`pwd`
|
||||
|
||||
# tcmalloc
|
||||
if [ ! -f gperftools/.libs/libtcmalloc.so ]; then
|
||||
git clone https://github.com/gperftools/gperftools.git
|
||||
cd gperftools
|
||||
./autogen.sh
|
||||
./configure
|
||||
make -j8
|
||||
cd ..
|
||||
fi
|
||||
|
||||
# download datasets for benchmark
|
||||
cd benchmarks
|
||||
./download_data.sh
|
||||
datasets=$(for file in $(find . -type f -name "dataset.*.pb" -not -path "./tmp/*"); do echo "$(pwd)/$file"; done | xargs)
|
||||
echo $datasets
|
||||
cd $oldpwd
|
||||
|
||||
# build Python protobuf
|
||||
./autogen.sh
|
||||
./configure CXXFLAGS="-fPIC -O2"
|
||||
make -j8
|
||||
cd python
|
||||
python setup.py build --cpp_implementation
|
||||
pip install . --user
|
||||
|
||||
|
||||
# build and run Python benchmark
|
||||
cd ../benchmarks
|
||||
make python-pure-python-benchmark
|
||||
make python-cpp-reflection-benchmark
|
||||
make -j8 python-cpp-generated-code-benchmark
|
||||
echo "[" > tmp/python_result.json
|
||||
echo "benchmarking pure python..."
|
||||
./python-pure-python-benchmark --json --behavior_prefix="pure-python-benchmark" $datasets >> tmp/python_result.json
|
||||
echo "," >> "tmp/python_result.json"
|
||||
echo "benchmarking python cpp reflection..."
|
||||
env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" LD_LIBRARY_PATH="$oldpwd/src/.libs" ./python-cpp-reflection-benchmark --json --behavior_prefix="cpp-reflection-benchmark" $datasets >> tmp/python_result.json
|
||||
echo "," >> "tmp/python_result.json"
|
||||
echo "benchmarking python cpp generated code..."
|
||||
env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" LD_LIBRARY_PATH="$oldpwd/src/.libs" ./python-cpp-generated-code-benchmark --json --behavior_prefix="cpp-generated-code-benchmark" $datasets >> tmp/python_result.json
|
||||
echo "]" >> "tmp/python_result.json"
|
||||
cd $oldpwd
|
||||
|
||||
# build CPP protobuf
|
||||
./configure
|
||||
make clean && make -j8
|
||||
|
||||
# build Java protobuf
|
||||
cd java
|
||||
mvn package
|
||||
cd ..
|
||||
|
||||
# build CPP benchmark
|
||||
cd benchmarks
|
||||
mv tmp/python_result.json . && make clean && make -j8 cpp-benchmark && mv python_result.json tmp
|
||||
echo "benchmarking cpp..."
|
||||
env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" ./cpp-benchmark --benchmark_min_time=5.0 --benchmark_out_format=json --benchmark_out="tmp/cpp_result.json" $datasets
|
||||
cd $oldpwd
|
||||
|
||||
# build go protobuf
|
||||
export PATH="`pwd`/src:$PATH"
|
||||
export GOPATH="$HOME/gocode"
|
||||
mkdir -p "$GOPATH/src/github.com/google"
|
||||
rm -f "$GOPATH/src/github.com/protocolbuffers/protobuf"
|
||||
ln -s "`pwd`" "$GOPATH/src/github.com/protocolbuffers/protobuf"
|
||||
export PATH="$GOPATH/bin:$PATH"
|
||||
go get github.com/golang/protobuf/protoc-gen-go
|
||||
|
||||
# build go benchmark
|
||||
cd benchmarks
|
||||
make go-benchmark
|
||||
echo "benchmarking go..."
|
||||
./go-benchmark $datasets > tmp/go_result.txt
|
||||
|
||||
# build java benchmark
|
||||
make java-benchmark
|
||||
echo "benchmarking java..."
|
||||
./java-benchmark -Cresults.file.options.file="tmp/java_result.json" $datasets
|
||||
|
||||
make js-benchmark
|
||||
echo "benchmarking js..."
|
||||
./js-benchmark $datasets --json_output=$(pwd)/tmp/node_result.json
|
||||
|
||||
make -j8 generate_proto3_data
|
||||
proto3_datasets=$(for file in $datasets; do echo $(pwd)/tmp/proto3_data/${file#$(pwd)}; done | xargs)
|
||||
echo $proto3_datasets
|
||||
|
||||
# build php benchmark
|
||||
make -j8 php-c-benchmark
|
||||
echo "benchmarking php_c..."
|
||||
./php-c-benchmark $proto3_datasets --json --behavior_prefix="php_c" > tmp/php_c_result.json
|
||||
|
||||
# upload result to bq
|
||||
make python_add_init
|
||||
env LD_LIBRARY_PATH="$oldpwd/src/.libs" python -m util.result_uploader -php_c="../tmp/php_c_result.json" \
|
||||
-cpp="../tmp/cpp_result.json" -java="../tmp/java_result.json" -go="../tmp/go_result.txt" -python="../tmp/python_result.json" -node="../tmp/node_result.json"
|
||||
cd $oldpwd
|
||||
8
tests.sh
8
tests.sh
@ -524,6 +524,11 @@ build_php_all() {
|
||||
build_php_compatibility
|
||||
}
|
||||
|
||||
build_benchmark() {
|
||||
use_php 7.1
|
||||
cd kokoro/linux/benchmark && ./run.sh
|
||||
}
|
||||
|
||||
# -------- main --------
|
||||
|
||||
if [ "$#" -ne 1 ]; then
|
||||
@ -556,7 +561,8 @@ Usage: $0 { cpp |
|
||||
php_compatibility |
|
||||
php7.1 |
|
||||
php7.1_c |
|
||||
php_all)
|
||||
php_all |
|
||||
benchmark)
|
||||
"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
Loading…
Reference in New Issue
Block a user