Fix bugs
This commit is contained in:
parent
5e732e35c1
commit
34843eddfe
@ -136,7 +136,7 @@ javac_middleman: $(java_benchmark_testing_files) protoc_middleman protoc_middlem
|
||||
java-benchmark: javac_middleman
|
||||
@echo "Writing shortcut script java-benchmark..."
|
||||
@echo '#! /bin/sh' > java-benchmark
|
||||
@echo 'java -cp '"tmp/java/target/*.jar"' com.google.protocolbuffers.ProtoBench $$@' >> java-benchmark
|
||||
@echo 'java -cp '"tmp/java/target/*.jar"' com.google.protobuf.ProtoBench $$@' >> java-benchmark
|
||||
@chmod +x java-benchmark
|
||||
|
||||
java: protoc_middleman protoc_middleman2 java-benchmark
|
||||
|
@ -5,19 +5,30 @@ This directory contains benchmarking schemas and data sets that you
|
||||
can use to test a variety of performance scenarios against your
|
||||
protobuf language runtime.
|
||||
|
||||
The schema for the datasets is described in `benchmarks.proto`.
|
||||
## Benchmark tools and build instructions
|
||||
|
||||
The benchmark is based on some submodules. To initialize the submodues:
|
||||
First, you need to follow the instruction in the root directory's README to
|
||||
build your language's protobuf, then:
|
||||
|
||||
### CPP
|
||||
We are using [google/benchmark](https://github.com/google/benchmark) as the
|
||||
benchmark tool for testing cpp. This is included as submodule under third_party
|
||||
directory. To init and build this tools, you need to do this under root dirctory:
|
||||
|
||||
For java:
|
||||
```
|
||||
$ ./initialize_submodule.sh java
|
||||
$ cd third_party
|
||||
$ git submodule update --init -r
|
||||
$ cd benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make && cd ../..
|
||||
```
|
||||
|
||||
For java:
|
||||
```
|
||||
$ ./initialize_submodule.sh cpp
|
||||
```
|
||||
### JAVA
|
||||
We're using maven to build the java benchmarks, which is the same as to build
|
||||
the Java protobuf. There're no other tools need to install. We're using
|
||||
[google/caliper](https://github.com/google/caliper) as benchmark tool, which
|
||||
can be automaticly included by maven.
|
||||
|
||||
|
||||
## Run instructions
|
||||
|
||||
To run all the benchmark dataset:
|
||||
|
||||
@ -49,6 +60,8 @@ $ make cpp-benchmark
|
||||
$ ./cpp-benchmark $(specific generated dataset file name)
|
||||
```
|
||||
|
||||
## Benchmark datasets
|
||||
|
||||
There's some big testing data not included in the directory initially, you need to
|
||||
run the following command to download the testing data:
|
||||
|
||||
@ -57,10 +70,13 @@ $ ./download_data.sh
|
||||
```
|
||||
|
||||
Each data set is in the format of benchmarks.proto:
|
||||
|
||||
1. name is the benchmark dataset's name.
|
||||
2. message_name is the benchmark's message type full name (including package and message name)
|
||||
3. payload is the list of raw data.
|
||||
|
||||
The schema for the datasets is described in `benchmarks.proto`.
|
||||
|
||||
Benchmark likely want to run several benchmarks against each data set (parse,
|
||||
serialize, possibly JSON, possibly using different APIs, etc).
|
||||
|
||||
|
@ -1,7 +0,0 @@
|
||||
#! /bin/sh
|
||||
|
||||
oldpwd=`pwd`
|
||||
cd "../third_party"
|
||||
git submodule update --init -r
|
||||
cd benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make && cd ..
|
||||
cd "$oldpwd"
|
@ -1,85 +1,59 @@
|
||||
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
~ Copyright 2011 Google Inc.
|
||||
~
|
||||
~ Licensed under the Apache License, Version 2.0 (the "License");
|
||||
~ you may not use this file except in compliance with the License.
|
||||
~ You may obtain a copy of the License at
|
||||
~
|
||||
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing, software
|
||||
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<!-- <parent>
|
||||
<groupId>com.google.protobuf</groupId>
|
||||
<artifactId>protobuf-parent</artifactId>
|
||||
<version>3.5.0</version>
|
||||
</parent>
|
||||
|
||||
<packaging>pom</packaging> -->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
||||
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<artifactId>protobuf-java-benchmark</artifactId>
|
||||
<groupId>com.google.protocolbuffers</groupId>
|
||||
<groupId>com.google.protobuf</groupId>
|
||||
<version>1.0.0</version>
|
||||
<name>Protocol Buffers [Benchmark]</name>
|
||||
<description>
|
||||
The benchmark tools for Protobuf Java.
|
||||
</description>
|
||||
|
||||
<description>The benchmark tools for Protobuf Java.</description>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.google.protobuf</groupId>
|
||||
<artifactId>protobuf-java</artifactId>
|
||||
<version>3.5.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.google.caliper</groupId>
|
||||
<artifactId>caliper</artifactId>
|
||||
<version>1.0-beta-2</version>
|
||||
</dependency>
|
||||
<groupId>com.google.protobuf</groupId>
|
||||
<artifactId>protobuf-java</artifactId>
|
||||
<version>3.5.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.google.caliper</groupId>
|
||||
<artifactId>caliper</artifactId>
|
||||
<version>1.0-beta-2</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
|
||||
<build>
|
||||
<pluginManagement>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<version>2.4.1</version>
|
||||
<configuration>
|
||||
<!-- get all project dependencies -->
|
||||
<descriptorRefs>
|
||||
<descriptorRef>jar-with-dependencies</descriptorRef>
|
||||
</descriptorRefs>
|
||||
<!-- MainClass in mainfest make a executable jar -->
|
||||
<archive>
|
||||
<manifest>
|
||||
<mainClass>com.mkyong.core.utils.App</mainClass>
|
||||
</manifest>
|
||||
</archive>
|
||||
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>make-assembly</id>
|
||||
<!-- bind to the packaging phase -->
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>single</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<version>2.4.1</version>
|
||||
<configuration>
|
||||
<!-- get all project dependencies -->
|
||||
<descriptorRefs>
|
||||
<descriptorRef>jar-with-dependencies</descriptorRef>
|
||||
</descriptorRefs>
|
||||
<!-- MainClass in mainfest make a executable jar -->
|
||||
<archive>
|
||||
<manifest>
|
||||
<mainClass>com.mkyong.core.utils.App</mainClass>
|
||||
</manifest>
|
||||
</archive>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>make-assembly</id>
|
||||
<!-- bind to the packaging phase -->
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>single</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.5.1</version>
|
||||
@ -114,35 +88,7 @@
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-shade-plugin</artifactId>
|
||||
<version>2.3</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>shade</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<shadedArtifactAttached>true</shadedArtifactAttached>
|
||||
<shadedClassifierName>all</shadedClassifierName>
|
||||
<transformers>
|
||||
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
|
||||
<manifestEntries>
|
||||
<Premain-Class>com.google.monitoring.runtime.instrumentation.AllocationInstrumenter</Premain-Class>
|
||||
<Can-Redefine-Classes>true</Can-Redefine-Classes>
|
||||
<Can-Retransform-Classes>true</Can-Retransform-Classes>
|
||||
</manifestEntries>
|
||||
</transformer>
|
||||
</transformers>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</pluginManagement>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
|
||||
</project>
|
@ -29,7 +29,7 @@
|
||||
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
|
||||
package com.google.protocolbuffers;
|
||||
package com.google.protobuf;
|
||||
|
||||
import com.google.caliper.BeforeExperiment;
|
||||
import com.google.caliper.Benchmark;
|
||||
@ -93,7 +93,7 @@ public class ProtoBench {
|
||||
return false;
|
||||
}
|
||||
argsList.add("-DdataFile=" + file);
|
||||
argsList.add("com.google.protocolbuffers.ProtoBenchCaliper");
|
||||
argsList.add("com.google.protobuf.ProtoBenchCaliper");
|
||||
|
||||
try {
|
||||
String args[] = new String[argsList.size()];
|
||||
@ -121,17 +121,9 @@ public class ProtoBench {
|
||||
} else if (benchmarkDataset.getMessageName().
|
||||
equals("benchmarks.google_message3.GoogleMessage3")) {
|
||||
temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE3");
|
||||
for (String opt : ProtoBenchCaliper
|
||||
.BenchmarkMessageType.GOOGLE_MESSAGE3.getSpecificCaliperOption()) {
|
||||
temp.add(opt);
|
||||
}
|
||||
} else if (benchmarkDataset.getMessageName().
|
||||
equals("benchmarks.google_message4.GoogleMessage4")) {
|
||||
temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE4");
|
||||
for (String opt : ProtoBenchCaliper
|
||||
.BenchmarkMessageType.GOOGLE_MESSAGE4.getSpecificCaliperOption()) {
|
||||
temp.add(opt);
|
||||
}
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
@ -142,6 +134,7 @@ public class ProtoBench {
|
||||
String benchmarkNames = "serializeToByteString,serializeToByteArray,serializeToMemoryStream"
|
||||
+ ",deserializeFromByteString,deserializeFromByteArray,deserializeFromMemoryStream";
|
||||
temp.add(benchmarkNames);
|
||||
temp.add("-Cinstrument.runtime.options.timingInterval=3000ms");
|
||||
|
||||
return temp;
|
||||
}
|
@ -1,5 +1,5 @@
|
||||
|
||||
package com.google.protocolbuffers;
|
||||
package com.google.protobuf;
|
||||
|
||||
import com.google.caliper.BeforeExperiment;
|
||||
import com.google.caliper.Benchmark;
|
||||
@ -26,7 +26,6 @@ public class ProtoBenchCaliper {
|
||||
return com.google.protobuf.benchmarks.BenchmarkMessage1Proto3.GoogleMessage1
|
||||
.getDefaultInstance();
|
||||
}
|
||||
@Override String[] getSpecificCaliperOption() { return null; }
|
||||
},
|
||||
GOOGLE_MESSAGE1_PROTO2 {
|
||||
@Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); }
|
||||
@ -35,7 +34,6 @@ public class ProtoBenchCaliper {
|
||||
return com.google.protobuf.benchmarks.BenchmarkMessage1Proto2.GoogleMessage1
|
||||
.getDefaultInstance();
|
||||
}
|
||||
@Override String[] getSpecificCaliperOption() { return null; }
|
||||
},
|
||||
GOOGLE_MESSAGE2 {
|
||||
@Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); }
|
||||
@ -43,7 +41,6 @@ public class ProtoBenchCaliper {
|
||||
Message getDefaultInstance() {
|
||||
return com.google.protobuf.benchmarks.BenchmarkMessage2.GoogleMessage2.getDefaultInstance();
|
||||
}
|
||||
@Override String[] getSpecificCaliperOption() { return null; }
|
||||
},
|
||||
GOOGLE_MESSAGE3 {
|
||||
@Override
|
||||
@ -64,12 +61,6 @@ public class ProtoBenchCaliper {
|
||||
Message getDefaultInstance() {
|
||||
return benchmarks.google_message3.BenchmarkMessage3.GoogleMessage3.getDefaultInstance();
|
||||
}
|
||||
@Override
|
||||
String[] getSpecificCaliperOption() {
|
||||
String[] opt = new String[1];
|
||||
opt[0] = "-Cinstrument.runtime.options.timingInterval=3000ms";
|
||||
return opt;
|
||||
}
|
||||
},
|
||||
GOOGLE_MESSAGE4 {
|
||||
@Override
|
||||
@ -85,17 +76,10 @@ public class ProtoBenchCaliper {
|
||||
Message getDefaultInstance() {
|
||||
return benchmarks.google_message4.BenchmarkMessage4.GoogleMessage4.getDefaultInstance();
|
||||
}
|
||||
@Override
|
||||
String[] getSpecificCaliperOption() {
|
||||
String[] opt = new String[1];
|
||||
opt[0] = "-Cinstrument.runtime.options.timingInterval=3000ms";
|
||||
return opt;
|
||||
}
|
||||
};
|
||||
|
||||
abstract ExtensionRegistry getExtensionRegistry();
|
||||
abstract Message getDefaultInstance();
|
||||
abstract String[] getSpecificCaliperOption();
|
||||
}
|
||||
|
||||
@Param
|
||||
@ -112,8 +96,6 @@ public class ProtoBenchCaliper {
|
||||
private List<ByteString> inputStringList;
|
||||
private List<Message> sampleMessageList;
|
||||
private int counter;
|
||||
private FileOutputStream devNull;
|
||||
private CodedOutputStream reuseDevNull;
|
||||
|
||||
@BeforeExperiment
|
||||
void setUp() throws IOException {
|
||||
@ -135,15 +117,6 @@ public class ProtoBenchCaliper {
|
||||
sampleMessageList.add(
|
||||
defaultMessage.newBuilderForType().mergeFrom(singleInputData, extensions).build());
|
||||
}
|
||||
devNull = null;
|
||||
reuseDevNull = null;
|
||||
|
||||
try {
|
||||
devNull = new FileOutputStream("/dev/null");
|
||||
reuseDevNull = CodedOutputStream.newInstance(devNull);
|
||||
} catch (FileNotFoundException e) {
|
||||
// ignore: this is probably Windows, where /dev/null does not exist
|
||||
}
|
||||
|
||||
counter = 0;
|
||||
}
|
5
tests.sh
5
tests.sh
@ -44,7 +44,10 @@ build_cpp() {
|
||||
# appears to be missing it: https://github.com/travis-ci/travis-ci/issues/6996
|
||||
if [[ $(type cmake 2>/dev/null) ]]; then
|
||||
# Verify benchmarking code can build successfully.
|
||||
cd benchmarks && ./initialize_submodule.sh cpp && make cpp-benchmark && cd ..
|
||||
git submodule init
|
||||
git submodule update
|
||||
cd third_party/benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make && cd ../..
|
||||
cd benchmarks && make cpp-benchmark && cd ..
|
||||
else
|
||||
echo ""
|
||||
echo "WARNING: Skipping validation of the bench marking code, cmake isn't installed."
|
||||
|
Loading…
Reference in New Issue
Block a user