AuroraMiddleware/protobuf
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

Merge pull request #4014 from BSBandme/JavaCaliper

Browse Source 
Add caliper supported to java benchmark
This commit is contained in:
Yilun Chong 2018-01-03 09:42:26 -08:00 committed by GitHub
commit 43caa38d6e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 489 additions and 338 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
benchmarks/Makefile.am Normal file → Executable file
View File

@ -20,7 +20,7 @@ benchmarks_protoc_inputs_proto2 = \
datasets/google_message4/benchmark_message4_3.proto
make_tmp_dir:
mkdir -p 'tmp'
mkdir -p 'tmp/java/src/main/java'
touch make_tmp_dir
if USE_EXTERNAL_PROTOC
@ -39,11 +39,11 @@ else
# relative to srcdir, which may not be the same as the current directory when
# building out-of-tree.
protoc_middleman: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs) $(well_known_type_protoc_inputs)
oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp $(benchmarks_protoc_inputs) )
oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp/java/src/main/java $(benchmarks_protoc_inputs) )
touch protoc_middleman
protoc_middleman2: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs_proto2) $(well_known_type_protoc_inputs)
oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2) )
oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp/java/src/main/java $(benchmarks_protoc_inputs_proto2) )
touch protoc_middleman2
endif
@ -100,6 +100,17 @@ $(benchmarks_protoc_outputs_header): protoc_middleman
$(benchmarks_protoc_outputs_proto2): protoc_middleman2
$(benchmarks_protoc_outputs_proto2_header): protoc_middleman2
initialize_submodule:
oldpwd=`pwd`
cd $(top_srcdir)/third_party
git submodule update --init -r
cd $(top_srcdir)/third_party/benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make
cd $$oldpwd
touch initialize_submodule
$(top_srcdir)/src/libprotobuf.la: initialize_submodule
$(top_srcdir)/third_party/benchmark/src/libbenchmark.a: initialize_submodule
AM_CXXFLAGS = $(NO_OPT_CXXFLAGS) $(PROTOBUF_OPT_FLAG) -Wall -Wwrite-strings -Woverloaded-virtual -Wno-sign-compare
bin_PROGRAMS = cpp-benchmark
@ -110,32 +121,33 @@ cpp_benchmark_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir) -I$(top_srcdir)/third_p
# Explicit deps because BUILT_SOURCES are only done before a "make all/check"
# so a direct "make test_cpp" could fail if parallel enough.
# See: https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html#Recording-Dependencies-manually
cpp_benchmark-cpp_benchmark.$(OBJEXT): $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2) $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header)
cpp_benchmark-cpp_benchmark.$(OBJEXT): $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2) $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header) $(top_srcdir)/src/libprotobuf.la $(top_srcdir)/third_party/benchmark/src/libbenchmark.a
nodist_cpp_benchmark_SOURCES = \
$(benchmarks_protoc_outputs) \
$(benchmarks_protoc_outputs_proto2) \
$(benchmarks_protoc_outputs_proto2_header) \
$(benchmarks_protoc_outputs_header)
cpp: protoc_middleman protoc_middleman2 cpp-benchmark
cpp: protoc_middleman protoc_middleman2 cpp-benchmark initialize_submodule
./cpp-benchmark $(all_data)
############ CPP RULES END ############
############# JAVA RULES ##############
javac_middleman: ProtoBench.java protoc_middleman protoc_middleman2
jar=`ls $(top_srcdir)/java/util/target/*.jar` && \
jar1=`ls $(top_srcdir)/java/core/target/*.jar` && \
javac -d tmp -classpath ../java/target/classes:$$jar:$$jar1 ProtoBench.java $(benchmark_java_protoc_outputs_proto2) $(benchmarks_java_protoc_outputs) `find tmp -type f -name "*.java"`
java_benchmark_testing_files = \
java/src/main/java/com/google/protobuf/ProtoBench.java \
java/src/main/java/com/google/protobuf/ProtoBenchCaliper.java
javac_middleman: $(java_benchmark_testing_files) protoc_middleman protoc_middleman2
cp -r java tmp && cd tmp/java && mvn clean compile assembly:single
cd ../..
@touch javac_middleman
java-benchmark: javac_middleman
@echo "Writing shortcut script java-benchmark..."
@echo '#! /bin/sh' > java-benchmark
@jar=`ls $(top_srcdir)/java/util/target/*.jar` && \
jar1=`ls $(top_srcdir)/java/core/target/*.jar` && \
echo java -classpath tmp:../java/target/classes:$$jar:$$jar1 com.google.protocolbuffers.ProtoBench '$$@' >> java-benchmark
@echo 'java -cp '"tmp/java/target/*.jar"' com.google.protobuf.ProtoBench $$@' >> java-benchmark
@chmod +x java-benchmark
java: protoc_middleman protoc_middleman2 java-benchmark
@ -151,6 +163,7 @@ CLEANFILES = \
$(benchmarks_protoc_outputs_header) \
$(benchmarks_protoc_outputs_proto2) \
$(benchmarks_protoc_outputs_proto2_header) \
initialize_submodule \
make_tmp_dir \
protoc_middleman \
protoc_middleman2 \

319
benchmarks/ProtoBench.java
View File

@ -1,319 +0,0 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2009 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.protocolbuffers;
import com.google.protobuf.ByteString;
import com.google.protobuf.CodedInputStream;
import com.google.protobuf.CodedOutputStream;
import com.google.protobuf.ExtensionRegistry;
import com.google.protobuf.Message;
import com.google.protobuf.benchmarks.Benchmarks.BenchmarkDataset;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.ArrayList;
import java.util.List;
public class ProtoBench {
private static final long MIN_SAMPLE_TIME_MS = 1 * 1000;
private static final long TARGET_TIME_MS = 5 * 1000;
private ProtoBench() {
// Prevent instantiation
}
public static void main(String[] args) {
if (args.length < 1) {
System.err.println("Usage: ./java-benchmark <input data>");
System.err.println("input data is in the format of \"benchmarks.proto\"");
System.exit(1);
}
boolean success = true;
for (int i = 0; i < args.length; i++) {
success &= runTest(args[i]);
}
System.exit(success ? 0 : 1);
}
public static ExtensionRegistry getExtensionsRegistry(BenchmarkDataset benchmarkDataset) {
ExtensionRegistry extensions = ExtensionRegistry.newInstance();
if (benchmarkDataset.getMessageName().equals("benchmarks.google_message3.GoogleMessage3")) {
benchmarks.google_message3.BenchmarkMessage38.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage37.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage36.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage35.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage34.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage33.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage32.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage31.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage3.registerAllExtensions(extensions);
} else if (benchmarkDataset.getMessageName().equals(
"benchmarks.google_message4.GoogleMessage4")) {
benchmarks.google_message4.BenchmarkMessage43.registerAllExtensions(extensions);
benchmarks.google_message4.BenchmarkMessage42.registerAllExtensions(extensions);
benchmarks.google_message4.BenchmarkMessage41.registerAllExtensions(extensions);
benchmarks.google_message4.BenchmarkMessage4.registerAllExtensions(extensions);
}
return extensions;
}
/**
* Return an message instance for one specific dataset, and register the extensions for that
* message.
*/
public static Message registerBenchmarks(BenchmarkDataset benchmarkDataset) {
if (benchmarkDataset.getMessageName().equals("benchmarks.proto3.GoogleMessage1")) {
return com.google.protobuf.benchmarks.BenchmarkMessage1Proto3.GoogleMessage1
.getDefaultInstance();
} else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage1")) {
return com.google.protobuf.benchmarks.BenchmarkMessage1Proto2.GoogleMessage1
.getDefaultInstance();
} else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage2")) {
return com.google.protobuf.benchmarks.BenchmarkMessage2.GoogleMessage2.getDefaultInstance();
} else if (benchmarkDataset.getMessageName().
equals("benchmarks.google_message3.GoogleMessage3")) {
return benchmarks.google_message3.BenchmarkMessage3.GoogleMessage3.getDefaultInstance();
} else if (benchmarkDataset.getMessageName().
equals("benchmarks.google_message4.GoogleMessage4")) {
return benchmarks.google_message4.BenchmarkMessage4.GoogleMessage4.getDefaultInstance();
} else {
return null;
}
}
/**
* Runs a single test. Error messages are displayed to stderr, and the return value indicates
* general success/failure.
*/
public static boolean runTest(String file) {
final Message defaultMessage;
BenchmarkDataset benchmarkDataset;
ExtensionRegistry extensions;
final byte[] inputData;
try {
inputData = readAllBytes(file);
benchmarkDataset = BenchmarkDataset.parseFrom(inputData);
} catch (IOException e) {
System.err.println("Unable to get input data");
return false;
}
defaultMessage = registerBenchmarks(benchmarkDataset);
extensions = getExtensionsRegistry(benchmarkDataset);
if (defaultMessage == null) {
System.err.println("Unable to get default message " + benchmarkDataset.getMessageName());
return false;
}
System.out.println("Benchmarking " + benchmarkDataset.getMessageName() + " with file " + file);
try {
List<byte[]> inputDataList = new ArrayList<byte[]>();
List<ByteArrayInputStream> inputStreamList = new ArrayList<ByteArrayInputStream>();
List<ByteString> inputStringList = new ArrayList<ByteString>();
List<Message> sampleMessageList = new ArrayList<Message>();
for (int i = 0; i < benchmarkDataset.getPayloadCount(); i++) {
byte[] singleInputData = benchmarkDataset.getPayload(i).toByteArray();
inputDataList.add(benchmarkDataset.getPayload(i).toByteArray());
inputStreamList.add(new ByteArrayInputStream(benchmarkDataset.getPayload(i).toByteArray()));
inputStringList.add(benchmarkDataset.getPayload(i));
sampleMessageList.add(
defaultMessage.newBuilderForType().mergeFrom(singleInputData, extensions).build());
}
FileOutputStream devNullTemp = null;
CodedOutputStream reuseDevNullTemp = null;
try {
devNullTemp = new FileOutputStream("/dev/null");
reuseDevNullTemp = CodedOutputStream.newInstance(devNullTemp);
} catch (FileNotFoundException e) {
// ignore: this is probably Windows, where /dev/null does not exist
}
final FileOutputStream devNull = devNullTemp;
final CodedOutputStream reuseDevNull = reuseDevNullTemp;
benchmark(
"Serialize to byte string",
inputData.length,
new Action() {
public void execute() {
for (int i = 0; i < sampleMessageList.size(); i++) {
sampleMessageList.get(i).toByteString();
}
}
});
benchmark(
"Serialize to byte array",
inputData.length,
new Action() {
public void execute() {
for (int i = 0; i < sampleMessageList.size(); i++) {
sampleMessageList.get(i).toByteString();
}
}
});
benchmark(
"Serialize to memory stream",
inputData.length,
new Action() {
public void execute() throws IOException {
ByteArrayOutputStream output = new ByteArrayOutputStream();
for (int i = 0; i < sampleMessageList.size(); i++) {
sampleMessageList.get(i).writeTo(output);
}
}
});
if (devNull != null) {
benchmark(
"Serialize to /dev/null with FileOutputStream",
inputData.length,
new Action() {
public void execute() throws IOException {
for (int i = 0; i < sampleMessageList.size(); i++) {
sampleMessageList.get(i).writeTo(devNull);
}
}
});
benchmark(
"Serialize to /dev/null reusing FileOutputStream",
inputData.length,
new Action() {
public void execute() throws IOException {
for (int i = 0; i < sampleMessageList.size(); i++) {
sampleMessageList.get(i).writeTo(reuseDevNull);
reuseDevNull.flush(); // force the write to the OutputStream
}
}
});
}
benchmark(
"Deserialize from byte string",
inputData.length,
new Action() {
public void execute() throws IOException {
for (int i = 0; i < inputStringList.size(); i++) {
defaultMessage
.newBuilderForType()
.mergeFrom(inputStringList.get(i), extensions)
.build();
}
}
});
benchmark(
"Deserialize from byte array",
inputData.length,
new Action() {
public void execute() throws IOException {
for (int i = 0; i < inputDataList.size(); i++) {
defaultMessage
.newBuilderForType()
.mergeFrom(CodedInputStream.newInstance(inputDataList.get(i)), extensions)
.build();
}
}
});
benchmark(
"Deserialize from memory stream",
inputData.length,
new Action() {
public void execute() throws IOException {
for (int i = 0; i < inputStreamList.size(); i++) {
defaultMessage
.newBuilderForType()
.mergeFrom(CodedInputStream.newInstance(inputStreamList.get(i)), extensions)
.build();
inputStreamList.get(i).reset();
}
}
});
System.out.println();
return true;
} catch (Exception e) {
System.err.println("Error: " + e.getMessage());
System.err.println("Detailed exception information:");
e.printStackTrace(System.err);
return false;
}
}
private static void benchmark(String name, long dataSize, Action action) throws IOException {
// Make sure it's JITted "reasonably" hard before running the first progress test
for (int i=0; i < 100; i++) {
action.execute();
}
// Run it progressively more times until we've got a reasonable sample
int iterations = 1;
long elapsed = timeAction(action, iterations);
while (elapsed < MIN_SAMPLE_TIME_MS) {
iterations *= 2;
elapsed = timeAction(action, iterations);
}
// Upscale the sample to the target time. Do this in floating point arithmetic
// to avoid overflow issues.
iterations = (int) ((TARGET_TIME_MS / (double) elapsed) * iterations);
elapsed = timeAction(action, iterations);
System.out.println(name + ": " + iterations + " iterations in "
+ (elapsed/1000f) + "s; "
+ (iterations * dataSize) / (elapsed * 1024 * 1024 / 1000f)
+ "MB/s");
}
private static long timeAction(Action action, int iterations) throws IOException {
System.gc();
long start = System.currentTimeMillis();
for (int i = 0; i < iterations; i++) {
action.execute();
}
long end = System.currentTimeMillis();
return end - start;
}
private static byte[] readAllBytes(String filename) throws IOException {
RandomAccessFile file = new RandomAccessFile(new File(filename), "r");
byte[] content = new byte[(int) file.length()];
file.readFully(content);
return content;
}
/**
* Interface used to capture a single action to benchmark.
*/
interface Action {
void execute() throws IOException;
}
}

42
benchmarks/README.md
View File

@ -5,7 +5,36 @@ This directory contains benchmarking schemas and data sets that you
can use to test a variety of performance scenarios against your
protobuf language runtime.
The schema for the datasets is described in `benchmarks.proto`.
## Prerequisite
First, you need to follow the instruction in the root directory's README to
build your language's protobuf, then:
### CPP
You need to install [cmake](https://cmake.org/) before building the benchmark.
We are using [google/benchmark](https://github.com/google/benchmark) as the
benchmark tool for testing cpp. This will be automaticly made during build the
cpp benchmark.
### JAVA
We're using maven to build the java benchmarks, which is the same as to build
the Java protobuf. There're no other tools need to install. We're using
[google/caliper](https://github.com/google/caliper) as benchmark tool, which
can be automaticly included by maven.
### Big data
There's some optional big testing data which is not included in the directory initially, you need to
run the following command to download the testing data:
```
$ ./download_data.sh
```
After doing this the big data file will automaticly generated in the benchmark directory.
## Run instructions
To run all the benchmark dataset:
@ -26,22 +55,27 @@ To run a specific dataset:
For java:
```
$ make java
$ ./java-benchmark $(specific generated dataset file name)
$ make java-benchmark
$ ./java-benchmark $(specific generated dataset file name) [-- $(caliper option)]
```
For cpp:
```
$ make cpp
$ make cpp-benchmark
$ ./cpp-benchmark $(specific generated dataset file name)
```
## Benchmark datasets
Each data set is in the format of benchmarks.proto:
1. name is the benchmark dataset's name.
2. message_name is the benchmark's message type full name (including package and message name)
3. payload is the list of raw data.
The schema for the datasets is described in `benchmarks.proto`.
Benchmark likely want to run several benchmarks against each data set (parse,
serialize, possibly JSON, possibly using different APIs, etc).

95
benchmarks/java/pom.xml Executable file
View File

@ -0,0 +1,95 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>protobuf-java-benchmark</artifactId>
<groupId>com.google.protobuf</groupId>
<version>1.0.0</version>
<name>Protocol Buffers [Benchmark]</name>
<description>The benchmark tools for Protobuf Java.</description>
<dependencies>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>3.5.0</version>
</dependency>
<dependency>
<groupId>com.google.caliper</groupId>
<artifactId>caliper</artifactId>
<version>1.0-beta-2</version>
</dependency>
</dependencies>
<build>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4.1</version>
<configuration>
<!-- get all project dependencies -->
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<!-- MainClass in mainfest make a executable jar -->
<archive>
<manifest>
<mainClass>com.mkyong.core.utils.App</mainClass>
</manifest>
</archive>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<!-- bind to the packaging phase -->
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.5.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>2.5</version>
<configuration>
<archive>
<manifest>
<addClasspath>true</addClasspath>
<mainClass>com.google.protocolbuffers.ProtoBench</mainClass>
</manifest>
</archive>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>2.4</version>
<executions>
<execution>
<id>attach-sources</id>
<goals>
<goal>jar-no-fork</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</pluginManagement>
</build>
</project>

148
benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java Executable file
View File

@ -0,0 +1,148 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2009 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.protobuf;
import com.google.caliper.BeforeExperiment;
import com.google.caliper.Benchmark;
import com.google.caliper.Param;
import com.google.caliper.runner.CaliperMain;
import com.google.protobuf.ByteString;
import com.google.protobuf.CodedInputStream;
import com.google.protobuf.CodedOutputStream;
import com.google.protobuf.ExtensionRegistry;
import com.google.protobuf.Message;
import com.google.protobuf.benchmarks.Benchmarks.BenchmarkDataset;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.RandomAccessFile;
import java.util.ArrayList;
import java.util.List;
public class ProtoBench {
private ProtoBench() {
// Prevent instantiation
}
public static void main(String[] args) {
if (args.length < 1) {
System.err.println("Usage: ./java-benchmark <input data>");
System.err.println("input data is in the format of \"benchmarks.proto\"");
System.exit(1);
}
boolean success = true;
for (int i = 0; i < args.length; i++) {
success &= runTest(args[i]);
}
System.exit(success ? 0 : 1);
}
/**
* Runs a single test with specific test data. Error messages are displayed to stderr,
* and the return value indicates general success/failure.
*/
public static boolean runTest(String file) {
byte[] inputData;
BenchmarkDataset benchmarkDataset;
try {
inputData = readAllBytes(file);
benchmarkDataset = BenchmarkDataset.parseFrom(inputData);
} catch (IOException e) {
System.err.println("Unable to get input data");
return false;
}
List<String> argsList = getCaliperOption(benchmarkDataset);
if (argsList == null) {
System.err.println("Unable to get default message " + benchmarkDataset.getMessageName());
return false;
}
argsList.add("-DdataFile=" + file);
argsList.add("com.google.protobuf.ProtoBenchCaliper");
try {
String args[] = new String[argsList.size()];
argsList.toArray(args);
CaliperMain.exitlessMain(args,
new PrintWriter(System.out, true), new PrintWriter(System.err, true));
return true;
} catch (Exception e) {
System.err.println("Error: " + e.getMessage());
System.err.println("Detailed exception information:");
e.printStackTrace(System.err);
return false;
}
}
private static List<String> getCaliperOption(final BenchmarkDataset benchmarkDataset) {
List<String> temp = new ArrayList<String>();
if (benchmarkDataset.getMessageName().equals("benchmarks.proto3.GoogleMessage1")) {
temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE1_PROTO3");
} else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage1")) {
temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE1_PROTO2");
} else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage2")) {
temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE2");
} else if (benchmarkDataset.getMessageName().
equals("benchmarks.google_message3.GoogleMessage3")) {
temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE3");
} else if (benchmarkDataset.getMessageName().
equals("benchmarks.google_message4.GoogleMessage4")) {
temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE4");
} else {
return null;
}
temp.add("-i");
temp.add("runtime");
temp.add("-b");
String benchmarkNames = "serializeToByteString,serializeToByteArray,serializeToMemoryStream"
+ ",deserializeFromByteString,deserializeFromByteArray,deserializeFromMemoryStream";
temp.add(benchmarkNames);
temp.add("-Cinstrument.runtime.options.timingInterval=3000ms");
return temp;
}
public static byte[] readAllBytes(String filename) throws IOException {
RandomAccessFile file = new RandomAccessFile(new File(filename), "r");
byte[] content = new byte[(int) file.length()];
file.readFully(content);
return content;
}
}

183
benchmarks/java/src/main/java/com/google/protobuf/ProtoBenchCaliper.java Executable file
View File

@ -0,0 +1,183 @@
package com.google.protobuf;
import com.google.caliper.BeforeExperiment;
import com.google.caliper.Benchmark;
import com.google.caliper.Param;
import com.google.protobuf.ByteString;
import com.google.protobuf.CodedOutputStream;
import com.google.protobuf.ExtensionRegistry;
import com.google.protobuf.Message;
import com.google.protobuf.benchmarks.Benchmarks.BenchmarkDataset;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public class ProtoBenchCaliper {
public enum BenchmarkMessageType {
GOOGLE_MESSAGE1_PROTO3 {
@Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); }
@Override
Message getDefaultInstance() {
return com.google.protobuf.benchmarks.BenchmarkMessage1Proto3.GoogleMessage1
.getDefaultInstance();
}
},
GOOGLE_MESSAGE1_PROTO2 {
@Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); }
@Override
Message getDefaultInstance() {
return com.google.protobuf.benchmarks.BenchmarkMessage1Proto2.GoogleMessage1
.getDefaultInstance();
}
},
GOOGLE_MESSAGE2 {
@Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); }
@Override
Message getDefaultInstance() {
return com.google.protobuf.benchmarks.BenchmarkMessage2.GoogleMessage2.getDefaultInstance();
}
},
GOOGLE_MESSAGE3 {
@Override
ExtensionRegistry getExtensionRegistry() {
ExtensionRegistry extensions = ExtensionRegistry.newInstance();
benchmarks.google_message3.BenchmarkMessage38.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage37.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage36.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage35.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage34.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage33.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage32.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage31.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage3.registerAllExtensions(extensions);
return extensions;
}
@Override
Message getDefaultInstance() {
return benchmarks.google_message3.BenchmarkMessage3.GoogleMessage3.getDefaultInstance();
}
},
GOOGLE_MESSAGE4 {
@Override
ExtensionRegistry getExtensionRegistry() {
ExtensionRegistry extensions = ExtensionRegistry.newInstance();
benchmarks.google_message4.BenchmarkMessage43.registerAllExtensions(extensions);
benchmarks.google_message4.BenchmarkMessage42.registerAllExtensions(extensions);
benchmarks.google_message4.BenchmarkMessage41.registerAllExtensions(extensions);
benchmarks.google_message4.BenchmarkMessage4.registerAllExtensions(extensions);
return extensions;
}
@Override
Message getDefaultInstance() {
return benchmarks.google_message4.BenchmarkMessage4.GoogleMessage4.getDefaultInstance();
}
};
abstract ExtensionRegistry getExtensionRegistry();
abstract Message getDefaultInstance();
}
@Param
private BenchmarkMessageType benchmarkMessageType;
@Param
private String dataFile;
private byte[] inputData;
private BenchmarkDataset benchmarkDataset;
private Message defaultMessage;
private ExtensionRegistry extensions;
private List<byte[]> inputDataList;
private List<ByteArrayInputStream> inputStreamList;
private List<ByteString> inputStringList;
private List<Message> sampleMessageList;
private int counter;
@BeforeExperiment
void setUp() throws IOException {
inputData = ProtoBench.readAllBytes(dataFile);
benchmarkDataset = BenchmarkDataset.parseFrom(inputData);
defaultMessage = benchmarkMessageType.getDefaultInstance();
extensions = benchmarkMessageType.getExtensionRegistry();
inputDataList = new ArrayList<byte[]>();
inputStreamList = new ArrayList<ByteArrayInputStream>();
inputStringList = new ArrayList<ByteString>();
sampleMessageList = new ArrayList<Message>();
for (int i = 0; i < benchmarkDataset.getPayloadCount(); i++) {
byte[] singleInputData = benchmarkDataset.getPayload(i).toByteArray();
inputDataList.add(benchmarkDataset.getPayload(i).toByteArray());
inputStreamList.add(new ByteArrayInputStream(
benchmarkDataset.getPayload(i).toByteArray()));
inputStringList.add(benchmarkDataset.getPayload(i));
sampleMessageList.add(
defaultMessage.newBuilderForType().mergeFrom(singleInputData, extensions).build());
}
counter = 0;
}
@Benchmark
void serializeToByteString(int reps) throws IOException {
for (int i = 0; i < reps; i++) {
sampleMessageList.get(counter % sampleMessageList.size()).toByteString();
counter++;
}
}
@Benchmark
void serializeToByteArray(int reps) throws IOException {
for (int i = 0; i < reps; i++) {
sampleMessageList.get(counter % sampleMessageList.size()).toByteArray();
counter++;
}
}
@Benchmark
void serializeToMemoryStream(int reps) throws IOException {
for (int i = 0; i < reps; i++) {
ByteArrayOutputStream output = new ByteArrayOutputStream();
sampleMessageList.get(counter % sampleMessageList.size()).writeTo(output);
counter++;
}
}
@Benchmark
void deserializeFromByteString(int reps) throws IOException {
for (int i = 0; i < reps; i++) {
defaultMessage
.newBuilderForType()
.mergeFrom(inputStringList.get(counter % inputStringList.size()), extensions)
.build();
counter++;
}
}
@Benchmark
void deserializeFromByteArray(int reps) throws IOException {
for (int i = 0; i < reps; i++) {
defaultMessage
.newBuilderForType()
.mergeFrom(inputDataList.get(counter % inputDataList.size()), extensions)
.build();
counter++;
}
}
@Benchmark
void deserializeFromMemoryStream(int reps) throws IOException {
for (int i = 0; i < reps; i++) {
defaultMessage
.newBuilderForType()
.mergeFrom(inputStreamList.get(counter % inputStreamList.size()), extensions)
.build();
inputStreamList.get(counter % inputStreamList.size()).reset();
counter++;
}
}
}

3
tests.sh
View File

@ -44,9 +44,6 @@ build_cpp() {
# appears to be missing it: https://github.com/travis-ci/travis-ci/issues/6996
if [[ $(type cmake 2>/dev/null) ]]; then
# Verify benchmarking code can build successfully.
git submodule init
git submodule update
cd third_party/benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make && cd ../..
cd benchmarks && make cpp-benchmark && cd ..
else
echo ""