Merge pull request #7360 from jtattermusch/refactor_googlebenchmark

Refactor some Protobuf C# microbenchmarks
This commit is contained in:
Jan Tattermusch 2020-04-13 17:34:22 +02:00 committed by GitHub
commit 50e03cdde3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 198 additions and 126 deletions

View File

@ -85,15 +85,15 @@ csharp_EXTRA_DIST= \
csharp/src/AddressBook/ListPeople.cs \
csharp/src/AddressBook/Program.cs \
csharp/src/AddressBook/SampleUsage.cs \
csharp/src/Google.Protobuf.Benchmarks/BenchmarkDatasetConfig.cs \
csharp/src/Google.Protobuf.Benchmarks/BenchmarkMessage1Proto3.cs \
csharp/src/Google.Protobuf.Benchmarks/Benchmarks.cs \
csharp/src/Google.Protobuf.Benchmarks/Google.Protobuf.Benchmarks.csproj \
csharp/src/Google.Protobuf.Benchmarks/GoogleMessageBenchmark.cs \
csharp/src/Google.Protobuf.Benchmarks/ParseRawPrimitivesBenchmark.cs \
csharp/src/Google.Protobuf.Benchmarks/ParseMessagesBenchmark.cs \
csharp/src/Google.Protobuf.Benchmarks/Program.cs \
csharp/src/Google.Protobuf.Benchmarks/SerializationBenchmark.cs \
csharp/src/Google.Protobuf.Benchmarks/SerializationConfig.cs \
csharp/src/Google.Protobuf.Benchmarks/wrapper_benchmark_messages.proto \
csharp/src/Google.Protobuf.Benchmarks/WrapperBenchmark.cs \
csharp/src/Google.Protobuf.Benchmarks/WrapperBenchmarkMessages.cs \
csharp/src/Google.Protobuf.Conformance/Conformance.cs \
csharp/src/Google.Protobuf.Conformance/Google.Protobuf.Conformance.csproj \

View File

@ -43,20 +43,20 @@ namespace Google.Protobuf.Benchmarks
/// <summary>
/// The configuration for a single serialization test, loaded from a dataset.
/// </summary>
public class SerializationConfig
public class BenchmarkDatasetConfig
{
private static readonly Dictionary<string, MessageParser> parsersByMessageName =
typeof(SerializationBenchmark).Assembly.GetTypes()
typeof(GoogleMessageBenchmark).Assembly.GetTypes()
.Where(t => typeof(IMessage).IsAssignableFrom(t))
.ToDictionary(
t => ((MessageDescriptor) t.GetProperty("Descriptor", BindingFlags.Static | BindingFlags.Public).GetValue(null)).FullName,
t => ((MessageParser) t.GetProperty("Parser", BindingFlags.Static | BindingFlags.Public).GetValue(null)));
public MessageParser Parser { get; }
public IEnumerable<ByteString> Payloads { get; }
public List<byte[]> Payloads { get; }
public string Name { get; }
public SerializationConfig(string resource)
public BenchmarkDatasetConfig(string resource, string shortName = null)
{
var data = LoadData(resource);
var dataset = BenchmarkDataset.Parser.ParseFrom(data);
@ -66,13 +66,13 @@ namespace Google.Protobuf.Benchmarks
throw new ArgumentException($"No parser for message {dataset.MessageName} in this assembly");
}
Parser = parser;
Payloads = dataset.Payload;
Name = dataset.Name;
Payloads = new List<byte[]>(dataset.Payload.Select(p => p.ToByteArray()));
Name = shortName ?? dataset.Name;
}
private static byte[] LoadData(string resource)
{
using (var stream = typeof(SerializationBenchmark).Assembly.GetManifestResourceStream($"Google.Protobuf.Benchmarks.{resource}"))
using (var stream = typeof(GoogleMessageBenchmark).Assembly.GetManifestResourceStream($"Google.Protobuf.Benchmarks.{resource}"))
{
if (stream == null)
{

View File

@ -38,23 +38,27 @@ using System.Linq;
namespace Google.Protobuf.Benchmarks
{
/// <summary>
/// Benchmark for serializing (to a MemoryStream) and deserializing (from a ByteString).
/// Benchmark for serializing and deserializing of standard datasets that are also
/// measured by benchmarks in other languages.
/// Over time we may wish to test the various different approaches to serialization and deserialization separately.
/// See https://github.com/protocolbuffers/protobuf/blob/master/benchmarks/README.md
/// See https://github.com/protocolbuffers/protobuf/blob/master/docs/performance.md
/// </summary>
[MemoryDiagnoser]
public class SerializationBenchmark
public class GoogleMessageBenchmark
{
/// <summary>
/// All the configurations to be tested. Add more datasets to the array as they're available.
/// All the datasets to be tested. Add more datasets to the array as they're available.
/// (When C# supports proto2, this will increase significantly.)
/// </summary>
public static SerializationConfig[] Configurations => new[]
public static BenchmarkDatasetConfig[] DatasetConfigurations => new[]
{
new SerializationConfig("dataset.google_message1_proto3.pb")
// short name is specified to make results table more readable
new BenchmarkDatasetConfig("dataset.google_message1_proto3.pb", "goog_msg1_proto3")
};
[ParamsSource(nameof(Configurations))]
public SerializationConfig Configuration { get; set; }
[ParamsSource(nameof(DatasetConfigurations))]
public BenchmarkDatasetConfig Dataset { get; set; }
private MessageParser parser;
/// <summary>
@ -67,8 +71,8 @@ namespace Google.Protobuf.Benchmarks
[GlobalSetup]
public void GlobalSetup()
{
parser = Configuration.Parser;
subTests = Configuration.Payloads.Select(p => new SubTest(p, parser.ParseFrom(p))).ToList();
parser = Dataset.Parser;
subTests = Dataset.Payloads.Select(p => new SubTest(p, parser.ParseFrom(p))).ToList();
}
[Benchmark]
@ -78,7 +82,7 @@ namespace Google.Protobuf.Benchmarks
public void ToByteArray() => subTests.ForEach(item => item.ToByteArray());
[Benchmark]
public void ParseFromByteString() => subTests.ForEach(item => item.ParseFromByteString(parser));
public void ParseFromByteArray() => subTests.ForEach(item => item.ParseFromByteArray(parser));
[Benchmark]
public void ParseFromStream() => subTests.ForEach(item => item.ParseFromStream(parser));
@ -87,13 +91,13 @@ namespace Google.Protobuf.Benchmarks
{
private readonly Stream destinationStream;
private readonly Stream sourceStream;
private readonly ByteString data;
private readonly byte[] data;
private readonly IMessage message;
public SubTest(ByteString data, IMessage message)
public SubTest(byte[] data, IMessage message)
{
destinationStream = new MemoryStream(data.Length);
sourceStream = new MemoryStream(data.ToByteArray());
sourceStream = new MemoryStream(data);
this.data = data;
this.message = message;
}
@ -108,7 +112,7 @@ namespace Google.Protobuf.Benchmarks
public void ToByteArray() => message.ToByteArray();
public void ParseFromByteString(MessageParser parser) => parser.ParseFrom(data);
public void ParseFromByteArray(MessageParser parser) => parser.ParseFrom(data);
public void ParseFromStream(MessageParser parser)
{

View File

@ -0,0 +1,170 @@
#region Copyright notice and license
// Protocol Buffers - Google's data interchange format
// Copyright 2019 Google Inc. All rights reserved.
// https://github.com/protocolbuffers/protobuf
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#endregion
using BenchmarkDotNet.Attributes;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Buffers;
using Google.Protobuf.WellKnownTypes;
namespace Google.Protobuf.Benchmarks
{
/// <summary>
/// Benchmark that tests parsing performance for various messages.
/// </summary>
[MemoryDiagnoser]
public class ParseMessagesBenchmark
{
const int MaxMessages = 100;
SubTest manyWrapperFieldsTest = new SubTest(CreateManyWrapperFieldsMessage(), ManyWrapperFieldsMessage.Parser, () => new ManyWrapperFieldsMessage(), MaxMessages);
SubTest manyPrimitiveFieldsTest = new SubTest(CreateManyPrimitiveFieldsMessage(), ManyPrimitiveFieldsMessage.Parser, () => new ManyPrimitiveFieldsMessage(), MaxMessages);
SubTest emptyMessageTest = new SubTest(new Empty(), Empty.Parser, () => new Empty(), MaxMessages);
public IEnumerable<int> MessageCountValues => new[] { 10, 100 };
[GlobalSetup]
public void GlobalSetup()
{
}
[Benchmark]
public IMessage ManyWrapperFieldsMessage_ParseFromByteArray()
{
return manyWrapperFieldsTest.ParseFromByteArray();
}
[Benchmark]
public IMessage ManyPrimitiveFieldsMessage_ParseFromByteArray()
{
return manyPrimitiveFieldsTest.ParseFromByteArray();
}
[Benchmark]
public IMessage EmptyMessage_ParseFromByteArray()
{
return emptyMessageTest.ParseFromByteArray();
}
[Benchmark]
[ArgumentsSource(nameof(MessageCountValues))]
public void ManyWrapperFieldsMessage_ParseDelimitedMessagesFromByteArray(int messageCount)
{
manyWrapperFieldsTest.ParseDelimitedMessagesFromByteArray(messageCount);
}
[Benchmark]
[ArgumentsSource(nameof(MessageCountValues))]
public void ManyPrimitiveFieldsMessage_ParseDelimitedMessagesFromByteArray(int messageCount)
{
manyPrimitiveFieldsTest.ParseDelimitedMessagesFromByteArray(messageCount);
}
private static ManyWrapperFieldsMessage CreateManyWrapperFieldsMessage()
{
// Example data match data of an internal benchmarks
return new ManyWrapperFieldsMessage()
{
Int64Field19 = 123,
Int64Field37 = 1000032,
Int64Field26 = 3453524500,
DoubleField79 = 1.2,
DoubleField25 = 234,
DoubleField9 = 123.3,
DoubleField28 = 23,
DoubleField7 = 234,
DoubleField50 = 2.45
};
}
private static ManyPrimitiveFieldsMessage CreateManyPrimitiveFieldsMessage()
{
// Example data match data of an internal benchmarks
return new ManyPrimitiveFieldsMessage()
{
Int64Field19 = 123,
Int64Field37 = 1000032,
Int64Field26 = 3453524500,
DoubleField79 = 1.2,
DoubleField25 = 234,
DoubleField9 = 123.3,
DoubleField28 = 23,
DoubleField7 = 234,
DoubleField50 = 2.45
};
}
private class SubTest
{
private readonly IMessage message;
private readonly MessageParser parser;
private readonly Func<IMessage> factory;
private readonly byte[] data;
private readonly byte[] multipleMessagesData;
public SubTest(IMessage message, MessageParser parser, Func<IMessage> factory, int maxMessageCount)
{
this.message = message;
this.parser = parser;
this.factory = factory;
this.data = message.ToByteArray();
this.multipleMessagesData = CreateBufferWithMultipleMessages(message, maxMessageCount);
}
public IMessage ParseFromByteArray() => parser.ParseFrom(data);
public void ParseDelimitedMessagesFromByteArray(int messageCount)
{
var input = new CodedInputStream(multipleMessagesData);
for (int i = 0; i < messageCount; i++)
{
var msg = factory();
input.ReadMessage(msg);
}
}
private static byte[] CreateBufferWithMultipleMessages(IMessage msg, int msgCount)
{
var ms = new MemoryStream();
var cos = new CodedOutputStream(ms);
for (int i = 0; i < msgCount; i++)
{
cos.WriteMessage(msg);
}
cos.Flush();
return ms.ToArray();
}
}
}
}

View File

@ -1,102 +0,0 @@
#region Copyright notice and license
// Protocol Buffers - Google's data interchange format
// Copyright 2019 Google Inc. All rights reserved.
// https://github.com/protocolbuffers/protobuf
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#endregion
using BenchmarkDotNet.Attributes;
using System.Collections.Generic;
using System.IO;
using System.Linq;
namespace Google.Protobuf.Benchmarks
{
/// <summary>
/// Benchmark that tests serialization/deserialization of wrapper fields.
/// </summary>
[MemoryDiagnoser]
public class WrapperBenchmark
{
byte[] manyWrapperFieldsData;
byte[] manyPrimitiveFieldsData;
[GlobalSetup]
public void GlobalSetup()
{
manyWrapperFieldsData = CreateManyWrapperFieldsMessage().ToByteArray();
manyPrimitiveFieldsData = CreateManyPrimitiveFieldsMessage().ToByteArray();
}
[Benchmark]
public ManyWrapperFieldsMessage ParseWrapperFields()
{
return ManyWrapperFieldsMessage.Parser.ParseFrom(manyWrapperFieldsData);
}
[Benchmark]
public ManyPrimitiveFieldsMessage ParsePrimitiveFields()
{
return ManyPrimitiveFieldsMessage.Parser.ParseFrom(manyPrimitiveFieldsData);
}
private static ManyWrapperFieldsMessage CreateManyWrapperFieldsMessage()
{
// Example data match data of an internal benchmarks
return new ManyWrapperFieldsMessage()
{
Int64Field19 = 123,
Int64Field37 = 1000032,
Int64Field26 = 3453524500,
DoubleField79 = 1.2,
DoubleField25 = 234,
DoubleField9 = 123.3,
DoubleField28 = 23,
DoubleField7 = 234,
DoubleField50 = 2.45
};
}
private static ManyPrimitiveFieldsMessage CreateManyPrimitiveFieldsMessage()
{
// Example data match data of an internal benchmarks
return new ManyPrimitiveFieldsMessage()
{
Int64Field19 = 123,
Int64Field37 = 1000032,
Int64Field26 = 3453524500,
DoubleField79 = 1.2,
DoubleField25 = 234,
DoubleField9 = 123.3,
DoubleField28 = 23,
DoubleField7 = 234,
DoubleField50 = 2.45
};
}
}
}