Merge pull request #7360 from jtattermusch/refactor_googlebenchmark

Refactor some Protobuf C# microbenchmarks

Makefile.am

@@ -85,15 +85,15 @@ csharp_EXTRA_DIST=                                                           \
   csharp/src/AddressBook/ListPeople.cs                                       \
   csharp/src/AddressBook/Program.cs                                          \
   csharp/src/AddressBook/SampleUsage.cs                                      \
+  csharp/src/Google.Protobuf.Benchmarks/BenchmarkDatasetConfig.cs            \
   csharp/src/Google.Protobuf.Benchmarks/BenchmarkMessage1Proto3.cs           \
   csharp/src/Google.Protobuf.Benchmarks/Benchmarks.cs                        \
   csharp/src/Google.Protobuf.Benchmarks/Google.Protobuf.Benchmarks.csproj    \
+  csharp/src/Google.Protobuf.Benchmarks/GoogleMessageBenchmark.cs            \
   csharp/src/Google.Protobuf.Benchmarks/ParseRawPrimitivesBenchmark.cs       \
+  csharp/src/Google.Protobuf.Benchmarks/ParseMessagesBenchmark.cs            \
   csharp/src/Google.Protobuf.Benchmarks/Program.cs                           \
-  csharp/src/Google.Protobuf.Benchmarks/SerializationBenchmark.cs            \
-  csharp/src/Google.Protobuf.Benchmarks/SerializationConfig.cs               \
   csharp/src/Google.Protobuf.Benchmarks/wrapper_benchmark_messages.proto     \
-  csharp/src/Google.Protobuf.Benchmarks/WrapperBenchmark.cs                  \
   csharp/src/Google.Protobuf.Benchmarks/WrapperBenchmarkMessages.cs          \
   csharp/src/Google.Protobuf.Conformance/Conformance.cs                      \
   csharp/src/Google.Protobuf.Conformance/Google.Protobuf.Conformance.csproj  \

csharp/src/Google.Protobuf.Benchmarks/BenchmarkDatasetConfig.cs

@@ -43,20 +43,20 @@ namespace Google.Protobuf.Benchmarks
     /// <summary>
     /// The configuration for a single serialization test, loaded from a dataset.
     /// </summary>
-    public class SerializationConfig
+    public class BenchmarkDatasetConfig
     {
         private static readonly Dictionary<string, MessageParser> parsersByMessageName = 
-            typeof(SerializationBenchmark).Assembly.GetTypes()
+            typeof(GoogleMessageBenchmark).Assembly.GetTypes()
                 .Where(t => typeof(IMessage).IsAssignableFrom(t))
                 .ToDictionary(
                     t => ((MessageDescriptor) t.GetProperty("Descriptor", BindingFlags.Static | BindingFlags.Public).GetValue(null)).FullName,
                     t => ((MessageParser) t.GetProperty("Parser", BindingFlags.Static | BindingFlags.Public).GetValue(null)));
 
         public MessageParser Parser { get; }
-        public IEnumerable<ByteString> Payloads { get; }
+        public List<byte[]> Payloads { get; }
         public string Name { get; }
 
-        public SerializationConfig(string resource)
+        public BenchmarkDatasetConfig(string resource, string shortName = null)
         {
             var data = LoadData(resource);
             var dataset = BenchmarkDataset.Parser.ParseFrom(data);
@@ -66,13 +66,13 @@ namespace Google.Protobuf.Benchmarks
                 throw new ArgumentException($"No parser for message {dataset.MessageName} in this assembly");
             }
             Parser = parser;
-            Payloads = dataset.Payload;
+            Payloads = new List<byte[]>(dataset.Payload.Select(p => p.ToByteArray()));
-            Name = dataset.Name;
+            Name = shortName ?? dataset.Name;
         }
 
         private static byte[] LoadData(string resource)
         {
-            using (var stream = typeof(SerializationBenchmark).Assembly.GetManifestResourceStream($"Google.Protobuf.Benchmarks.{resource}"))
+            using (var stream = typeof(GoogleMessageBenchmark).Assembly.GetManifestResourceStream($"Google.Protobuf.Benchmarks.{resource}"))
             {
                 if (stream == null)
                 {

csharp/src/Google.Protobuf.Benchmarks/GoogleMessageBenchmark.cs

@@ -38,23 +38,27 @@ using System.Linq;
 namespace Google.Protobuf.Benchmarks
 {
     /// <summary>
-    /// Benchmark for serializing (to a MemoryStream) and deserializing (from a ByteString).
+    /// Benchmark for serializing and deserializing of standard datasets that are also
+    /// measured by benchmarks in other languages.
     /// Over time we may wish to test the various different approaches to serialization and deserialization separately.
+    /// See https://github.com/protocolbuffers/protobuf/blob/master/benchmarks/README.md
+    /// See https://github.com/protocolbuffers/protobuf/blob/master/docs/performance.md
     /// </summary>
     [MemoryDiagnoser]
-    public class SerializationBenchmark
+    public class GoogleMessageBenchmark
     {
         /// <summary>
-        /// All the configurations to be tested. Add more datasets to the array as they're available.
+        /// All the datasets to be tested. Add more datasets to the array as they're available.
         /// (When C# supports proto2, this will increase significantly.)
         /// </summary>
-        public static SerializationConfig[] Configurations => new[]
+        public static BenchmarkDatasetConfig[] DatasetConfigurations => new[]
         {
-            new SerializationConfig("dataset.google_message1_proto3.pb")
+            // short name is specified to make results table more readable
+            new BenchmarkDatasetConfig("dataset.google_message1_proto3.pb", "goog_msg1_proto3")
         };
 
-        [ParamsSource(nameof(Configurations))]
+        [ParamsSource(nameof(DatasetConfigurations))]
-        public SerializationConfig Configuration { get; set; }
+        public BenchmarkDatasetConfig Dataset { get; set; }
 
         private MessageParser parser;
         /// <summary>
@@ -67,8 +71,8 @@ namespace Google.Protobuf.Benchmarks
         [GlobalSetup]
         public void GlobalSetup()
         {
-            parser = Configuration.Parser;
+            parser = Dataset.Parser;
-            subTests = Configuration.Payloads.Select(p => new SubTest(p, parser.ParseFrom(p))).ToList();
+            subTests = Dataset.Payloads.Select(p => new SubTest(p, parser.ParseFrom(p))).ToList();
         }
 
         [Benchmark]
@@ -78,7 +82,7 @@ namespace Google.Protobuf.Benchmarks
         public void ToByteArray() => subTests.ForEach(item => item.ToByteArray());
 
         [Benchmark]
-        public void ParseFromByteString() => subTests.ForEach(item => item.ParseFromByteString(parser));
+        public void ParseFromByteArray() => subTests.ForEach(item => item.ParseFromByteArray(parser));
 
         [Benchmark]
         public void ParseFromStream() => subTests.ForEach(item => item.ParseFromStream(parser));
@@ -87,13 +91,13 @@ namespace Google.Protobuf.Benchmarks
         {
             private readonly Stream destinationStream;
             private readonly Stream sourceStream;
-            private readonly ByteString data;
+            private readonly byte[] data;
             private readonly IMessage message;
 
-            public SubTest(ByteString data, IMessage message)
+            public SubTest(byte[] data, IMessage message)
             {
                 destinationStream = new MemoryStream(data.Length);
-                sourceStream = new MemoryStream(data.ToByteArray());
+                sourceStream = new MemoryStream(data);
                 this.data = data;
                 this.message = message;
             }
@@ -108,7 +112,7 @@ namespace Google.Protobuf.Benchmarks
 
             public void ToByteArray() => message.ToByteArray();
 
-            public void ParseFromByteString(MessageParser parser) => parser.ParseFrom(data);
+            public void ParseFromByteArray(MessageParser parser) => parser.ParseFrom(data);
 
             public void ParseFromStream(MessageParser parser)
             {

csharp/src/Google.Protobuf.Benchmarks/ParseMessagesBenchmark.cs

@@ -0,0 +1,170 @@
+#region Copyright notice and license
+// Protocol Buffers - Google's data interchange format
+// Copyright 2019 Google Inc.  All rights reserved.
+// https://github.com/protocolbuffers/protobuf
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#endregion
+
+using BenchmarkDotNet.Attributes;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Buffers;
+using Google.Protobuf.WellKnownTypes;
+
+namespace Google.Protobuf.Benchmarks
+{
+    /// <summary>
+    /// Benchmark that tests parsing performance for various messages.
+    /// </summary>
+    [MemoryDiagnoser]
+    public class ParseMessagesBenchmark
+    {
+        const int MaxMessages = 100;
+
+        SubTest manyWrapperFieldsTest = new SubTest(CreateManyWrapperFieldsMessage(), ManyWrapperFieldsMessage.Parser, () => new ManyWrapperFieldsMessage(), MaxMessages);
+        SubTest manyPrimitiveFieldsTest = new SubTest(CreateManyPrimitiveFieldsMessage(), ManyPrimitiveFieldsMessage.Parser, () => new ManyPrimitiveFieldsMessage(), MaxMessages);
+        SubTest emptyMessageTest = new SubTest(new Empty(), Empty.Parser, () => new Empty(), MaxMessages);
+
+        public IEnumerable<int> MessageCountValues => new[] { 10, 100 };
+
+        [GlobalSetup]
+        public void GlobalSetup()
+        {
+        }
+
+        [Benchmark]
+        public IMessage ManyWrapperFieldsMessage_ParseFromByteArray()
+        {
+            return manyWrapperFieldsTest.ParseFromByteArray();
+        }
+
+        [Benchmark]
+        public IMessage ManyPrimitiveFieldsMessage_ParseFromByteArray()
+        {
+            return manyPrimitiveFieldsTest.ParseFromByteArray();
+        }
+
+        [Benchmark]
+        public IMessage EmptyMessage_ParseFromByteArray()
+        {
+            return emptyMessageTest.ParseFromByteArray();
+        }
+
+        [Benchmark]
+        [ArgumentsSource(nameof(MessageCountValues))]
+        public void ManyWrapperFieldsMessage_ParseDelimitedMessagesFromByteArray(int messageCount)
+        {
+            manyWrapperFieldsTest.ParseDelimitedMessagesFromByteArray(messageCount);
+        }
+
+        [Benchmark]
+        [ArgumentsSource(nameof(MessageCountValues))]
+        public void ManyPrimitiveFieldsMessage_ParseDelimitedMessagesFromByteArray(int messageCount)
+        {
+            manyPrimitiveFieldsTest.ParseDelimitedMessagesFromByteArray(messageCount);
+        }
+
+        private static ManyWrapperFieldsMessage CreateManyWrapperFieldsMessage()
+        {
+            // Example data match data of an internal benchmarks
+            return new ManyWrapperFieldsMessage()
+            {
+                Int64Field19 = 123,
+                Int64Field37 = 1000032,
+                Int64Field26 = 3453524500,
+                DoubleField79 = 1.2,
+                DoubleField25 = 234,
+                DoubleField9 = 123.3,
+                DoubleField28 = 23,
+                DoubleField7 = 234,
+                DoubleField50 = 2.45
+            };
+        }
+
+        private static ManyPrimitiveFieldsMessage CreateManyPrimitiveFieldsMessage()
+        {
+            // Example data match data of an internal benchmarks
+            return new ManyPrimitiveFieldsMessage()
+            {
+                Int64Field19 = 123,
+                Int64Field37 = 1000032,
+                Int64Field26 = 3453524500,
+                DoubleField79 = 1.2,
+                DoubleField25 = 234,
+                DoubleField9 = 123.3,
+                DoubleField28 = 23,
+                DoubleField7 = 234,
+                DoubleField50 = 2.45
+            };
+        }
+
+        private class SubTest
+        {
+            private readonly IMessage message;
+            private readonly MessageParser parser;
+            private readonly Func<IMessage> factory;
+            private readonly byte[] data;
+            private readonly byte[] multipleMessagesData;
+
+            public SubTest(IMessage message, MessageParser parser, Func<IMessage> factory, int maxMessageCount)
+            {
+                this.message = message;
+                this.parser = parser;
+                this.factory = factory;
+                this.data = message.ToByteArray();
+                this.multipleMessagesData = CreateBufferWithMultipleMessages(message, maxMessageCount);
+            }
+
+            public IMessage ParseFromByteArray() => parser.ParseFrom(data);
+
+            public void ParseDelimitedMessagesFromByteArray(int messageCount)
+            {
+                var input = new CodedInputStream(multipleMessagesData);
+                for (int i = 0; i < messageCount; i++)
+                {
+                    var msg = factory();
+                    input.ReadMessage(msg);
+                }
+            }
+
+            private static byte[] CreateBufferWithMultipleMessages(IMessage msg, int msgCount)
+            {
+                var ms = new MemoryStream();
+                var cos = new CodedOutputStream(ms);
+                for (int i = 0; i < msgCount; i++)
+                {
+                    cos.WriteMessage(msg);
+                }
+                cos.Flush();
+                return ms.ToArray();
+            }
+        }
+    }
+}

csharp/src/Google.Protobuf.Benchmarks/WrapperBenchmark.cs

@@ -1,102 +0,0 @@
-#region Copyright notice and license
-// Protocol Buffers - Google's data interchange format
-// Copyright 2019 Google Inc.  All rights reserved.
-// https://github.com/protocolbuffers/protobuf
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-//     * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-//     * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-//     * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#endregion
-
-using BenchmarkDotNet.Attributes;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-
-namespace Google.Protobuf.Benchmarks
-{
-    /// <summary>
-    /// Benchmark that tests serialization/deserialization of wrapper fields.
-    /// </summary>
-    [MemoryDiagnoser]
-    public class WrapperBenchmark
-    {
-        byte[] manyWrapperFieldsData;
-        byte[] manyPrimitiveFieldsData;
-
-        [GlobalSetup]
-        public void GlobalSetup()
-        {
-            manyWrapperFieldsData = CreateManyWrapperFieldsMessage().ToByteArray();
-            manyPrimitiveFieldsData = CreateManyPrimitiveFieldsMessage().ToByteArray();
-        }
-
-        [Benchmark]
-        public ManyWrapperFieldsMessage ParseWrapperFields()
-        {
-            return ManyWrapperFieldsMessage.Parser.ParseFrom(manyWrapperFieldsData);
-        }
-
-        [Benchmark]
-        public ManyPrimitiveFieldsMessage ParsePrimitiveFields()
-        {
-            return ManyPrimitiveFieldsMessage.Parser.ParseFrom(manyPrimitiveFieldsData);
-        }
-
-        private static ManyWrapperFieldsMessage CreateManyWrapperFieldsMessage()
-        {
-            // Example data match data of an internal benchmarks
-            return new ManyWrapperFieldsMessage()
-            {
-                Int64Field19 = 123,
-                Int64Field37 = 1000032,
-                Int64Field26 = 3453524500,
-                DoubleField79 = 1.2,
-                DoubleField25 = 234,
-                DoubleField9 = 123.3,
-                DoubleField28 = 23,
-                DoubleField7 = 234,
-                DoubleField50 = 2.45
-            };
-        }
-
-        private static ManyPrimitiveFieldsMessage CreateManyPrimitiveFieldsMessage()
-        {
-            // Example data match data of an internal benchmarks
-            return new ManyPrimitiveFieldsMessage()
-            {
-                Int64Field19 = 123,
-                Int64Field37 = 1000032,
-                Int64Field26 = 3453524500,
-                DoubleField79 = 1.2,
-                DoubleField25 = 234,
-                DoubleField9 = 123.3,
-                DoubleField28 = 23,
-                DoubleField7 = 234,
-                DoubleField50 = 2.45
-            };
-        }
-    }
-}