AuroraMiddleware/protobuf
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

Sync from Piper @455586341

Browse Source 
PROTOBUF_SYNC_PIPER
This commit is contained in:
Jorg Brown 2022-06-17 04:13:20 -07:00
parent 5a9bf6c7de
commit a3e1de0113
61 changed files with 1201 additions and 1189 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
BUILD.bazel
View File

@ -22,7 +22,7 @@ exports_files(["LICENSE"])
#
# java_proto_library(
# name = "any_java_proto",
# deps = ["@com_google_protobuf//:any_proto],
# deps = ["@com_google_protobuf//:any_proto"],
# )
################################################################################

6
CHANGES.txt
View File

@ -1,3 +1,9 @@
Unreleased version
* Handle reflection for message splitting.
* make metadata fields lazy.
* Extend visibility of plugin library to upb
2022-05-27 version 21.1 (C++/Java/Python/PHP/Objective-C/C#/Ruby)
C++

279
conformance/conformance_cpp.cc
View File

@ -32,165 +32,176 @@
#include <stdarg.h>
#include <unistd.h>
#include <memory>
#include <string>
#include <utility>
#include <google/protobuf/stubs/logging.h>
#include <google/protobuf/stubs/common.h>
#include <google/protobuf/message.h>
#include <google/protobuf/text_format.h>
#include <google/protobuf/util/json_util.h>
#include <google/protobuf/util/type_resolver_util.h>
#include <google/protobuf/stubs/status.h>
#include <google/protobuf/stubs/statusor.h>
#include "conformance.pb.h"
#include "conformance.pb.h"
#include <google/protobuf/test_messages_proto2.pb.h>
#include <google/protobuf/test_messages_proto3.pb.h>
#include <google/protobuf/test_messages_proto3.pb.h>
#include <google/protobuf/util/type_resolver.h>
#include <google/protobuf/stubs/status.h>
#include <google/protobuf/stubs/status_macros.h>
using conformance::ConformanceRequest;
using conformance::ConformanceResponse;
using google::protobuf::Descriptor;
using google::protobuf::DescriptorPool;
using google::protobuf::Message;
using google::protobuf::MessageFactory;
using google::protobuf::TextFormat;
using google::protobuf::util::BinaryToJsonString;
using google::protobuf::util::JsonParseOptions;
using google::protobuf::util::JsonToBinaryString;
using google::protobuf::util::NewTypeResolverForDescriptorPool;
using google::protobuf::util::TypeResolver;
using protobuf_test_messages::proto3::TestAllTypesProto3;
using protobuf_test_messages::proto2::TestAllTypesProto2;
using std::string;
static const char kTypeUrlPrefix[] = "type.googleapis.com";
const char* kFailures[] = {
};
static string GetTypeUrl(const Descriptor* message) {
return string(kTypeUrlPrefix) + "/" + message->full_name();
}
int test_count = 0;
bool verbose = false;
TypeResolver* type_resolver;
string* type_url;
// Must be included last.
#include <google/protobuf/port_def.inc>
namespace google {
namespace protobuf {
namespace {
using ::conformance::ConformanceRequest;
using ::conformance::ConformanceResponse;
using ::google::protobuf::util::BinaryToJsonString;
using ::google::protobuf::util::JsonParseOptions;
using ::google::protobuf::util::JsonToBinaryString;
using ::google::protobuf::util::NewTypeResolverForDescriptorPool;
using ::google::protobuf::util::TypeResolver;
using ::protobuf_test_messages::proto2::TestAllTypesProto2;
using ::protobuf_test_messages::proto3::TestAllTypesProto3;
using util::Status;
bool CheckedRead(int fd, void *buf, size_t len) {
size_t ofs = 0;
util::Status ReadFd(int fd, char* buf, size_t len) {
while (len > 0) {
ssize_t bytes_read = read(fd, (char*)buf + ofs, len);
ssize_t bytes_read = read(fd, buf, len);
if (bytes_read == 0) return false;
if (bytes_read == 0) {
return util::DataLossError("unexpected EOF");
}
if (bytes_read < 0) {
GOOGLE_LOG(FATAL) << "Error reading from test runner: " << strerror(errno);
return util::ErrnoToStatus(errno, "error reading from test runner");
}
len -= bytes_read;
ofs += bytes_read;
buf += bytes_read;
}
return true;
return util::OkStatus();
}
void CheckedWrite(int fd, const void *buf, size_t len) {
util::Status WriteFd(int fd, const void* buf, size_t len) {
if (write(fd, buf, len) != len) {
GOOGLE_LOG(FATAL) << "Error writing to test runner: " << strerror(errno);
return util::ErrnoToStatus(errno, "error reading to test runner");
}
return util::OkStatus();
}
void DoTest(const ConformanceRequest& request, ConformanceResponse* response) {
Message *test_message;
google::protobuf::LinkMessageReflection<TestAllTypesProto2>();
google::protobuf::LinkMessageReflection<TestAllTypesProto3>();
const Descriptor *descriptor = DescriptorPool::generated_pool()->FindMessageTypeByName(
request.message_type());
if (!descriptor) {
GOOGLE_LOG(FATAL) << "No such message type: " << request.message_type();
class Harness {
public:
Harness() {
google::protobuf::LinkMessageReflection<TestAllTypesProto2>();
google::protobuf::LinkMessageReflection<TestAllTypesProto3>();
resolver_.reset(NewTypeResolverForDescriptorPool(
"type.googleapis.com", DescriptorPool::generated_pool()));
type_url_ = StrCat("type.googleapis.com/",
TestAllTypesProto3::GetDescriptor()->full_name());
}
test_message = MessageFactory::generated_factory()->GetPrototype(descriptor)->New();
util::StatusOr<ConformanceResponse> RunTest(
const ConformanceRequest& request);
// Returns Ok(true) if we're done processing requests.
util::StatusOr<bool> ServeConformanceRequest();
private:
bool verbose_ = false;
std::unique_ptr<TypeResolver> resolver_;
std::string type_url_;
};
util::StatusOr<ConformanceResponse> Harness::RunTest(
const ConformanceRequest& request) {
const Descriptor* descriptor =
DescriptorPool::generated_pool()->FindMessageTypeByName(
request.message_type());
if (descriptor == nullptr) {
return util::NotFoundError(
StrCat("No such message type: ", request.message_type()));
}
std::unique_ptr<Message> test_message(
MessageFactory::generated_factory()->GetPrototype(descriptor)->New());
ConformanceResponse response;
switch (request.payload_case()) {
case ConformanceRequest::kProtobufPayload: {
if (!test_message->ParseFromString(request.protobuf_payload())) {
// Getting parse details would involve something like:
// http://stackoverflow.com/questions/22121922/how-can-i-get-more-details-about-errors-generated-during-protobuf-parsing-c
response->set_parse_error("Parse error (no more details available).");
return;
response.set_parse_error("parse error (no more details available)");
return response;
}
break;
}
case ConformanceRequest::kJsonPayload: {
string proto_binary;
JsonParseOptions options;
options.ignore_unknown_fields =
(request.test_category() ==
conformance::JSON_IGNORE_UNKNOWN_PARSING_TEST);
conformance::JSON_IGNORE_UNKNOWN_PARSING_TEST);
std::string proto_binary;
util::Status status =
JsonToBinaryString(type_resolver, *type_url, request.json_payload(),
JsonToBinaryString(resolver_.get(), type_url_, request.json_payload(),
&proto_binary, options);
if (!status.ok()) {
response->set_parse_error(string("Parse error: ") +
std::string(status.message()));
return;
response.set_parse_error(
StrCat("parse error: ", status.message()));
return response;
}
if (!test_message->ParseFromString(proto_binary)) {
response->set_runtime_error(
"Parsing JSON generates invalid proto output.");
return;
response.set_runtime_error(
"parsing JSON generated invalid proto output");
return response;
}
break;
}
case ConformanceRequest::kTextPayload: {
if (!TextFormat::ParseFromString(request.text_payload(), test_message)) {
response->set_parse_error("Parse error");
return;
if (!TextFormat::ParseFromString(request.text_payload(),
test_message.get())) {
response.set_parse_error("parse error (no more details available)");
return response;
}
break;
}
case ConformanceRequest::PAYLOAD_NOT_SET:
GOOGLE_LOG(FATAL) << "Request didn't have payload.";
break;
return util::InvalidArgumentError("request didn't have payload");
default:
GOOGLE_LOG(FATAL) << "unknown payload type: " << request.payload_case();
break;
}
conformance::FailureSet failures;
if (descriptor == failures.GetDescriptor()) {
for (const char* s : kFailures) failures.add_failure(s);
test_message = &failures;
return util::InvalidArgumentError(
StrCat("unknown payload type", request.payload_case()));
}
switch (request.requested_output_format()) {
case conformance::UNSPECIFIED:
GOOGLE_LOG(FATAL) << "Unspecified output format";
break;
return util::InvalidArgumentError("unspecified output format");
case conformance::PROTOBUF: {
GOOGLE_CHECK(test_message->SerializeToString(
response->mutable_protobuf_payload()));
GOOGLE_CHECK(
test_message->SerializeToString(response.mutable_protobuf_payload()));
break;
}
case conformance::JSON: {
string proto_binary;
std::string proto_binary;
GOOGLE_CHECK(test_message->SerializeToString(&proto_binary));
util::Status status =
BinaryToJsonString(type_resolver, *type_url, proto_binary,
response->mutable_json_payload());
BinaryToJsonString(resolver_.get(), type_url_, proto_binary,
response.mutable_json_payload());
if (!status.ok()) {
response->set_serialize_error(
string("Failed to serialize JSON output: ") +
std::string(status.message()));
return;
response.set_serialize_error(StrCat(
"failed to serialize JSON output: ", status.message()));
}
break;
}
@ -199,70 +210,66 @@ void DoTest(const ConformanceRequest& request, ConformanceResponse* response) {
TextFormat::Printer printer;
printer.SetHideUnknownFields(!request.print_unknown_fields());
GOOGLE_CHECK(printer.PrintToString(*test_message,
response->mutable_text_payload()));
response.mutable_text_payload()));
break;
}
default:
GOOGLE_LOG(FATAL) << "Unknown output format: "
<< request.requested_output_format();
return util::InvalidArgumentError(StrCat(
"unknown output format", request.requested_output_format()));
}
return response;
}
bool DoTestIo() {
string serialized_input;
string serialized_output;
util::StatusOr<bool> Harness::ServeConformanceRequest() {
uint32_t in_len;
if (!ReadFd(STDIN_FILENO, reinterpret_cast<char*>(&in_len), sizeof(in_len))
.ok()) {
// EOF means we're done.
return true;
}
std::string serialized_input;
serialized_input.resize(in_len);
RETURN_IF_ERROR(ReadFd(STDIN_FILENO, serialized_input.data(), in_len));
ConformanceRequest request;
ConformanceResponse response;
uint32_t bytes;
GOOGLE_CHECK(request.ParseFromString(serialized_input));
if (!CheckedRead(STDIN_FILENO, &bytes, sizeof(uint32_t))) {
// EOF.
return false;
util::StatusOr<ConformanceResponse> response = RunTest(request);
RETURN_IF_ERROR(response.status());
std::string serialized_output;
response->SerializeToString(&serialized_output);
uint32_t out_len = static_cast<uint32_t>(serialized_output.size());
RETURN_IF_ERROR(WriteFd(STDOUT_FILENO, &out_len, sizeof(out_len)));
RETURN_IF_ERROR(WriteFd(STDOUT_FILENO, serialized_output.data(), out_len));
if (verbose_) {
GOOGLE_LOG(INFO) << "conformance-cpp: request=" << request.ShortDebugString()
<< ", response=" << response->ShortDebugString();
}
serialized_input.resize(bytes);
if (!CheckedRead(STDIN_FILENO, (char*)serialized_input.c_str(), bytes)) {
GOOGLE_LOG(ERROR) << "Unexpected EOF on stdin. " << strerror(errno);
}
if (!request.ParseFromString(serialized_input)) {
GOOGLE_LOG(FATAL) << "Parse of ConformanceRequest proto failed.";
return false;
}
DoTest(request, &response);
response.SerializeToString(&serialized_output);
bytes = serialized_output.size();
CheckedWrite(STDOUT_FILENO, &bytes, sizeof(uint32_t));
CheckedWrite(STDOUT_FILENO, serialized_output.c_str(), bytes);
if (verbose) {
fprintf(stderr, "conformance-cpp: request=%s, response=%s\n",
request.ShortDebugString().c_str(),
response.ShortDebugString().c_str());
}
test_count++;
return true;
return false;
}
} // namespace
} // namespace protobuf
} // namespace google
int main() {
type_resolver = NewTypeResolverForDescriptorPool(
kTypeUrlPrefix, DescriptorPool::generated_pool());
type_url = new string(GetTypeUrl(TestAllTypesProto3::descriptor()));
while (1) {
if (!google::protobuf::DoTestIo()) {
fprintf(stderr, "conformance-cpp: received EOF from test runner "
"after %d tests, exiting\n", test_count);
return 0;
google::protobuf::Harness harness;
int total_runs = 0;
while (true) {
auto is_done = harness.ServeConformanceRequest();
if (!is_done.ok()) {
GOOGLE_LOG(FATAL) << is_done.status();
}
if (*is_done) {
break;
}
total_runs++;
}
GOOGLE_LOG(INFO) << "conformance-cpp: recieved EOF from test runner after "
<< total_runs << " tests";
}

1
csharp/.gitignore vendored
View File

@ -18,6 +18,7 @@ lib/NUnit
# Untracked files
#
.vs
.cr
*.user
*.suo
*.nupkg

4
csharp/src/Google.Protobuf.Test/ByteStringTest.cs
View File

@ -41,9 +41,7 @@ using System.Buffers;
using System.Runtime.InteropServices;
using System.Threading;
using System.Runtime.CompilerServices;
#if !NET35
using System.Threading.Tasks;
#endif
namespace Google.Protobuf
{
@ -349,7 +347,6 @@ namespace Google.Protobuf
Assert.AreEqual(expected, actual, $"{expected.ToBase64()} != {actual.ToBase64()}");
}
#if !NET35
[Test]
public async Task FromStreamAsync_Seekable()
{
@ -373,7 +370,6 @@ namespace Google.Protobuf
ByteString expected = ByteString.CopyFrom(2, 3, 4);
Assert.AreEqual(expected, actual, $"{expected.ToBase64()} != {actual.ToBase64()}");
}
#endif
[Test]
public void GetHashCode_Regression()

2
csharp/src/Google.Protobuf.Test/Collections/MapFieldTest.cs
View File

@ -635,7 +635,6 @@ namespace Google.Protobuf.Collections
Assert.IsTrue(input.IsAtEnd);
}
#if !NET35
[Test]
public void IDictionaryKeys_Equals_IReadOnlyDictionaryKeys()
{
@ -649,7 +648,6 @@ namespace Google.Protobuf.Collections
var map = new MapField<string, string> { { "foo", "bar" }, { "x", "y" } };
CollectionAssert.AreEquivalent(((IDictionary<string, string>)map).Values, ((IReadOnlyDictionary<string, string>)map).Values);
}
#endif
private static KeyValuePair<TKey, TValue> NewKeyValuePair<TKey, TValue>(TKey key, TValue value)
{

67
csharp/src/Google.Protobuf.Test/Compatibility/StreamExtensionsTest.cs
View File

@ -1,67 +0,0 @@
#region Copyright notice and license
// Protocol Buffers - Google's data interchange format
// Copyright 2015 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#endregion
#if NET35
using System;
using System.IO;
using NUnit.Framework;
using Google.Protobuf.Compatibility;
namespace Google.Protobuf.Test.Compatibility
{
public class StreamExtensionsTest
{
[Test]
public void CopyToNullArgument()
{
var memoryStream = new MemoryStream();
Assert.Throws<ArgumentNullException>(() => memoryStream.CopyTo(null));
}
[Test]
public void CopyToTest()
{
byte[] bytesToStream = new byte[] { 0x31, 0x08, 0xFF, 0x00 };
Stream source = new MemoryStream(bytesToStream);
Stream destination = new MemoryStream((int)source.Length);
source.CopyTo(destination);
destination.Seek(0, SeekOrigin.Begin);
Assert.AreEqual(0x31, destination.ReadByte());
Assert.AreEqual(0x08, destination.ReadByte());
Assert.AreEqual(0xFF, destination.ReadByte());
Assert.AreEqual(0x00, destination.ReadByte());
Assert.AreEqual(-1, destination.ReadByte());
}
}
}
#endif

2
csharp/src/Google.Protobuf.Test/Compatibility/TypeExtensionsTest.cs
View File

@ -34,7 +34,6 @@ using System;
using System.Collections.Generic;
using System.Reflection;
#if !NET35
namespace Google.Protobuf.Compatibility
{
public class TypeExtensionsTest
@ -114,4 +113,3 @@ namespace Google.Protobuf.Compatibility
}
}
}
#endif

2
csharp/src/Google.Protobuf.Test/FieldCodecTest.cs
View File

@ -169,7 +169,6 @@ namespace Google.Protobuf
// WriteTagAndValue ignores default values
var stream = new MemoryStream();
CodedOutputStream codedOutput;
#if !NET35
codedOutput = new CodedOutputStream(stream);
codec.WriteTagAndValue(codedOutput, codec.DefaultValue);
codedOutput.Flush();
@ -179,7 +178,6 @@ namespace Google.Protobuf
{
Assert.AreEqual(default(T), codec.DefaultValue);
}
#endif
// The plain ValueWriter/ValueReader delegates don't.
if (codec.DefaultValue != null) // This part isn't appropriate for message types.

7
csharp/src/Google.Protobuf/ByteString.cs
View File

@ -37,13 +37,8 @@ using System.IO;
using System.Runtime.InteropServices;
using System.Security;
using System.Text;
#if !NET35
using System.Threading;
using System.Threading.Tasks;
#endif
#if NET35
using Google.Protobuf.Compatibility;
#endif
namespace Google.Protobuf
{
@ -186,7 +181,6 @@ namespace Google.Protobuf
return AttachBytes(bytes);
}
#if !NET35
/// <summary>
/// Constructs a <see cref="ByteString"/> from data in the given stream, asynchronously.
/// </summary>
@ -200,7 +194,6 @@ namespace Google.Protobuf
ProtoPreconditions.CheckNotNull(stream, nameof(stream));
return ByteStringAsync.FromStreamAsyncCore(stream, cancellationToken);
}
#endif
/// <summary>
/// Constructs a <see cref="ByteString" /> from the given array. The contents

2
csharp/src/Google.Protobuf/ByteStringAsync.cs
View File

@ -43,7 +43,6 @@ namespace Google.Protobuf
/// </summary>
internal static class ByteStringAsync
{
#if !NET35
internal static async Task<ByteString> FromStreamAsyncCore(Stream stream, CancellationToken cancellationToken)
{
int capacity = stream.CanSeek ? checked((int)(stream.Length - stream.Position)) : 0;
@ -59,6 +58,5 @@ namespace Google.Protobuf
#endif
return ByteString.AttachBytes(bytes);
}
#endif
}
}

8
csharp/src/Google.Protobuf/Collections/MapField.cs
View File

@ -68,10 +68,7 @@ namespace Google.Protobuf.Collections
/// in future versions.
/// </para>
/// </remarks>
public sealed class MapField<TKey, TValue> : IDeepCloneable<MapField<TKey, TValue>>, IDictionary<TKey, TValue>, IEquatable<MapField<TKey, TValue>>, IDictionary
#if !NET35
, IReadOnlyDictionary<TKey, TValue>
#endif
public sealed class MapField<TKey, TValue> : IDeepCloneable<MapField<TKey, TValue>>, IDictionary<TKey, TValue>, IEquatable<MapField<TKey, TValue>>, IDictionary, IReadOnlyDictionary<TKey, TValue>
{
private static readonly EqualityComparer<TValue> ValueEqualityComparer = ProtobufEqualityComparers.GetEqualityComparer<TValue>();
private static readonly EqualityComparer<TKey> KeyEqualityComparer = ProtobufEqualityComparers.GetEqualityComparer<TKey>();
@ -600,11 +597,8 @@ namespace Google.Protobuf.Collections
#endregion
#region IReadOnlyDictionary explicit interface implementation
#if !NET35
IEnumerable<TKey> IReadOnlyDictionary<TKey, TValue>.Keys => Keys;
IEnumerable<TValue> IReadOnlyDictionary<TKey, TValue>.Values => Values;
#endif
#endregion
private class DictionaryEnumerator : IDictionaryEnumerator

147
csharp/src/Google.Protobuf/Collections/ReadOnlyDictionary.cs
View File

@ -1,147 +0,0 @@
#region Copyright notice and license
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#endregion
using System;
using System.Collections;
using System.Collections.Generic;
namespace Google.Protobuf.Collections
{
/// <summary>
/// Read-only wrapper around another dictionary.
/// </summary>
internal sealed class ReadOnlyDictionary<TKey, TValue> : IDictionary<TKey, TValue>
{
private readonly IDictionary<TKey, TValue> wrapped;
public ReadOnlyDictionary(IDictionary<TKey, TValue> wrapped)
{
this.wrapped = wrapped;
}
public void Add(TKey key, TValue value)
{
throw new InvalidOperationException();
}
public bool ContainsKey(TKey key)
{
return wrapped.ContainsKey(key);
}
public ICollection<TKey> Keys
{
get { return wrapped.Keys; }
}
public bool Remove(TKey key)
{
throw new InvalidOperationException();
}
public bool TryGetValue(TKey key, out TValue value)
{
return wrapped.TryGetValue(key, out value);
}
public ICollection<TValue> Values
{
get { return wrapped.Values; }
}
public TValue this[TKey key]
{
get { return wrapped[key]; }
set { throw new InvalidOperationException(); }
}
public void Add(KeyValuePair<TKey, TValue> item)
{
throw new InvalidOperationException();
}
public void Clear()
{
throw new InvalidOperationException();
}
public bool Contains(KeyValuePair<TKey, TValue> item)
{
return wrapped.Contains(item);
}
public void CopyTo(KeyValuePair<TKey, TValue>[] array, int arrayIndex)
{
wrapped.CopyTo(array, arrayIndex);
}
public int Count
{
get { return wrapped.Count; }
}
public bool IsReadOnly
{
get { return true; }
}
public bool Remove(KeyValuePair<TKey, TValue> item)
{
throw new InvalidOperationException();
}
public IEnumerator<KeyValuePair<TKey, TValue>> GetEnumerator()
{
return wrapped.GetEnumerator();
}
IEnumerator IEnumerable.GetEnumerator()
{
return ((IEnumerable) wrapped).GetEnumerator();
}
public override bool Equals(object obj)
{
return wrapped.Equals(obj);
}
public override int GetHashCode()
{
return wrapped.GetHashCode();
}
public override string ToString()
{
return wrapped.ToString();
}
}
}

5
csharp/src/Google.Protobuf/Collections/RepeatedField.cs
View File

@ -48,10 +48,7 @@ namespace Google.Protobuf.Collections
/// supported by Protocol Buffers but nor does it guarantee that all operations will work in such cases.
/// </remarks>
/// <typeparam name="T">The element type of the repeated field.</typeparam>
public sealed class RepeatedField<T> : IList<T>, IList, IDeepCloneable<RepeatedField<T>>, IEquatable<RepeatedField<T>>
#if !NET35
, IReadOnlyList<T>
#endif
public sealed class RepeatedField<T> : IList<T>, IList, IDeepCloneable<RepeatedField<T>>, IEquatable<RepeatedField<T>>, IReadOnlyList<T>
{
private static readonly EqualityComparer<T> EqualityComparer = ProtobufEqualityComparers.GetEqualityComparer<T>();
private static readonly T[] EmptyArray = new T[0];

8
csharp/src/Google.Protobuf/Compatibility/PropertyInfoExtensions.cs
View File

@ -47,11 +47,7 @@ namespace Google.Protobuf.Compatibility
/// </summary>
internal static MethodInfo GetGetMethod(this PropertyInfo target)
{
#if NET35
var method = target.GetGetMethod();
#else
var method = target.GetMethod;
#endif
return method != null && method.IsPublic ? method : null;
}
@ -61,11 +57,7 @@ namespace Google.Protobuf.Compatibility
/// </summary>
internal static MethodInfo GetSetMethod(this PropertyInfo target)
{
#if NET35
var method = target.GetSetMethod();
#else
var method = target.SetMethod;
#endif
return method != null && method.IsPublic ? method : null;
}
}

66
csharp/src/Google.Protobuf/Compatibility/StreamExtensions.cs
View File

@ -1,66 +0,0 @@
#region Copyright notice and license
// Protocol Buffers - Google's data interchange format
// Copyright 2015 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#endregion
#if NET35
using System;
using System.IO;
namespace Google.Protobuf.Compatibility
{
/// <summary>
/// Extension methods for <see cref="Stream"/> in order to provide
/// backwards compatibility with .NET 3.5
/// </summary>
public static class StreamExtensions
{
// 81920 seems to be the default buffer size used in .NET 4.5.1
private const int BUFFER_SIZE = 81920;
/// <summary>
/// Write the contents of the current stream to the destination stream
/// </summary>
public static void CopyTo(this Stream source, Stream destination)
{
if (destination == null)
{
throw new ArgumentNullException(nameof(destination));
}
byte[] buffer = new byte[BUFFER_SIZE];
int numBytesRead;
while ((numBytesRead = source.Read(buffer, 0, buffer.Length)) > 0) {
destination.Write(buffer, 0, numBytesRead);
}
}
}
}
#endif

2
csharp/src/Google.Protobuf/Compatibility/TypeExtensions.cs
View File

@ -34,7 +34,6 @@ using System;
using System.Diagnostics.CodeAnalysis;
using System.Reflection;
#if !NET35
namespace Google.Protobuf.Compatibility
{
/// <summary>
@ -112,4 +111,3 @@ namespace Google.Protobuf.Compatibility
}
}
}
#endif

4
csharp/src/Google.Protobuf/Reflection/MessageDescriptor.cs
View File

@ -36,10 +36,6 @@ using System.Collections.ObjectModel;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Reflection;
#if NET35
// Needed for ReadOnlyDictionary, which does not exist in .NET 3.5
using Google.Protobuf.Collections;
#endif
namespace Google.Protobuf.Reflection
{

5
csharp/src/Google.Protobuf/WellKnownTypes/FieldMaskPartial.cs
View File

@ -63,12 +63,7 @@ namespace Google.Protobuf.WellKnownTypes
if (firstInvalid == null)
{
var writer = new StringWriter();
#if NET35
var query = paths.Select(JsonFormatter.ToJsonName);
JsonFormatter.WriteString(writer, string.Join(",", query.ToArray()));
#else
JsonFormatter.WriteString(writer, string.Join(",", paths.Select(JsonFormatter.ToJsonName)));
#endif
return writer.ToString();
}
else

4
docs/options.md
View File

@ -328,3 +328,7 @@ with info about your project (name and website) so we can add an entry for you.
1. Protoc-gen-xo
* Website: https://github.com/xo/ecosystem
* Extension: 1147
1. Ballerina gRPC
* Website: https://github.com/ballerina-platform/module-ballerina-grpc
* Extension: 1148

822
java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java
View File

File diff suppressed because it is too large Load Diff

19
java/core/src/test/java/com/google/protobuf/DescriptorsTest.java
View File

@ -463,15 +463,15 @@ public class DescriptorsTest {
/** Tests that parsing an unknown enum throws an exception */
@Test
public void testParseUnknownEnum() {
FieldDescriptorProto.Builder field = FieldDescriptorProto.newBuilder()
.setLabel(FieldDescriptorProto.Label.LABEL_OPTIONAL)
.setTypeName("UnknownEnum")
.setType(FieldDescriptorProto.Type.TYPE_ENUM)
.setName("bar")
.setNumber(1);
DescriptorProto.Builder messageType = DescriptorProto.newBuilder()
.setName("Foo")
.addField(field);
FieldDescriptorProto.Builder field =
FieldDescriptorProto.newBuilder()
.setLabel(FieldDescriptorProto.Label.LABEL_OPTIONAL)
.setTypeName("UnknownEnum")
.setType(FieldDescriptorProto.Type.TYPE_ENUM)
.setName("bar")
.setNumber(1);
DescriptorProto.Builder messageType =
DescriptorProto.newBuilder().setName("Foo").addField(field);
FileDescriptorProto fooProto =
FileDescriptorProto.newBuilder()
.setName("foo.proto")
@ -486,7 +486,6 @@ public class DescriptorsTest {
}
}
/**
* Tests the translate/crosslink for an example where a message field's name and type name are the
* same.

2
java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java
View File

@ -1943,7 +1943,7 @@ public class GeneratedMessageTest {
@Test
public void
extendableBuilder_extensionFieldContainingBuilder_setRepeatedFieldOverwritesElement() {
extendableBuilder_extensionFieldContainingBuilder_setRepeatedFieldOverwritesElement() {
TestAllExtensions.Builder builder = TestAllExtensions.newBuilder();
builder.addRepeatedField(REPEATED_NESTED_MESSAGE_EXTENSION, NestedMessage.getDefaultInstance());
// Calling getRepeatedFieldBuilder and ignoring the returned Builder should have no

25
objectivec/README.md
View File

@ -200,12 +200,13 @@ supported keys are:
entry can be made as "no_package:PATH=prefix", where PATH is the
path for the .proto file.
* `use_package_as_prefix` and `proto_package_prefix_exceptions_path`: The
`value` for `use_package_as_prefix` can be `yes` or `no`, and indicates
if a prefix should be derived from the proto package for all the symbols
for files that don't have the `objc_class_prefix` file option (mentioned
above). This helps ensure the symbols are more unique and means there is
less chance of ObjC class name collisions.
* `use_package_as_prefix`, `package_as_prefix_forced_prefix` and
`proto_package_prefix_exceptions_path`: The `value` for
`use_package_as_prefix` can be `yes` or `no`, and indicates if a prefix
should be derived from the proto package for all the symbols for files that
don't have the `objc_class_prefix` file option (mentioned above). This helps
ensure the symbols are more unique and means there is less chance of ObjC
class name collisions.
To help in migrating code to using this support,
`proto_package_prefix_exceptions_path` can be used to provide the path
@ -213,10 +214,16 @@ supported keys are:
if prefixed with `#`). These package won't get the derived prefix, allowing
migrations to the behavior one proto package at a time across a code base.
`package_as_prefix_forced_prefix` can be used to provide a value that will
be used before all prefixes derived from the packages to help group all of
these types with a common prefix. Thus it only makes sense to use it when
`use_package_as_prefix` is also enabled. For example, setting this to
"XYZ\_" and generating a file with the package "something" defining
"MyMessage", would have Objective-C class be `XYZ_Something_MyMessage`.
`use_package_as_prefix` currently defaults to `no` (existing behavior), but
in the future (as a breaking change), that is likely to change since it
helps prepare folks before they end up using a lot of protos and getting a
lot of collisions.
that could change in the future as it helps avoid collisions when more
protos get added to the build. Note that this would be a breaking change.
* `headers_use_forward_declarations`: The `value` for this can be `yes` or
`no`, and indicates if the generated headers use forward declarations for

35
php/src/Google/Protobuf/FieldDescriptor.php
View File

@ -39,6 +39,7 @@ class FieldDescriptor
{
use GetPublicDescriptorTrait;
/** @var \Google\Protobuf\Internal\FieldDescriptor $internal_desc */
private $internal_desc;
/**
@ -81,6 +82,32 @@ class FieldDescriptor
return $this->internal_desc->getType();
}
/**
* @return OneofDescriptor
*/
public function getContainingOneof()
{
return $this->getPublicDescriptor($this->internal_desc->getContainingOneof());
}
/**
* Gets the field's containing oneof, only if non-synthetic.
*
* @return null|OneofDescriptor
*/
public function getRealContainingOneof()
{
return $this->getPublicDescriptor($this->internal_desc->getRealContainingOneof());
}
/**
* @return boolean
*/
public function hasOptionalKeyword()
{
return $this->internal_desc->hasOptionalKeyword();
}
/**
* @return Descriptor Returns a descriptor for the field type if the field type is a message, otherwise throws \Exception
* @throws \Exception
@ -114,12 +141,4 @@ class FieldDescriptor
{
return $this->internal_desc->isMap();
}
/**
* @return boolean
*/
public function hasOptionalKeyword()
{
return $this->internal_desc->hasOptionalKeyword();
}
}

38
php/src/Google/Protobuf/Internal/FieldDescriptor.php
View File

@ -46,8 +46,11 @@ class FieldDescriptor
private $message_type;
private $enum_type;
private $packed;
private $is_map;
private $oneof_index = -1;
private $proto3_optional;
/** @var OneofDescriptor $containing_oneof */
private $containing_oneof;
public function __construct()
{
@ -169,6 +172,32 @@ class FieldDescriptor
return $this->packed;
}
public function getProto3Optional()
{
return $this->proto3_optional;
}
public function setProto3Optional($proto3_optional)
{
$this->proto3_optional = $proto3_optional;
}
public function getContainingOneof()
{
return $this->containing_oneof;
}
public function setContainingOneof($containing_oneof)
{
$this->containing_oneof = $containing_oneof;
}
public function getRealContainingOneof()
{
return !is_null($this->containing_oneof) && !$this->containing_oneof->isSynthetic()
? $this->containing_oneof : null;
}
public function isPackable()
{
return $this->isRepeated() && self::isTypePackable($this->type);
@ -214,6 +243,10 @@ class FieldDescriptor
$field_type !== GPBType::BYTES);
}
/**
* @param FieldDescriptorProto $proto
* @return FieldDescriptor
*/
public static function getFieldDescriptor($proto)
{
$type_name = null;
@ -248,8 +281,6 @@ class FieldDescriptor
$field = new FieldDescriptor();
$field->setName($proto->getName());
$json_name = $proto->hasJsonName() ? $proto->getJsonName() :
lcfirst(implode('', array_map('ucwords', explode('_', $proto->getName()))));
if ($proto->hasJsonName()) {
$json_name = $proto->getJsonName();
} else {
@ -269,6 +300,7 @@ class FieldDescriptor
$field->setLabel($proto->getLabel());
$field->setPacked($packed);
$field->setOneofIndex($oneof_index);
$field->setProto3Optional($proto->getProto3Optional());
// At this time, the message/enum type may have not been added to pool.
// So we use the type name as place holder and will replace it with the

9
php/src/Google/Protobuf/Internal/OneofDescriptor.php
View File

@ -37,6 +37,7 @@ class OneofDescriptor
use HasPublicDescriptorTrait;
private $name;
/** @var \Google\Protobuf\FieldDescriptor[] $fields */
private $fields;
public function __construct()
@ -64,13 +65,21 @@ class OneofDescriptor
return $this->fields;
}
public function isSynthetic()
{
return !is_null($this->fields) && count($this->fields) === 1
&& $this->fields[0]->getProto3Optional();
}
public static function buildFromProto($oneof_proto, $desc, $index)
{
$oneof = new OneofDescriptor();
$oneof->setName($oneof_proto->getName());
foreach ($desc->getField() as $field) {
/** @var FieldDescriptor $field */
if ($field->getOneofIndex() == $index) {
$oneof->addField($field);
$field->setContainingOneof($oneof);
}
}
return $oneof;

9
php/src/Google/Protobuf/OneofDescriptor.php
View File

@ -38,6 +38,7 @@ class OneofDescriptor
{
use GetPublicDescriptorTrait;
/** @var \Google\Protobuf\Internal\OneofDescriptor $internal_desc */
private $internal_desc;
/**
@ -62,6 +63,12 @@ class OneofDescriptor
*/
public function getField($index)
{
if (
is_null($this->internal_desc->getFields())
|| !isset($this->internal_desc->getFields()[$index])
) {
return null;
}
return $this->getPublicDescriptor($this->internal_desc->getFields()[$index]);
}
@ -75,6 +82,6 @@ class OneofDescriptor
public function isSynthetic()
{
return $this->internal_desc->isSynthetic();
return $this->internal_desc->isSynthetic();
}
}

2
src/Makefile.am
View File

@ -429,6 +429,8 @@ libprotoc_la_SOURCES = \
google/protobuf/compiler/java/message_field_lite.h \
google/protobuf/compiler/java/message_lite.cc \
google/protobuf/compiler/java/message_lite.h \
google/protobuf/compiler/java/message_serialization.cc \
google/protobuf/compiler/java/message_serialization.h \
google/protobuf/compiler/java/name_resolver.cc \
google/protobuf/compiler/java/name_resolver.h \
google/protobuf/compiler/java/options.h \

2
src/file_lists.cmake
View File

@ -369,6 +369,7 @@ set(libprotoc_srcs
${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/message_field.cc
${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/message_field_lite.cc
${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/message_lite.cc
${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/message_serialization.cc
${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/name_resolver.cc
${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/primitive_field.cc
${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/primitive_field_lite.cc
@ -461,6 +462,7 @@ set(libprotoc_hdrs
${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/message_field.h
${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/message_field_lite.h
${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/message_lite.h
${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/message_serialization.h
${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/name_resolver.h
${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/names.h
${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/options.h

21
src/google/protobuf/arena_impl.h
View File

@ -55,6 +55,13 @@ namespace google {
namespace protobuf {
namespace internal {
// To prevent sharing cache lines between threads
#ifdef __cpp_aligned_new
enum { kCacheAlignment = 64 };
#else
enum { kCacheAlignment = alignof(max_align_t) }; // do the best we can
#endif
inline constexpr size_t AlignUpTo8(size_t n) {
// Align n to next multiple of 8 (from Hacker's Delight, Chapter 3.)
return (n + 7) & static_cast<size_t>(-8);
@ -497,10 +504,10 @@ class PROTOBUF_EXPORT ThreadSafeArena {
// have fallback function calls in tail position. This substantially improves
// code for the happy path.
PROTOBUF_NDEBUG_INLINE bool MaybeAllocateAligned(size_t n, void** out) {
SerialArena* a;
SerialArena* arena;
if (PROTOBUF_PREDICT_TRUE(!alloc_policy_.should_record_allocs() &&
GetSerialArenaFromThreadCache(&a))) {
return a->MaybeAllocateAligned(n, out);
GetSerialArenaFromThreadCache(&arena))) {
return arena->MaybeAllocateAligned(n, out);
}
return false;
}
@ -564,7 +571,7 @@ class PROTOBUF_EXPORT ThreadSafeArena {
// fast path optimizes the case where a single thread uses multiple arenas.
ThreadCache* tc = &thread_cache();
SerialArena* serial = hint_.load(std::memory_order_acquire);
if (PROTOBUF_PREDICT_TRUE(serial != NULL && serial->owner() == tc)) {
if (PROTOBUF_PREDICT_TRUE(serial != nullptr && serial->owner() == tc)) {
*arena = serial;
return true;
}
@ -602,7 +609,7 @@ class PROTOBUF_EXPORT ThreadSafeArena {
#ifdef _MSC_VER
#pragma warning(disable : 4324)
#endif
struct alignas(64) ThreadCache {
struct alignas(kCacheAlignment) ThreadCache {
#if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL)
// If we are using the ThreadLocalStorage class to store the ThreadCache,
// then the ThreadCache's default constructor has to be responsible for
@ -610,7 +617,7 @@ class PROTOBUF_EXPORT ThreadSafeArena {
ThreadCache()
: next_lifecycle_id(0),
last_lifecycle_id_seen(-1),
last_serial_arena(NULL) {}
last_serial_arena(nullptr) {}
#endif
// Number of per-thread lifecycle IDs to reserve. Must be power of two.
@ -633,7 +640,7 @@ class PROTOBUF_EXPORT ThreadSafeArena {
#ifdef _MSC_VER
#pragma warning(disable : 4324)
#endif
struct alignas(64) CacheAlignedLifecycleIdGenerator {
struct alignas(kCacheAlignment) CacheAlignedLifecycleIdGenerator {
std::atomic<LifecycleIdAtomic> id;
};
static CacheAlignedLifecycleIdGenerator lifecycle_id_generator_;

6
src/google/protobuf/arenastring.cc
View File

@ -200,7 +200,7 @@ std::string* ArenaStringPtr::Release() {
if (IsDefault()) return nullptr;
std::string* released = tagged_ptr_.Get();
if (!tagged_ptr_.IsAllocated()) {
if (tagged_ptr_.IsArena()) {
released = tagged_ptr_.IsMutable() ? new std::string(std::move(*released))
: new std::string(*released);
}
@ -229,9 +229,7 @@ void ArenaStringPtr::SetAllocated(std::string* value, Arena* arena) {
}
void ArenaStringPtr::Destroy() {
if (tagged_ptr_.IsAllocated()) {
delete tagged_ptr_.Get();
}
delete tagged_ptr_.GetIfAllocated();
}
void ArenaStringPtr::ClearToEmpty() {

27
src/google/protobuf/arenastring.h
View File

@ -96,13 +96,12 @@ class PROTOBUF_EXPORT LazyString {
class TaggedStringPtr {
public:
// Bit flags qualifying string properties. We can use up to 3 bits as
// ptr_ is guaranteed and enforced to be aligned on 8 byte boundaries.
// Bit flags qualifying string properties. We can use 2 bits as
// ptr_ is guaranteed and enforced to be aligned on 4 byte boundaries.
enum Flags {
kArenaBit = 0x1, // ptr is arena allocated
kAllocatedBit = 0x2, // ptr is heap allocated
kMutableBit = 0x4, // ptr contents are fully mutable
kMask = 0x7 // Bit mask
kMutableBit = 0x2, // ptr contents are fully mutable
kMask = 0x3 // Bit mask
};
// Composed logical types
@ -112,7 +111,7 @@ class TaggedStringPtr {
// Allocated strings are mutable and (as the name implies) owned.
// A heap allocated string must be deleted.
kAllocated = kAllocatedBit | kMutableBit,
kAllocated = kMutableBit,
// Mutable arena strings are strings where the string instance is owned
// by the arena, but the string contents itself are owned by the string
@ -166,8 +165,16 @@ class TaggedStringPtr {
// Returns true if the current string is an immutable default value.
inline bool IsDefault() const { return (as_int() & kMask) == kDefault; }
// Returns true if the current string is a heap allocated mutable value.
inline bool IsAllocated() const { return as_int() & kAllocatedBit; }
// If the current string is a heap-allocated mutable value, returns a pointer
// to it. Returns nullptr otherwise.
inline std::string *GetIfAllocated() const {
auto allocated = as_int() ^ kAllocated;
if (allocated & kMask) return nullptr;
auto ptr = reinterpret_cast<std::string*>(allocated);
PROTOBUF_ASSUME(ptr != nullptr);
return ptr;
}
// Returns true if the current string is an arena allocated value.
// This means it's either a mutable or fixed size arena string.
@ -224,8 +231,8 @@ static_assert(std::is_trivial<TaggedStringPtr>::value,
// Because ArenaStringPtr is used in oneof unions, its constructor is a NOP and
// the field is always manually initialized via method calls.
//
// See TaggedPtr for more information about the types of string values being
// held, and the mutable and ownership invariants for each type.
// See TaggedStringPtr for more information about the types of string values
// being held, and the mutable and ownership invariants for each type.
struct PROTOBUF_EXPORT ArenaStringPtr {
ArenaStringPtr() = default;
constexpr ArenaStringPtr(ExplicitlyConstructedArenaString* default_value,

28
src/google/protobuf/arenaz_sampler.cc
View File

@ -61,12 +61,13 @@ PROTOBUF_THREAD_LOCAL absl::profiling_internal::ExponentialBiased
} // namespace
PROTOBUF_THREAD_LOCAL int64_t global_next_sample = 1LL << 10;
PROTOBUF_THREAD_LOCAL SamplingState global_sampling_state = {
.next_sample = int64_t{1} << 10, .sample_stride = int64_t{1} << 10};
ThreadSafeArenaStats::ThreadSafeArenaStats() { PrepareForSampling(); }
ThreadSafeArenaStats::ThreadSafeArenaStats() { PrepareForSampling(0); }
ThreadSafeArenaStats::~ThreadSafeArenaStats() = default;
void ThreadSafeArenaStats::PrepareForSampling() {
void ThreadSafeArenaStats::PrepareForSampling(int64_t stride) {
num_allocations.store(0, std::memory_order_relaxed);
num_resets.store(0, std::memory_order_relaxed);
bytes_requested.store(0, std::memory_order_relaxed);
@ -74,6 +75,7 @@ void ThreadSafeArenaStats::PrepareForSampling() {
bytes_wasted.store(0, std::memory_order_relaxed);
max_bytes_allocated.store(0, std::memory_order_relaxed);
thread_ids.store(0, std::memory_order_relaxed);
weight = stride;
// The inliner makes hardcoded skip_count difficult (especially when combined
// with LTO). We use the ability to exclude stacks by regex when encoding
// instead.
@ -105,12 +107,15 @@ void RecordAllocateSlow(ThreadSafeArenaStats* info, size_t requested,
info->thread_ids.fetch_or(tid, std::memory_order_relaxed);
}
ThreadSafeArenaStats* SampleSlow(int64_t* next_sample) {
bool first = *next_sample < 0;
*next_sample = g_exponential_biased_generator.GetStride(
ThreadSafeArenaStats* SampleSlow(SamplingState& sampling_state) {
bool first = sampling_state.next_sample < 0;
const int64_t next_stride = g_exponential_biased_generator.GetStride(
g_arenaz_sample_parameter.load(std::memory_order_relaxed));
// Small values of interval are equivalent to just sampling next time.
ABSL_ASSERT(*next_sample >= 1);
ABSL_ASSERT(next_stride >= 1);
sampling_state.next_sample = next_stride;
const int64_t old_stride =
absl::exchange(sampling_state.sample_stride, next_stride);
// g_arenaz_enabled can be dynamically flipped, we need to set a threshold low
// enough that we will start sampling in a reasonable time, so we just use the
@ -119,11 +124,11 @@ ThreadSafeArenaStats* SampleSlow(int64_t* next_sample) {
// We will only be negative on our first count, so we should just retry in
// that case.
if (first) {
if (PROTOBUF_PREDICT_TRUE(--*next_sample > 0)) return nullptr;
return SampleSlow(next_sample);
if (PROTOBUF_PREDICT_TRUE(--sampling_state.next_sample > 0)) return nullptr;
return SampleSlow(sampling_state);
}
return GlobalThreadSafeArenazSampler().Register();
return GlobalThreadSafeArenazSampler().Register(old_stride);
}
void SetThreadSafeArenazEnabled(bool enabled) {
@ -150,7 +155,8 @@ void SetThreadSafeArenazMaxSamples(int32_t max) {
void SetThreadSafeArenazGlobalNextSample(int64_t next_sample) {
if (next_sample >= 0) {
global_next_sample = next_sample;
global_sampling_state.next_sample = next_sample;
global_sampling_state.sample_stride = next_sample;
} else {
ABSL_RAW_LOG(ERROR, "Invalid thread safe arenaz next sample: %lld",
static_cast<long long>(next_sample)); // NOLINT(runtime/int)

31
src/google/protobuf/arenaz_sampler.h
View File

@ -58,8 +58,11 @@ struct ThreadSafeArenaStats
~ThreadSafeArenaStats();
// Puts the object into a clean state, fills in the logically `const` members,
// blocking for any readers that are currently sampling the object.
void PrepareForSampling() ABSL_EXCLUSIVE_LOCKS_REQUIRED(init_mu);
// blocking for any readers that are currently sampling the object. The
// 'stride' parameter is the number of ThreadSafeArenas that were instantiated
// between this sample and the previous one.
void PrepareForSampling(int64_t stride)
ABSL_EXCLUSIVE_LOCKS_REQUIRED(init_mu);
// These fields are mutated by the various Record* APIs and need to be
// thread-safe.
@ -91,7 +94,18 @@ struct ThreadSafeArenaStats
}
};
ThreadSafeArenaStats* SampleSlow(int64_t* next_sample);
struct SamplingState {
// Number of ThreadSafeArenas that should be instantiated before the next
// ThreadSafeArena is sampled. This variable is decremented with each
// instantiation.
int64_t next_sample;
// When we make a sampling decision, we record that distance between from the
// previous sample so we can weight each sample. 'distance' here is the
// number of instantiations of ThreadSafeArena.
int64_t sample_stride;
};
ThreadSafeArenaStats* SampleSlow(SamplingState& sampling_state);
void UnsampleSlow(ThreadSafeArenaStats* info);
class ThreadSafeArenaStatsHandle {
@ -138,24 +152,27 @@ class ThreadSafeArenaStatsHandle {
using ThreadSafeArenazSampler =
::absl::profiling_internal::SampleRecorder<ThreadSafeArenaStats>;
extern PROTOBUF_THREAD_LOCAL int64_t global_next_sample;
extern PROTOBUF_THREAD_LOCAL SamplingState global_sampling_state;
// Returns an RAII sampling handle that manages registration and unregistation
// with the global sampler.
inline ThreadSafeArenaStatsHandle Sample() {
if (PROTOBUF_PREDICT_TRUE(--global_next_sample > 0)) {
if (PROTOBUF_PREDICT_TRUE(--global_sampling_state.next_sample > 0)) {
return ThreadSafeArenaStatsHandle(nullptr);
}
return ThreadSafeArenaStatsHandle(SampleSlow(&global_next_sample));
return ThreadSafeArenaStatsHandle(SampleSlow(global_sampling_state));
}
#else
using SamplingState = int64_t;
struct ThreadSafeArenaStats {
static void RecordAllocateStats(ThreadSafeArenaStats*, size_t /*requested*/,
size_t /*allocated*/, size_t /*wasted*/) {}
};
ThreadSafeArenaStats* SampleSlow(int64_t* next_sample);
ThreadSafeArenaStats* SampleSlow(SamplingState& next_sample);
void UnsampleSlow(ThreadSafeArenaStats* info);
class ThreadSafeArenaStatsHandle {

61
src/google/protobuf/arenaz_sampler_test.cc
View File

@ -64,8 +64,9 @@ std::vector<size_t> GetBytesAllocated(ThreadSafeArenazSampler* s) {
return res;
}
ThreadSafeArenaStats* Register(ThreadSafeArenazSampler* s, size_t size) {
auto* info = s->Register();
ThreadSafeArenaStats* Register(ThreadSafeArenazSampler* s, size_t size,
int64_t stride) {
auto* info = s->Register(stride);
assert(info != nullptr);
info->bytes_allocated.store(size);
return info;
@ -79,8 +80,9 @@ namespace {
TEST(ThreadSafeArenaStatsTest, PrepareForSampling) {
ThreadSafeArenaStats info;
constexpr int64_t kTestStride = 107;
MutexLock l(&info.init_mu);
info.PrepareForSampling();
info.PrepareForSampling(kTestStride);
EXPECT_EQ(info.num_allocations.load(), 0);
EXPECT_EQ(info.num_resets.load(), 0);
@ -88,6 +90,7 @@ TEST(ThreadSafeArenaStatsTest, PrepareForSampling) {
EXPECT_EQ(info.bytes_allocated.load(), 0);
EXPECT_EQ(info.bytes_wasted.load(), 0);
EXPECT_EQ(info.max_bytes_allocated.load(), 0);
EXPECT_EQ(info.weight, kTestStride);
info.num_allocations.store(1, std::memory_order_relaxed);
info.num_resets.store(1, std::memory_order_relaxed);
@ -96,19 +99,21 @@ TEST(ThreadSafeArenaStatsTest, PrepareForSampling) {
info.bytes_wasted.store(1, std::memory_order_relaxed);
info.max_bytes_allocated.store(1, std::memory_order_relaxed);
info.PrepareForSampling();
info.PrepareForSampling(2 * kTestStride);
EXPECT_EQ(info.num_allocations.load(), 0);
EXPECT_EQ(info.num_resets.load(), 0);
EXPECT_EQ(info.bytes_requested.load(), 0);
EXPECT_EQ(info.bytes_allocated.load(), 0);
EXPECT_EQ(info.bytes_wasted.load(), 0);
EXPECT_EQ(info.max_bytes_allocated.load(), 0);
EXPECT_EQ(info.weight, 2 * kTestStride);
}
TEST(ThreadSafeArenaStatsTest, RecordAllocateSlow) {
ThreadSafeArenaStats info;
constexpr int64_t kTestStride = 458;
MutexLock l(&info.init_mu);
info.PrepareForSampling();
info.PrepareForSampling(kTestStride);
RecordAllocateSlow(&info, /*requested=*/100, /*allocated=*/128, /*wasted=*/0);
EXPECT_EQ(info.num_allocations.load(), 1);
EXPECT_EQ(info.num_resets.load(), 0);
@ -128,8 +133,9 @@ TEST(ThreadSafeArenaStatsTest, RecordAllocateSlow) {
TEST(ThreadSafeArenaStatsTest, RecordResetSlow) {
ThreadSafeArenaStats info;
constexpr int64_t kTestStride = 584;
MutexLock l(&info.init_mu);
info.PrepareForSampling();
info.PrepareForSampling(kTestStride);
EXPECT_EQ(info.num_resets.load(), 0);
EXPECT_EQ(info.bytes_allocated.load(), 0);
RecordAllocateSlow(&info, /*requested=*/100, /*allocated=*/128, /*wasted=*/0);
@ -143,11 +149,12 @@ TEST(ThreadSafeArenaStatsTest, RecordResetSlow) {
TEST(ThreadSafeArenazSamplerTest, SmallSampleParameter) {
SetThreadSafeArenazEnabled(true);
SetThreadSafeArenazSampleParameter(100);
constexpr int64_t kTestStride = 0;
for (int i = 0; i < 1000; ++i) {
int64_t next_sample = 0;
ThreadSafeArenaStats* sample = SampleSlow(&next_sample);
EXPECT_GT(next_sample, 0);
SamplingState sampling_state = {kTestStride, kTestStride};
ThreadSafeArenaStats* sample = SampleSlow(sampling_state);
EXPECT_GT(sampling_state.next_sample, 0);
EXPECT_NE(sample, nullptr);
UnsampleSlow(sample);
}
@ -156,11 +163,12 @@ TEST(ThreadSafeArenazSamplerTest, SmallSampleParameter) {
TEST(ThreadSafeArenazSamplerTest, LargeSampleParameter) {
SetThreadSafeArenazEnabled(true);
SetThreadSafeArenazSampleParameter(std::numeric_limits<int32_t>::max());
constexpr int64_t kTestStride = 0;
for (int i = 0; i < 1000; ++i) {
int64_t next_sample = 0;
ThreadSafeArenaStats* sample = SampleSlow(&next_sample);
EXPECT_GT(next_sample, 0);
SamplingState sampling_state = {kTestStride, kTestStride};
ThreadSafeArenaStats* sample = SampleSlow(sampling_state);
EXPECT_GT(sampling_state.next_sample, 0);
EXPECT_NE(sample, nullptr);
UnsampleSlow(sample);
}
@ -187,7 +195,8 @@ TEST(ThreadSafeArenazSamplerTest, Sample) {
TEST(ThreadSafeArenazSamplerTest, Handle) {
auto& sampler = GlobalThreadSafeArenazSampler();
ThreadSafeArenaStatsHandle h(sampler.Register());
constexpr int64_t kTestStride = 17;
ThreadSafeArenaStatsHandle h(sampler.Register(kTestStride));
auto* info = ThreadSafeArenaStatsHandlePeer::GetInfo(&h);
info->bytes_allocated.store(0x12345678, std::memory_order_relaxed);
@ -195,6 +204,7 @@ TEST(ThreadSafeArenazSamplerTest, Handle) {
sampler.Iterate([&](const ThreadSafeArenaStats& h) {
if (&h == info) {
EXPECT_EQ(h.bytes_allocated.load(), 0x12345678);
EXPECT_EQ(h.weight, kTestStride);
found = true;
}
});
@ -216,10 +226,11 @@ TEST(ThreadSafeArenazSamplerTest, Handle) {
TEST(ThreadSafeArenazSamplerTest, Registration) {
ThreadSafeArenazSampler sampler;
auto* info1 = Register(&sampler, 1);
constexpr int64_t kTestStride = 100;
auto* info1 = Register(&sampler, 1, kTestStride);
EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(1));
auto* info2 = Register(&sampler, 2);
auto* info2 = Register(&sampler, 2, kTestStride);
EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(1, 2));
info1->bytes_allocated.store(3);
EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(3, 2));
@ -231,16 +242,17 @@ TEST(ThreadSafeArenazSamplerTest, Registration) {
TEST(ThreadSafeArenazSamplerTest, Unregistration) {
ThreadSafeArenazSampler sampler;
std::vector<ThreadSafeArenaStats*> infos;
constexpr int64_t kTestStride = 200;
for (size_t i = 0; i < 3; ++i) {
infos.push_back(Register(&sampler, i));
infos.push_back(Register(&sampler, i, kTestStride));
}
EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(0, 1, 2));
sampler.Unregister(infos[1]);
EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(0, 2));
infos.push_back(Register(&sampler, 3));
infos.push_back(Register(&sampler, 4));
infos.push_back(Register(&sampler, 3, kTestStride));
infos.push_back(Register(&sampler, 4, kTestStride));
EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(0, 2, 3, 4));
sampler.Unregister(infos[3]);
EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(0, 2, 4));
@ -257,18 +269,19 @@ TEST(ThreadSafeArenazSamplerTest, MultiThreaded) {
ThreadPool pool(10);
for (int i = 0; i < 10; ++i) {
pool.Schedule([&sampler, &stop]() {
const int64_t sampling_stride = 11 + i % 3;
pool.Schedule([&sampler, &stop, sampling_stride]() {
std::random_device rd;
std::mt19937 gen(rd());
std::vector<ThreadSafeArenaStats*> infoz;
while (!stop.HasBeenNotified()) {
if (infoz.empty()) {
infoz.push_back(sampler.Register());
infoz.push_back(sampler.Register(sampling_stride));
}
switch (std::uniform_int_distribution<>(0, 1)(gen)) {
case 0: {
infoz.push_back(sampler.Register());
infoz.push_back(sampler.Register(sampling_stride));
break;
}
case 1: {
@ -277,6 +290,7 @@ TEST(ThreadSafeArenazSamplerTest, MultiThreaded) {
ThreadSafeArenaStats* info = infoz[p];
infoz[p] = infoz.back();
infoz.pop_back();
EXPECT_EQ(info->weight, sampling_stride);
sampler.Unregister(info);
break;
}
@ -292,9 +306,10 @@ TEST(ThreadSafeArenazSamplerTest, MultiThreaded) {
TEST(ThreadSafeArenazSamplerTest, Callback) {
ThreadSafeArenazSampler sampler;
constexpr int64_t kTestStride = 203;
auto* info1 = Register(&sampler, 1);
auto* info2 = Register(&sampler, 2);
auto* info1 = Register(&sampler, 1, kTestStride);
auto* info2 = Register(&sampler, 2, kTestStride);
static const ThreadSafeArenaStats* expected;

9
src/google/protobuf/compiler/cpp/enum_field.cc
View File

@ -252,7 +252,9 @@ void RepeatedEnumFieldGenerator::GeneratePrivateMembers(
format("::$proto_ns$::RepeatedField<int> $name$_;\n");
if (descriptor_->is_packed() &&
HasGeneratedMethods(descriptor_->file(), options_)) {
format("mutable std::atomic<int> $cached_byte_size_name$;\n");
format(
"mutable ::$proto_ns$::internal::CachedSize "
"$cached_byte_size_name$;\n");
}
}
@ -364,7 +366,7 @@ void RepeatedEnumFieldGenerator::GenerateSerializeWithCachedSizesToArray(
format(
"{\n"
" int byte_size = "
"$cached_byte_size_field$.load(std::memory_order_relaxed);\n"
"$cached_byte_size_field$.Get();\n"
" if (byte_size > 0) {\n"
" target = stream->WriteEnumPacked(\n"
" $number$, $field$, byte_size, target);\n"
@ -402,8 +404,7 @@ void RepeatedEnumFieldGenerator::GenerateByteSize(io::Printer* printer) const {
"::_pbi::WireFormatLite::Int32Size(static_cast<$int32$>(data_size));\n"
"}\n"
"int cached_size = ::_pbi::ToCachedSize(data_size);\n"
"$cached_byte_size_field$.store(cached_size,\n"
" std::memory_order_relaxed);\n"
"$cached_byte_size_field$.Set(cached_size);\n"
"total_size += data_size;\n");
} else {
format("total_size += ($tag_size$UL * count) + data_size;\n");

5
src/google/protobuf/compiler/cpp/message.cc
View File

@ -2101,6 +2101,11 @@ void MessageGenerator::GenerateClassMethods(io::Printer* printer) {
"static constexpr int32_t kHasBitsOffset =\n"
" 8 * PROTOBUF_FIELD_OFFSET($classname$, _impl_._has_bits_);\n");
}
if (descriptor_->real_oneof_decl_count() > 0) {
format(
"static constexpr int32_t kOneofCaseOffset =\n"
" PROTOBUF_FIELD_OFFSET($classtype$, $oneof_case$);\n");
}
for (auto field : FieldRange(descriptor_)) {
field_generators_.get(field).GenerateInternalAccessorDeclarations(printer);
if (IsFieldStripped(field, options_)) {

18
src/google/protobuf/compiler/cpp/parse_function_generator.cc
View File

@ -292,23 +292,11 @@ TailCallTableInfo::TailCallTableInfo(
const std::vector<int>& has_bit_indices,
const std::vector<int>& inlined_string_indices,
MessageSCCAnalyzer* scc_analyzer) {
int oneof_count = descriptor->real_oneof_decl_count();
// If this message has any oneof fields, store the case offset in the first
// auxiliary entry.
if (oneof_count > 0) {
GOOGLE_LOG_IF(DFATAL, ordered_fields.empty())
<< "Invalid message: " << descriptor->full_name() << " has "
<< oneof_count << " oneof declarations, but no fields";
aux_entries.push_back(StrCat("_fl::Offset{offsetof(",
ClassName(descriptor),
", _impl_._oneof_case_)}"));
}
// If this message has any inlined string fields, store the donation state
// offset in the second auxiliary entry.
if (!inlined_string_indices.empty()) {
aux_entries.resize(2); // pad if necessary
aux_entries[1] =
aux_entries.resize(1); // pad if necessary
aux_entries[0] =
StrCat("_fl::Offset{offsetof(", ClassName(descriptor),
", _impl_._inlined_string_donated_)}");
}
@ -1102,7 +1090,7 @@ void ParseFunctionGenerator::GenerateFieldEntries(Formatter& format) {
FieldMemberName(field, /*cold=*/false));
}
if (oneof) {
format("$1$, ", oneof->index());
format("_Internal::kOneofCaseOffset + $1$, ", 4 * oneof->index());
} else if (num_hasbits_ > 0 || IsMapEntryMessage(descriptor_)) {
if (entry.hasbit_idx >= 0) {
format("_Internal::kHasBitsOffset + $1$, ", entry.hasbit_idx);

9
src/google/protobuf/compiler/cpp/primitive_field.cc
View File

@ -328,7 +328,9 @@ void RepeatedPrimitiveFieldGenerator::GeneratePrivateMembers(
format("::$proto_ns$::RepeatedField< $type$ > $name$_;\n");
if (descriptor_->is_packed() && FixedSize(descriptor_->type()) == -1 &&
HasGeneratedMethods(descriptor_->file(), options_)) {
format("mutable std::atomic<int> $cached_byte_size_name$;\n");
format(
"mutable ::$proto_ns$::internal::CachedSize "
"$cached_byte_size_name$;\n");
}
}
@ -433,7 +435,7 @@ void RepeatedPrimitiveFieldGenerator::GenerateSerializeWithCachedSizesToArray(
format(
"{\n"
" int byte_size = "
"$cached_byte_size_field$.load(std::memory_order_relaxed);\n"
"$cached_byte_size_field$.Get();\n"
" if (byte_size > 0) {\n"
" target = stream->Write$declared_type$Packed(\n"
" $number$, _internal_$name$(), byte_size, target);\n"
@ -484,8 +486,7 @@ void RepeatedPrimitiveFieldGenerator::GenerateByteSize(
if (FixedSize(descriptor_->type()) == -1) {
format(
"int cached_size = ::_pbi::ToCachedSize(data_size);\n"
"$cached_byte_size_field$.store(cached_size,\n"
" std::memory_order_relaxed);\n");
"$cached_byte_size_field$.Set(cached_size);\n");
}
format("total_size += data_size;\n");
} else {

38
src/google/protobuf/compiler/java/message.cc
View File

@ -53,6 +53,7 @@
#include <google/protobuf/compiler/java/helpers.h>
#include <google/protobuf/compiler/java/message_builder.h>
#include <google/protobuf/compiler/java/message_builder_lite.h>
#include <google/protobuf/compiler/java/message_serialization.h>
#include <google/protobuf/compiler/java/name_resolver.h>
#include <google/protobuf/descriptor.pb.h>
@ -587,13 +588,6 @@ void ImmutableMessageGenerator::GenerateMessageSerializationMethods(
std::unique_ptr<const FieldDescriptor*[]> sorted_fields(
SortFieldsByNumber(descriptor_));
std::vector<const Descriptor::ExtensionRange*> sorted_extensions;
sorted_extensions.reserve(descriptor_->extension_range_count());
for (int i = 0; i < descriptor_->extension_range_count(); ++i) {
sorted_extensions.push_back(descriptor_->extension_range(i));
}
std::sort(sorted_extensions.begin(), sorted_extensions.end(),
ExtensionRangeOrdering());
printer->Print(
"@java.lang.Override\n"
"public void writeTo(com.google.protobuf.CodedOutputStream output)\n"
@ -628,19 +622,8 @@ void ImmutableMessageGenerator::GenerateMessageSerializationMethods(
}
}
// Merge the fields and the extension ranges, both sorted by field number.
for (int i = 0, j = 0;
i < descriptor_->field_count() || j < sorted_extensions.size();) {
if (i == descriptor_->field_count()) {
GenerateSerializeOneExtensionRange(printer, sorted_extensions[j++]);
} else if (j == sorted_extensions.size()) {
GenerateSerializeOneField(printer, sorted_fields[i++]);
} else if (sorted_fields[i]->number() < sorted_extensions[j]->start) {
GenerateSerializeOneField(printer, sorted_fields[i++]);
} else {
GenerateSerializeOneExtensionRange(printer, sorted_extensions[j++]);
}
}
GenerateSerializeFieldsAndExtensions(printer, field_generators_, descriptor_,
sorted_fields.get());
if (descriptor_->options().message_set_wire_format()) {
printer->Print("unknownFields.writeAsMessageSetTo(output);\n");
@ -770,17 +753,6 @@ void ImmutableMessageGenerator::GenerateParseFromMethods(io::Printer* printer) {
GeneratedCodeVersionSuffix());
}
void ImmutableMessageGenerator::GenerateSerializeOneField(
io::Printer* printer, const FieldDescriptor* field) {
field_generators_.get(field).GenerateSerializationCode(printer);
}
void ImmutableMessageGenerator::GenerateSerializeOneExtensionRange(
io::Printer* printer, const Descriptor::ExtensionRange* range) {
printer->Print("extensionWriter.writeUntil($end$, output);\n", "end",
StrCat(range->end));
}
// ===================================================================
void ImmutableMessageGenerator::GenerateBuilder(io::Printer* printer) {
@ -1017,8 +989,8 @@ void ImmutableMessageGenerator::GenerateEqualsAndHashCode(
" return true;\n"
"}\n"
"if (!(obj instanceof $classname$)) {\n"
// don't simply return false because mutable and immutable types
// can be equal
// don't simply return false because mutable and immutable types
// can be equal
" return super.equals(obj);\n"
"}\n"
"$classname$ other = ($classname$) obj;\n"

4
src/google/protobuf/compiler/java/message.h
View File

@ -123,10 +123,6 @@ class ImmutableMessageGenerator : public MessageGenerator {
void GenerateMessageSerializationMethods(io::Printer* printer);
void GenerateParseFromMethods(io::Printer* printer);
void GenerateSerializeOneField(io::Printer* printer,
const FieldDescriptor* field);
void GenerateSerializeOneExtensionRange(
io::Printer* printer, const Descriptor::ExtensionRange* range);
void GenerateBuilder(io::Printer* printer);
void GenerateIsInitialized(io::Printer* printer);

35
csharp/src/Google.Protobuf/Compatibility/MethodInfoExtensions.cs → src/google/protobuf/compiler/java/message_serialization.cc
View File

@ -1,6 +1,5 @@
#region Copyright notice and license
// Protocol Buffers - Google's data interchange format
// Copyright 2017 Google Inc. All rights reserved.
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
@ -28,20 +27,24 @@
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#endregion
#if NET35
using System;
using System.Reflection;
#include <google/protobuf/compiler/java/message_serialization.h>
namespace Google.Protobuf.Compatibility
{
// .NET Core (at least netstandard1.0) doesn't have Delegate.CreateDelegate, and .NET 3.5 doesn't have
// MethodInfo.CreateDelegate. Proxy from one to the other on .NET 3.5...
internal static class MethodInfoExtensions
{
internal static Delegate CreateDelegate(this MethodInfo method, Type type) =>
Delegate.CreateDelegate(type, method);
}
#include <google/protobuf/io/printer.h>
#include <google/protobuf/descriptor.h>
namespace google {
namespace protobuf {
namespace compiler {
namespace java {
void GenerateSerializeExtensionRange(io::Printer* printer,
const Descriptor::ExtensionRange* range) {
printer->Print("extensionWriter.writeUntil($end$, output);\n", "end",
StrCat(range->end));
}
#endif
} // namespace java
} // namespace compiler
} // namespace protobuf
} // namespace google

91
src/google/protobuf/compiler/java/message_serialization.h Normal file
View File

@ -0,0 +1,91 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef GOOGLE_PROTOBUF_COMPILER_JAVA_MESSAGE_SERIALIZATION_H__
#define GOOGLE_PROTOBUF_COMPILER_JAVA_MESSAGE_SERIALIZATION_H__
#include <algorithm>
#include <vector>
#include <google/protobuf/io/printer.h>
#include <google/protobuf/descriptor.h>
#include <google/protobuf/compiler/java/field.h>
#include <google/protobuf/compiler/java/helpers.h>
namespace google {
namespace protobuf {
namespace compiler {
namespace java {
// Generates code to serialize a single extension range.
void GenerateSerializeExtensionRange(io::Printer* printer,
const Descriptor::ExtensionRange* range);
// Generates code to serialize all fields and extension ranges for the specified
// message descriptor, sorting serialization calls in increasing order by field
// number.
//
// Templatized to support different field generator implementations.
template <typename FieldGenerator>
void GenerateSerializeFieldsAndExtensions(
io::Printer* printer,
const FieldGeneratorMap<FieldGenerator>& field_generators,
const Descriptor* descriptor, const FieldDescriptor** sorted_fields) {
std::vector<const Descriptor::ExtensionRange*> sorted_extensions;
sorted_extensions.reserve(descriptor->extension_range_count());
for (int i = 0; i < descriptor->extension_range_count(); ++i) {
sorted_extensions.push_back(descriptor->extension_range(i));
}
std::sort(sorted_extensions.begin(), sorted_extensions.end(),
ExtensionRangeOrdering());
// Merge the fields and the extension ranges, both sorted by field number.
for (int i = 0, j = 0;
i < descriptor->field_count() || j < sorted_extensions.size();) {
if (i == descriptor->field_count()) {
GenerateSerializeExtensionRange(printer, sorted_extensions[j++]);
} else if (j == sorted_extensions.size()) {
field_generators.get(sorted_fields[i++])
.GenerateSerializationCode(printer);
} else if (sorted_fields[i]->number() < sorted_extensions[j]->start) {
field_generators.get(sorted_fields[i++])
.GenerateSerializationCode(printer);
} else {
GenerateSerializeExtensionRange(printer, sorted_extensions[j++]);
}
}
}
} // namespace java
} // namespace compiler
} // namespace protobuf
} // namespace google
#endif // GOOGLE_PROTOBUF_COMPILER_JAVA_MESSAGE_SERIALIZATION_H__

4
src/google/protobuf/compiler/objectivec/objectivec_file.h
View File

@ -31,9 +31,9 @@
#ifndef GOOGLE_PROTOBUF_COMPILER_OBJECTIVEC_FILE_H__
#define GOOGLE_PROTOBUF_COMPILER_OBJECTIVEC_FILE_H__
#include <map>
#include <set>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <vector>
#include <google/protobuf/descriptor.h>
#include <google/protobuf/io/printer.h>

4
src/google/protobuf/compiler/objectivec/objectivec_generator.cc
View File

@ -230,6 +230,10 @@ bool ObjectiveCGenerator::GenerateAll(
// - A comment can go on a line after a expected package/prefix pair.
// (i.e. - "some.proto.package # comment")
SetProtoPackagePrefixExceptionList(options[i].second);
} else if (options[i].first == "package_as_prefix_forced_prefix") {
// String to use as the prefix when deriving a prefix from the package
// name. So this only applies when use_package_as_prefix is also used.
SetForcedPackagePrefix(options[i].second);
} else if (options[i].first == "headers_use_forward_declarations") {
if (!StringToBool(options[i].second,
&generation_options.headers_use_forward_declarations)) {

11
src/google/protobuf/compiler/objectivec/objectivec_helpers.cc
View File

@ -134,6 +134,7 @@ class PrefixModeStorage {
// When using a proto package as the prefix, this should be added as the
// prefix in front of it.
const std::string& forced_package_prefix() const { return forced_prefix_; }
void set_forced_package_prefix(const std::string& prefix) { forced_prefix_ = prefix; }
private:
bool use_package_name_;
@ -155,8 +156,6 @@ PrefixModeStorage::PrefixModeStorage() {
exception_path_ = exception_path;
}
// This one is a not expected to be common, so it doesn't get a generation
// option, just the env var.
const char* prefix = getenv("GPB_OBJC_USE_PACKAGE_AS_PREFIX_PREFIX");
if (prefix) {
forced_prefix_ = prefix;
@ -254,6 +253,14 @@ void SetProtoPackagePrefixExceptionList(const std::string& file_path) {
g_prefix_mode.set_exception_path(file_path);
}
std::string GetForcedPackagePrefix() {
return g_prefix_mode.forced_package_prefix();
}
void SetForcedPackagePrefix(const std::string& prefix) {
g_prefix_mode.set_forced_package_prefix(prefix);
}
Options::Options() {
// While there are generator options, also support env variables to help with
// build systems where it isn't as easy to hook in for add the generation

4
src/google/protobuf/compiler/objectivec/objectivec_helpers.h
View File

@ -63,6 +63,10 @@ void PROTOC_EXPORT SetUseProtoPackageAsDefaultPrefix(bool on_or_off);
std::string PROTOC_EXPORT GetProtoPackagePrefixExceptionList();
void PROTOC_EXPORT SetProtoPackagePrefixExceptionList(
const std::string& file_path);
// Get/Set a prefix to add before the prefix generated from the package name.
// This is only used when UseProtoPackageAsDefaultPrefix() is True.
std::string PROTOC_EXPORT GetForcedPackagePrefix();
void PROTOC_EXPORT SetForcedPackagePrefix(const std::string& prefix);
// Generator Prefix Validation Options (see objectivec_generator.cc for a
// description of each):

15
src/google/protobuf/descriptor.pb.cc
View File

@ -10437,7 +10437,7 @@ uint8_t* SourceCodeInfo_Location::_InternalSerialize(
// repeated int32 path = 1 [packed = true];
{
int byte_size = _impl_._path_cached_byte_size_.load(std::memory_order_relaxed);
int byte_size = _impl_._path_cached_byte_size_.Get();
if (byte_size > 0) {
target = stream->WriteInt32Packed(
1, _internal_path(), byte_size, target);
@ -10446,7 +10446,7 @@ uint8_t* SourceCodeInfo_Location::_InternalSerialize(
// repeated int32 span = 2 [packed = true];
{
int byte_size = _impl_._span_cached_byte_size_.load(std::memory_order_relaxed);
int byte_size = _impl_._span_cached_byte_size_.Get();
if (byte_size > 0) {
target = stream->WriteInt32Packed(
2, _internal_span(), byte_size, target);
@ -10509,8 +10509,7 @@ size_t SourceCodeInfo_Location::ByteSizeLong() const {
::_pbi::WireFormatLite::Int32Size(static_cast<int32_t>(data_size));
}
int cached_size = ::_pbi::ToCachedSize(data_size);
_impl_._path_cached_byte_size_.store(cached_size,
std::memory_order_relaxed);
_impl_._path_cached_byte_size_.Set(cached_size);
total_size += data_size;
}
@ -10523,8 +10522,7 @@ size_t SourceCodeInfo_Location::ByteSizeLong() const {
::_pbi::WireFormatLite::Int32Size(static_cast<int32_t>(data_size));
}
int cached_size = ::_pbi::ToCachedSize(data_size);
_impl_._span_cached_byte_size_.store(cached_size,
std::memory_order_relaxed);
_impl_._span_cached_byte_size_.Set(cached_size);
total_size += data_size;
}
@ -10996,7 +10994,7 @@ uint8_t* GeneratedCodeInfo_Annotation::_InternalSerialize(
// repeated int32 path = 1 [packed = true];
{
int byte_size = _impl_._path_cached_byte_size_.load(std::memory_order_relaxed);
int byte_size = _impl_._path_cached_byte_size_.Get();
if (byte_size > 0) {
target = stream->WriteInt32Packed(
1, _internal_path(), byte_size, target);
@ -11051,8 +11049,7 @@ size_t GeneratedCodeInfo_Annotation::ByteSizeLong() const {
::_pbi::WireFormatLite::Int32Size(static_cast<int32_t>(data_size));
}
int cached_size = ::_pbi::ToCachedSize(data_size);
_impl_._path_cached_byte_size_.store(cached_size,
std::memory_order_relaxed);
_impl_._path_cached_byte_size_.Set(cached_size);
total_size += data_size;
}

6
src/google/protobuf/descriptor.pb.h
View File

@ -7962,9 +7962,9 @@ class PROTOBUF_EXPORT SourceCodeInfo_Location final :
::PROTOBUF_NAMESPACE_ID::internal::HasBits<1> _has_bits_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t > path_;
mutable std::atomic<int> _path_cached_byte_size_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _path_cached_byte_size_;
::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t > span_;
mutable std::atomic<int> _span_cached_byte_size_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _span_cached_byte_size_;
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string> leading_detached_comments_;
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr leading_comments_;
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr trailing_comments_;
@ -8350,7 +8350,7 @@ class PROTOBUF_EXPORT GeneratedCodeInfo_Annotation final :
::PROTOBUF_NAMESPACE_ID::internal::HasBits<1> _has_bits_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t > path_;
mutable std::atomic<int> _path_cached_byte_size_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _path_cached_byte_size_;
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr source_file_;
int32_t begin_;
int32_t end_;

55
src/google/protobuf/generated_message_tctable_lite.cc
View File

@ -459,30 +459,36 @@ inline PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedParseMessageAuxImpl(
if (PROTOBUF_PREDICT_FALSE(data.coded_tag<TagType>() != 0)) {
PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS);
}
auto saved_tag = UnalignedLoad<TagType>(ptr);
ptr += sizeof(TagType);
SyncHasbits(msg, hasbits, table);
auto* aux = table->field_aux(data.aux_idx());
if (aux_is_table) {
auto* inner_table = aux->table;
auto& field = RefAt<RepeatedPtrFieldBase>(msg, data.offset());
const auto expected_tag = UnalignedLoad<TagType>(ptr);
const auto aux = *table->field_aux(data.aux_idx());
auto& field = RefAt<RepeatedPtrFieldBase>(msg, data.offset());
do {
ptr += sizeof(TagType);
MessageLite* submsg = field.Add<GenericTypeHandler<MessageLite>>(
inner_table->default_instance);
if (group_coding) {
return ctx->ParseGroup<TcParser>(submsg, ptr, FastDecodeTag(saved_tag),
inner_table);
aux_is_table ? aux.table->default_instance : aux.message_default());
if (aux_is_table) {
if (group_coding) {
ptr = ctx->ParseGroup<TcParser>(submsg, ptr,
FastDecodeTag(expected_tag), aux.table);
} else {
ptr = ctx->ParseMessage<TcParser>(submsg, ptr, aux.table);
}
} else {
if (group_coding) {
ptr = ctx->ParseGroup(submsg, ptr, FastDecodeTag(expected_tag));
} else {
ptr = ctx->ParseMessage(submsg, ptr);
}
}
return ctx->ParseMessage<TcParser>(submsg, ptr, inner_table);
} else {
const MessageLite* default_instance = aux->message_default();
auto& field = RefAt<RepeatedPtrFieldBase>(msg, data.offset());
MessageLite* submsg =
field.Add<GenericTypeHandler<MessageLite>>(default_instance);
if (group_coding) {
return ctx->ParseGroup(submsg, ptr, FastDecodeTag(saved_tag));
if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) {
PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_PASS);
}
return ctx->ParseMessage(submsg, ptr);
}
if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) {
PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_PASS);
}
} while (UnalignedLoad<TagType>(ptr) == expected_tag);
PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS);
}
const char* TcParser::FastMdR1(PROTOBUF_TC_PARAM_DECL) {
@ -1356,11 +1362,8 @@ bool TcParser::ChangeOneof(const TcParseTableBase* table,
const TcParseTableBase::FieldEntry& entry,
uint32_t field_num, ParseContext* ctx,
MessageLite* msg) {
// The _oneof_case_ array offset is stored in the first aux entry.
uint32_t oneof_case_offset = table->field_aux(0u)->offset;
// The _oneof_case_ array index is stored in the has-bit index.
uint32_t* oneof_case =
&TcParser::RefAt<uint32_t>(msg, oneof_case_offset) + entry.has_idx;
// The _oneof_case_ value offset is stored in the has-bit index.
uint32_t* oneof_case = &TcParser::RefAt<uint32_t>(msg, entry.has_idx);
uint32_t current_case = *oneof_case;
*oneof_case = field_num;

51
src/google/protobuf/generated_message_util.h
View File

@ -185,13 +185,58 @@ T* GetOwnedMessage(Arena* message_arena, T* submessage,
// Hide atomic from the public header and allow easy change to regular int
// on platforms where the atomic might have a perf impact.
//
// CachedSize is like std::atomic<int> but with some important changes:
//
// 1) CachedSize uses Get / Set rather than load / store.
// 2) CachedSize always uses relaxed ordering.
// 3) CachedSize is assignable and copy-constructible.
// 4) CachedSize has a constexpr default constructor, and a constexpr
// constructor that takes an int argument.
// 5) If the compiler supports the __atomic_load_n / __atomic_store_n builtins,
// then CachedSize is trivially copyable.
//
// Developed at https://godbolt.org/z/vYcx7zYs1 ; supports gcc, clang, MSVC.
class PROTOBUF_EXPORT CachedSize {
private:
using Scalar = int;
public:
int Get() const { return size_.load(std::memory_order_relaxed); }
void Set(int size) { size_.store(size, std::memory_order_relaxed); }
constexpr CachedSize() noexcept : atom_(Scalar{}) {}
// NOLINTNEXTLINE(google-explicit-constructor)
constexpr CachedSize(Scalar desired) noexcept : atom_(desired) {}
#if PROTOBUF_BUILTIN_ATOMIC
constexpr CachedSize(const CachedSize& other) = default;
Scalar Get() const noexcept {
return __atomic_load_n(&atom_, __ATOMIC_RELAXED);
}
void Set(Scalar desired) noexcept {
__atomic_store_n(&atom_, desired, __ATOMIC_RELAXED);
}
#else
CachedSize(const CachedSize& other) noexcept : atom_(other.Get()) {}
CachedSize& operator=(const CachedSize& other) noexcept {
Set(other.Get());
return *this;
}
Scalar Get() const noexcept { //
return atom_.load(std::memory_order_relaxed);
}
void Set(Scalar desired) noexcept {
atom_.store(desired, std::memory_order_relaxed);
}
#endif
private:
std::atomic<int> size_{0};
#if PROTOBUF_BUILTIN_ATOMIC
Scalar atom_;
#else
std::atomic<Scalar> atom_;
#endif
};
PROTOBUF_EXPORT void DestroyMessage(const void* message);

5
src/google/protobuf/port_def.inc
View File

@ -139,6 +139,11 @@
#define PROTOBUF_BUILTIN_BSWAP64(x) __builtin_bswap64(x)
#endif
// Portable check for gcc-style atomic built-ins
#if __has_builtin(__atomic_load_n)
#define PROTOBUF_BUILTIN_ATOMIC 1
#endif
// Portable check for GCC minimum version:
// https://gcc.gnu.org/onlinedocs/cpp/Common-Predefined-Macros.html
#if defined(__GNUC__) && defined(__GNUC_MINOR__) \

1
src/google/protobuf/port_undef.inc
View File

@ -38,6 +38,7 @@
#undef PROTOBUF_BUILTIN_BSWAP16
#undef PROTOBUF_BUILTIN_BSWAP32
#undef PROTOBUF_BUILTIN_BSWAP64
#undef PROTOBUF_BUILTIN_ATOMIC
#undef PROTOBUF_GNUC_MIN
#undef PROTOBUF_MSC_VER_MIN
#undef PROTOBUF_CPLUSPLUS_MIN

2
src/google/protobuf/struct.pb.cc
View File

@ -419,6 +419,8 @@ void Struct::InternalSwap(Struct* other) {
class Value::_Internal {
public:
static constexpr int32_t kOneofCaseOffset =
PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::Value, _impl_._oneof_case_);
static const ::PROTOBUF_NAMESPACE_ID::Struct& struct_value(const Value* msg);
static const ::PROTOBUF_NAMESPACE_ID::ListValue& list_value(const Value* msg);
};

9
src/google/protobuf/stubs/status.cc
View File

@ -28,9 +28,10 @@
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <google/protobuf/stubs/status.h>
#include <stdio.h>
#include <string.h>
#include <ostream>
#include <stdio.h>
#include <string>
#include <utility>
@ -256,6 +257,12 @@ Status UnknownError(StringPiece message) {
return Status(StatusCode::kUnknown, message);
}
Status ErrnoToStatus(int error_number, StringPiece message) {
// We will take an Abseil dependency soon, so no reason to do anything
// elaborate here.
return InternalError(StrCat(message, ": ", strerror(error_number)));
}
} // namespace status_internal
} // namespace util
} // namespace protobuf

10
src/google/protobuf/stubs/status.h
View File

@ -31,10 +31,12 @@
#ifndef GOOGLE_PROTOBUF_STUBS_STATUS_H_
#define GOOGLE_PROTOBUF_STUBS_STATUS_H_
#include <google/protobuf/stubs/stringpiece.h>
#include <google/protobuf/stubs/strutil.h>
#include <string>
#include <google/protobuf/stubs/stringpiece.h>
// Must be included last.
#include <google/protobuf/port_def.inc>
namespace google {
@ -147,6 +149,8 @@ PROTOBUF_EXPORT Status UnavailableError(StringPiece message);
PROTOBUF_EXPORT Status UnimplementedError(StringPiece message);
PROTOBUF_EXPORT Status UnknownError(StringPiece message);
PROTOBUF_EXPORT Status ErrnoToStatus(int error_number, StringPiece message);
} // namespace status_internal
using ::google::protobuf::util::status_internal::Status;
@ -187,6 +191,8 @@ using ::google::protobuf::util::status_internal::UnavailableError;
using ::google::protobuf::util::status_internal::UnimplementedError;
using ::google::protobuf::util::status_internal::UnknownError;
using ::google::protobuf::util::status_internal::ErrnoToStatus;
} // namespace util
} // namespace protobuf
} // namespace google

65
src/google/protobuf/stubs/strutil.cc
View File

@ -129,11 +129,10 @@ void StripWhitespace(std::string *str) {
// it only replaces the first instance of "old."
// ----------------------------------------------------------------------
void StringReplace(const std::string &s, const std::string &oldsub,
const std::string &newsub, bool replace_all,
std::string *res) {
void StringReplace(StringPiece s, StringPiece oldsub, StringPiece newsub,
bool replace_all, std::string *res) {
if (oldsub.empty()) {
res->append(s); // if empty, append the given string.
StrAppend(res, s); // if empty, append the given string.
return;
}
@ -144,11 +143,11 @@ void StringReplace(const std::string &s, const std::string &oldsub,
if (pos == std::string::npos) {
break;
}
res->append(s, start_pos, pos - start_pos);
res->append(newsub);
StrAppend(res, s.substr(start_pos, pos - start_pos));
StrAppend(res, newsub);
start_pos = pos + oldsub.size(); // start searching again after the "old"
} while (replace_all);
res->append(s, start_pos, s.length() - start_pos);
StrAppend(res, s.substr(start_pos, s.length() - start_pos));
}
// ----------------------------------------------------------------------
@ -160,8 +159,8 @@ void StringReplace(const std::string &s, const std::string &oldsub,
// happened or not.
// ----------------------------------------------------------------------
std::string StringReplace(const std::string &s, const std::string &oldsub,
const std::string &newsub, bool replace_all) {
std::string StringReplace(StringPiece s, StringPiece oldsub, StringPiece newsub,
bool replace_all) {
std::string ret;
StringReplace(s, oldsub, newsub, replace_all, &ret);
return ret;
@ -445,22 +444,24 @@ int UnescapeCEscapeSequences(const char *source, char *dest,
// In the first and second calls, the length of dest is returned. In the
// the third call, the new string is returned.
// ----------------------------------------------------------------------
int UnescapeCEscapeString(const std::string &src, std::string *dest) {
int UnescapeCEscapeString(StringPiece src, std::string *dest) {
return UnescapeCEscapeString(src, dest, nullptr);
}
int UnescapeCEscapeString(const std::string &src, std::string *dest,
int UnescapeCEscapeString(StringPiece src, std::string *dest,
std::vector<std::string> *errors) {
std::unique_ptr<char[]> unescaped(new char[src.size() + 1]);
int len = UnescapeCEscapeSequences(src.c_str(), unescaped.get(), errors);
int len = UnescapeCEscapeSequences(std::string(src).c_str(), unescaped.get(),
errors);
GOOGLE_CHECK(dest);
dest->assign(unescaped.get(), len);
return len;
}
std::string UnescapeCEscapeString(const std::string &src) {
std::string UnescapeCEscapeString(StringPiece src) {
std::unique_ptr<char[]> unescaped(new char[src.size() + 1]);
int len = UnescapeCEscapeSequences(src.c_str(), unescaped.get(), nullptr);
int len = UnescapeCEscapeSequences(std::string(src).c_str(), unescaped.get(),
nullptr);
return std::string(unescaped.get(), len);
}
@ -592,7 +593,7 @@ void CEscapeAndAppend(StringPiece src, std::string *dest) {
}
}
std::string CEscape(const std::string &src) {
std::string CEscape(StringPiece src) {
std::string dest;
CEscapeAndAppend(src, &dest);
return dest;
@ -600,7 +601,7 @@ std::string CEscape(const std::string &src) {
namespace strings {
std::string Utf8SafeCEscape(const std::string &src) {
std::string Utf8SafeCEscape(StringPiece src) {
const int dest_length = src.size() * 4 + 1; // Maximum possible expansion
std::unique_ptr<char[]> dest(new char[dest_length]);
const int len = CEscapeInternal(src.data(), src.size(),
@ -609,7 +610,7 @@ std::string Utf8SafeCEscape(const std::string &src) {
return std::string(dest.get(), len);
}
std::string CHexEscape(const std::string &src) {
std::string CHexEscape(StringPiece src) {
const int dest_length = src.size() * 4 + 1; // Maximum possible expansion
std::unique_ptr<char[]> dest(new char[dest_length]);
const int len = CEscapeInternal(src.data(), src.size(),
@ -663,7 +664,7 @@ uint32_t strtou32_adaptor(const char *nptr, char **endptr, int base) {
return static_cast<uint32_t>(result);
}
inline bool safe_parse_sign(std::string *text /*inout*/,
inline bool safe_parse_sign(StringPiece *text /*inout*/,
bool *negative_ptr /*output*/) {
const char* start = text->data();
const char* end = start + text->size();
@ -692,7 +693,7 @@ inline bool safe_parse_sign(std::string *text /*inout*/,
}
template <typename IntType>
bool safe_parse_positive_int(std::string text, IntType *value_p) {
bool safe_parse_positive_int(StringPiece text, IntType *value_p) {
int base = 10;
IntType value = 0;
const IntType vmax = std::numeric_limits<IntType>::max();
@ -725,7 +726,7 @@ bool safe_parse_positive_int(std::string text, IntType *value_p) {
}
template <typename IntType>
bool safe_parse_negative_int(const std::string &text, IntType *value_p) {
bool safe_parse_negative_int(StringPiece text, IntType *value_p) {
int base = 10;
IntType value = 0;
const IntType vmin = std::numeric_limits<IntType>::min();
@ -765,7 +766,7 @@ bool safe_parse_negative_int(const std::string &text, IntType *value_p) {
}
template <typename IntType>
bool safe_int_internal(std::string text, IntType *value_p) {
bool safe_int_internal(StringPiece text, IntType *value_p) {
*value_p = 0;
bool negative;
if (!safe_parse_sign(&text, &negative)) {
@ -779,7 +780,7 @@ bool safe_int_internal(std::string text, IntType *value_p) {
}
template <typename IntType>
bool safe_uint_internal(std::string text, IntType *value_p) {
bool safe_uint_internal(StringPiece text, IntType *value_p) {
*value_p = 0;
bool negative;
if (!safe_parse_sign(&text, &negative) || negative) {
@ -1341,19 +1342,19 @@ bool safe_strtod(const char* str, double* value) {
return *str != '\0' && *endptr == '\0';
}
bool safe_strto32(const std::string &str, int32_t *value) {
bool safe_strto32(StringPiece str, int32_t *value) {
return safe_int_internal(str, value);
}
bool safe_strtou32(const std::string &str, uint32_t *value) {
bool safe_strtou32(StringPiece str, uint32_t *value) {
return safe_uint_internal(str, value);
}
bool safe_strto64(const std::string &str, int64_t *value) {
bool safe_strto64(StringPiece str, int64_t *value) {
return safe_int_internal(str, value);
}
bool safe_strtou64(const std::string &str, uint64_t *value) {
bool safe_strtou64(StringPiece str, uint64_t *value) {
return safe_uint_internal(str, value);
}
@ -1611,8 +1612,8 @@ void StrAppend(std::string *result, const AlphaNum &a, const AlphaNum &b,
GOOGLE_DCHECK_EQ(out, begin + result->size());
}
int GlobalReplaceSubstring(const std::string &substring,
const std::string &replacement, std::string *s) {
int GlobalReplaceSubstring(StringPiece substring, StringPiece replacement,
std::string *s) {
GOOGLE_CHECK(s != nullptr);
if (s->empty() || substring.empty())
return 0;
@ -2332,15 +2333,15 @@ int UTF8FirstLetterNumBytes(const char* src, int len) {
// (1) determines the presence of LF (first one is ok)
// (2) if yes, removes any CR, else convert every CR to LF
void CleanStringLineEndings(const std::string &src, std::string *dst,
void CleanStringLineEndings(StringPiece src, std::string *dst,
bool auto_end_last_line) {
if (dst->empty()) {
dst->append(src);
StrAppend(dst, src);
CleanStringLineEndings(dst, auto_end_last_line);
} else {
std::string tmp = src;
std::string tmp(src);
CleanStringLineEndings(&tmp, auto_end_last_line);
dst->append(tmp);
StrAppend(dst, tmp);
}
}

65
src/google/protobuf/stubs/strutil.h
View File

@ -118,12 +118,11 @@ inline bool HasPrefixString(StringPiece str, StringPiece prefix) {
memcmp(str.data(), prefix.data(), prefix.size()) == 0;
}
inline std::string StripPrefixString(const std::string& str,
const std::string& prefix) {
inline std::string StripPrefixString(StringPiece str, StringPiece prefix) {
if (HasPrefixString(str, prefix)) {
return str.substr(prefix.size());
return str.substr(prefix.size()).ToString();
} else {
return str;
return str.ToString();
}
}
@ -141,12 +140,11 @@ inline bool HasSuffixString(StringPiece str, StringPiece suffix) {
suffix.size()) == 0;
}
inline std::string StripSuffixString(const std::string& str,
const std::string& suffix) {
inline std::string StripSuffixString(StringPiece str, StringPiece suffix) {
if (HasSuffixString(str, suffix)) {
return str.substr(0, str.size() - suffix.size());
return str.substr(0, str.size() - suffix.size()).ToString();
} else {
return str;
return str.ToString();
}
}
@ -192,8 +190,8 @@ inline void UpperString(std::string* s) {
inline void ToUpper(std::string* s) { UpperString(s); }
inline std::string ToUpper(const std::string& s) {
std::string out = s;
inline std::string ToUpper(StringPiece s) {
std::string out(s);
UpperString(&out);
return out;
}
@ -206,10 +204,8 @@ inline std::string ToUpper(const std::string& s) {
// happened or not.
// ----------------------------------------------------------------------
PROTOBUF_EXPORT std::string StringReplace(const std::string& s,
const std::string& oldsub,
const std::string& newsub,
bool replace_all);
PROTOBUF_EXPORT std::string StringReplace(StringPiece s, StringPiece oldsub,
StringPiece newsub, bool replace_all);
// ----------------------------------------------------------------------
// SplitStringUsing()
@ -314,12 +310,10 @@ PROTOBUF_EXPORT int UnescapeCEscapeSequences(const char* source, char* dest,
// the third call, the new string is returned.
// ----------------------------------------------------------------------
PROTOBUF_EXPORT int UnescapeCEscapeString(const std::string& src,
std::string* dest);
PROTOBUF_EXPORT int UnescapeCEscapeString(const std::string& src,
std::string* dest,
PROTOBUF_EXPORT int UnescapeCEscapeString(StringPiece src, std::string* dest);
PROTOBUF_EXPORT int UnescapeCEscapeString(StringPiece src, std::string* dest,
std::vector<std::string>* errors);
PROTOBUF_EXPORT std::string UnescapeCEscapeString(const std::string& src);
PROTOBUF_EXPORT std::string UnescapeCEscapeString(StringPiece src);
// ----------------------------------------------------------------------
// CEscape()
@ -328,7 +322,7 @@ PROTOBUF_EXPORT std::string UnescapeCEscapeString(const std::string& src);
//
// Escaped chars: \n, \r, \t, ", ', \, and !isprint().
// ----------------------------------------------------------------------
PROTOBUF_EXPORT std::string CEscape(const std::string& src);
PROTOBUF_EXPORT std::string CEscape(StringPiece src);
// ----------------------------------------------------------------------
// CEscapeAndAppend()
@ -339,10 +333,10 @@ PROTOBUF_EXPORT void CEscapeAndAppend(StringPiece src, std::string* dest);
namespace strings {
// Like CEscape() but does not escape bytes with the upper bit set.
PROTOBUF_EXPORT std::string Utf8SafeCEscape(const std::string& src);
PROTOBUF_EXPORT std::string Utf8SafeCEscape(StringPiece src);
// Like CEscape() but uses hex (\x) escapes instead of octals.
PROTOBUF_EXPORT std::string CHexEscape(const std::string& src);
PROTOBUF_EXPORT std::string CHexEscape(StringPiece src);
} // namespace strings
// ----------------------------------------------------------------------
@ -399,35 +393,23 @@ inline uint64_t strtou64(const char *nptr, char **endptr, int base) {
// ----------------------------------------------------------------------
PROTOBUF_EXPORT bool safe_strtob(StringPiece str, bool* value);
PROTOBUF_EXPORT bool safe_strto32(const std::string& str, int32_t* value);
PROTOBUF_EXPORT bool safe_strtou32(const std::string& str, uint32_t* value);
PROTOBUF_EXPORT bool safe_strto32(StringPiece str, int32_t* value);
PROTOBUF_EXPORT bool safe_strtou32(StringPiece str, uint32_t* value);
inline bool safe_strto32(const char* str, int32_t* value) {
return safe_strto32(std::string(str), value);
}
inline bool safe_strto32(StringPiece str, int32_t* value) {
return safe_strto32(str.ToString(), value);
}
inline bool safe_strtou32(const char* str, uint32_t* value) {
return safe_strtou32(std::string(str), value);
}
inline bool safe_strtou32(StringPiece str, uint32_t* value) {
return safe_strtou32(str.ToString(), value);
}
PROTOBUF_EXPORT bool safe_strto64(const std::string& str, int64_t* value);
PROTOBUF_EXPORT bool safe_strtou64(const std::string& str, uint64_t* value);
PROTOBUF_EXPORT bool safe_strto64(StringPiece str, int64_t* value);
PROTOBUF_EXPORT bool safe_strtou64(StringPiece str, uint64_t* value);
inline bool safe_strto64(const char* str, int64_t* value) {
return safe_strto64(std::string(str), value);
}
inline bool safe_strto64(StringPiece str, int64_t* value) {
return safe_strto64(str.ToString(), value);
}
inline bool safe_strtou64(const char* str, uint64_t* value) {
return safe_strtou64(std::string(str), value);
}
inline bool safe_strtou64(StringPiece str, uint64_t* value) {
return safe_strtou64(str.ToString(), value);
}
PROTOBUF_EXPORT bool safe_strtof(const char* str, float* value);
PROTOBUF_EXPORT bool safe_strtod(const char* str, double* value);
@ -798,8 +780,8 @@ PROTOBUF_EXPORT std::string ToHex(uint64_t num);
//
// NOTE: The string pieces must not overlap s.
// ----------------------------------------------------------------------
PROTOBUF_EXPORT int GlobalReplaceSubstring(const std::string& substring,
const std::string& replacement,
PROTOBUF_EXPORT int GlobalReplaceSubstring(StringPiece substring,
StringPiece replacement,
std::string* s);
// ----------------------------------------------------------------------
@ -917,8 +899,7 @@ PROTOBUF_EXPORT int UTF8FirstLetterNumBytes(const char* src, int len);
//
// (1) determines the presence of LF (first one is ok)
// (2) if yes, removes any CR, else convert every CR to LF
PROTOBUF_EXPORT void CleanStringLineEndings(const std::string& src,
std::string* dst,
PROTOBUF_EXPORT void CleanStringLineEndings(StringPiece src, std::string* dst,
bool auto_end_last_line);
// Same as above, but transforms the argument in place.

2
src/google/protobuf/util/internal/proto_writer.cc
View File

@ -73,7 +73,6 @@ ProtoWriter::ProtoWriter(TypeResolver* type_resolver,
element_(nullptr),
size_insert_(),
output_(output),
buffer_(),
adapter_(&buffer_),
stream_(new CodedOutputStream(&adapter_)),
listener_(listener),
@ -95,7 +94,6 @@ ProtoWriter::ProtoWriter(const TypeInfo* typeinfo,
element_(nullptr),
size_insert_(),
output_(output),
buffer_(),
adapter_(&buffer_),
stream_(new CodedOutputStream(&adapter_)),
listener_(listener),

112
src/google/protobuf/util/json_util_test.cc
View File

@ -84,7 +84,7 @@ util::Status GetStatus(const util::StatusOr<T>& s) {
}
MATCHER_P(StatusIs, status,
StrCat(".status() is ", testing::PrintToString(status))) {
StrCat(".status() is ", testing::PrintToString(status))) {
return GetStatus(arg).code() == status;
}
@ -412,6 +412,110 @@ TEST(JsonUtilTest, TestDynamicMessage) {
EXPECT_EQ(*message_json, *generated_json);
}
TEST(JsonUtilTest, TestParsingAny) {
StringPiece input = R"json(
{
"value": {
"@type": "type.googleapis.com/proto3.TestMessage",
"int32_value": 5,
"string_value": "expected_value",
"message_value": {"value": 1}
}
}
)json";
TestAny m;
ASSERT_OK(FromJson(input, &m));
TestMessage t;
EXPECT_TRUE(m.value().UnpackTo(&t));
EXPECT_EQ(t.int32_value(), 5);
EXPECT_EQ(t.string_value(), "expected_value");
EXPECT_EQ(t.message_value().value(), 1);
EXPECT_THAT(
ToJson(m),
IsOkAndHolds(
R"json({"value":{"@type":"type.googleapis.com/proto3.TestMessage","int32Value":5,"stringValue":"expected_value","messageValue":{"value":1}}})json"));
}
TEST(JsonUtilTest, TestParsingAnyMiddleAtType) {
StringPiece input = R"json(
{
"value": {
"int32_value": 5,
"string_value": "expected_value",
"@type": "type.googleapis.com/proto3.TestMessage",
"message_value": {"value": 1}
}
}
)json";
TestAny m;
ASSERT_OK(FromJson(input, &m));
TestMessage t;
EXPECT_TRUE(m.value().UnpackTo(&t));
EXPECT_EQ(t.int32_value(), 5);
EXPECT_EQ(t.string_value(), "expected_value");
EXPECT_EQ(t.message_value().value(), 1);
}
TEST(JsonUtilTest, TestParsingAnyEndAtType) {
StringPiece input = R"json(
{
"value": {
"int32_value": 5,
"string_value": "expected_value",
"message_value": {"value": 1},
"@type": "type.googleapis.com/proto3.TestMessage"
}
}
)json";
TestAny m;
ASSERT_OK(FromJson(input, &m));
TestMessage t;
EXPECT_TRUE(m.value().UnpackTo(&t));
EXPECT_EQ(t.int32_value(), 5);
EXPECT_EQ(t.string_value(), "expected_value");
EXPECT_EQ(t.message_value().value(), 1);
}
TEST(JsonUtilTest, TestParsingNestedAnys) {
StringPiece input = R"json(
{
"value": {
"value": {
"int32_value": 5,
"string_value": "expected_value",
"message_value": {"value": 1},
"@type": "type.googleapis.com/proto3.TestMessage"
},
"@type": "type.googleapis.com/google.protobuf.Any"
}
}
)json";
TestAny m;
ASSERT_OK(FromJson(input, &m));
google::protobuf::Any inner;
EXPECT_TRUE(m.value().UnpackTo(&inner));
TestMessage t;
EXPECT_TRUE(inner.UnpackTo(&t));
EXPECT_EQ(t.int32_value(), 5);
EXPECT_EQ(t.string_value(), "expected_value");
EXPECT_EQ(t.message_value().value(), 1);
EXPECT_THAT(
ToJson(m),
IsOkAndHolds(
R"json({"value":{"@type":"type.googleapis.com/google.protobuf.Any","value":{"@type":"type.googleapis.com/proto3.TestMessage","int32Value":5,"stringValue":"expected_value","messageValue":{"value":1}}}})json"));
}
TEST(JsonUtilTest, TestParsingUnknownAnyFields) {
StringPiece input = R"json(
{
@ -672,9 +776,9 @@ TEST(JsonUtilTest, TestWrongJsonInput) {
auto* resolver = NewTypeResolverForDescriptorPool(
"type.googleapis.com", DescriptorPool::generated_pool());
EXPECT_THAT(JsonToBinaryStream(resolver, message_type, &input_stream,
&output_stream),
StatusIs(util::StatusCode::kInvalidArgument));
EXPECT_THAT(
JsonToBinaryStream(resolver, message_type, &input_stream, &output_stream),
StatusIs(util::StatusCode::kInvalidArgument));
delete resolver;
}