AuroraMiddleware/protobuf
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

Merge branch 'sync-piper' into sync-stage

Browse Source 
# Conflicts:
#	java/core/src/test/java/com/google/protobuf/TextFormatTest.java
This commit is contained in:
Joshua Haberman 2020-05-26 00:26:47 -07:00
commit eee43838f3
41 changed files with 676 additions and 364 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java
View File

@ -234,7 +234,7 @@ public abstract class GeneratedMessageLite<
* It doesn't use or modify any memoized value.
* <li>{@code GET_MEMOIZED_IS_INITIALIZED} returns the memoized {@code isInitialized} byte
* value.
* <li>{@code SET_MEMOIZED_IS_INITIALIZED} sets the memoized {@code isInitilaized} byte value to
* <li>{@code SET_MEMOIZED_IS_INITIALIZED} sets the memoized {@code isInitialized} byte value to
* 1 if the first parameter is not null, or to 0 if the first parameter is null.
* <li>{@code NEW_BUILDER} returns a {@code BuilderType} instance.
* </ul>
@ -461,7 +461,7 @@ public abstract class GeneratedMessageLite<
throws IOException {
copyOnWrite();
try {
// TODO(yilunchong): Try to make input with type CodedInpuStream.ArrayDecoder use
// TODO(yilunchong): Try to make input with type CodedInputStream.ArrayDecoder use
// fast path.
Protobuf.getInstance().schemaFor(instance).mergeFrom(
instance, CodedInputStreamReader.forCodedInput(input), extensionRegistry);

6
java/core/src/test/java/com/google/protobuf/TextFormatTest.java
View File

@ -1419,10 +1419,10 @@ public class TextFormatTest extends TestCase {
+ " value: -1\n"
+ "}\n";
TestMap msg = TextFormat.parse(input, TestMap.class);
int val1 = msg.getInt32ToInt32Field().get(1);
int i1 = msg.getInt32ToInt32Field().get(1);
TestMap msg2 = TextFormat.parse(msg.toString(), TestMap.class);
int val2 = msg2.getInt32ToInt32Field().get(1);
assertEquals(val1, val2);
int i2 = msg2.getInt32ToInt32Field().get(1);
assertEquals(i1, i2);
}
public void testMapShortForm() throws Exception {

42
python/google/protobuf/internal/enum_type_wrapper.py
View File

@ -48,32 +48,38 @@ class EnumTypeWrapper(object):
def __init__(self, enum_type):
"""Inits EnumTypeWrapper with an EnumDescriptor."""
self._enum_type = enum_type
self.DESCRIPTOR = enum_type
self.DESCRIPTOR = enum_type # pylint: disable=invalid-name
def Name(self, number):
def Name(self, number): # pylint: disable=invalid-name
"""Returns a string containing the name of an enum value."""
if number in self._enum_type.values_by_number:
try:
return self._enum_type.values_by_number[number].name
except KeyError:
pass # fall out to break exception chaining
if not isinstance(number, six.integer_types):
raise TypeError('Enum value for %s must be an int, but got %r.' % (
self._enum_type.name, number))
raise TypeError(
'Enum value for {} must be an int, but got {} {!r}.'.format(
self._enum_type.name, type(number), number))
else:
# %r here to handle the odd case when you pass in a boolean.
raise ValueError('Enum %s has no name defined for value %r' % (
# repr here to handle the odd case when you pass in a boolean.
raise ValueError('Enum {} has no name defined for value {!r}'.format(
self._enum_type.name, number))
def Value(self, name):
def Value(self, name): # pylint: disable=invalid-name
"""Returns the value corresponding to the given enum name."""
if name in self._enum_type.values_by_name:
try:
return self._enum_type.values_by_name[name].number
raise ValueError('Enum %s has no value defined for name %s' % (
except KeyError:
pass # fall out to break exception chaining
raise ValueError('Enum {} has no value defined for name {!r}'.format(
self._enum_type.name, name))
def keys(self):
"""Return a list of the string names in the enum.
These are returned in the order they were defined in the .proto file.
Returns:
A list of strs, in the order they were defined in the .proto file.
"""
return [value_descriptor.name
@ -82,7 +88,8 @@ class EnumTypeWrapper(object):
def values(self):
"""Return a list of the integer values in the enum.
These are returned in the order they were defined in the .proto file.
Returns:
A list of ints, in the order they were defined in the .proto file.
"""
return [value_descriptor.number
@ -91,13 +98,18 @@ class EnumTypeWrapper(object):
def items(self):
"""Return a list of the (name, value) pairs of the enum.
These are returned in the order they were defined in the .proto file.
Returns:
A list of (str, int) pairs, in the order they were defined
in the .proto file.
"""
return [(value_descriptor.name, value_descriptor.number)
for value_descriptor in self._enum_type.values]
def __getattr__(self, name):
"""Returns the value corresponding to the given enum name."""
if name in self._enum_type.values_by_name:
try:
return self._enum_type.values_by_name[name].number
raise AttributeError
except KeyError:
pass # fall out to break exception chaining
raise AttributeError('Enum {} has no value defined for name {!r}'.format(
self._enum_type.name, name))

13
python/google/protobuf/internal/json_format_test.py
View File

@ -1157,6 +1157,19 @@ class JsonFormatTest(JsonFormatBase):
'Failed to parse any_value field: Can not find message descriptor by'
' type_url: type.googleapis.com/proto3.MessageType..')
def testParseDictUnknownValueType(self):
class UnknownClass(object):
def __str__(self):
return 'v'
message = json_format_proto3_pb2.TestValue()
self.assertRaisesRegexp(
json_format.ParseError,
r"Value v has unexpected type <class '.*\.UnknownClass'>.",
json_format.ParseDict,
{'value': UnknownClass()},
message)
def testMessageToDict(self):
message = json_format_proto3_pb2.TestMessage()
message.int32_value = 12345

3
python/google/protobuf/json_format.py
View File

@ -647,7 +647,8 @@ class _Parser(object):
elif isinstance(value, _INT_OR_FLOAT):
message.number_value = value
else:
raise ParseError('Unexpected type for Value message.')
raise ParseError('Value {0} has unexpected type {1}.'.format(
value, type(value)))
def _ConvertListValueMessage(self, value, message):
"""Convert a JSON representation into ListValue message."""

14
src/google/protobuf/compiler/cpp/cpp_enum.cc
View File

@ -180,14 +180,17 @@ void EnumGenerator::GenerateDefinition(io::Printer* printer) {
if (HasDescriptorMethods(descriptor_->file(), options_)) {
format(
"inline bool $classname$_Parse(\n"
" const std::string& name, $classname$* value) {\n"
" ::PROTOBUF_NAMESPACE_ID::ConstStringParam name, $classname$* "
"value) "
"{\n"
" return ::$proto_ns$::internal::ParseNamedEnum<$classname$>(\n"
" $classname$_descriptor(), name, value);\n"
"}\n");
} else {
format(
"bool $classname$_Parse(\n"
" const std::string& name, $classname$* value);\n");
" ::PROTOBUF_NAMESPACE_ID::ConstStringParam name, $classname$* "
"value);\n");
}
}
@ -253,7 +256,8 @@ void EnumGenerator::GenerateSymbolImports(io::Printer* printer) const {
" return $classname$_Name(enum_t_value);\n"
"}\n");
format(
"static inline bool $nested_name$_Parse(const std::string& name,\n"
"static inline bool "
"$nested_name$_Parse(::PROTOBUF_NAMESPACE_ID::ConstStringParam name,\n"
" $resolved_name$* value) {\n"
" return $classname$_Parse(name, value);\n"
"}\n");
@ -383,7 +387,9 @@ void EnumGenerator::GenerateMethods(int idx, io::Printer* printer) {
CountUniqueValues(descriptor_));
format(
"bool $classname$_Parse(\n"
" const std::string& name, $classname$* value) {\n"
" ::PROTOBUF_NAMESPACE_ID::ConstStringParam name, $classname$* "
"value) "
"{\n"
" int int_value;\n"
" bool success = ::$proto_ns$::internal::LookUpEnumValue(\n"
" $classname$_entries, $1$, name, &int_value);\n"

4
src/google/protobuf/compiler/java/java_helpers.cc
View File

@ -75,8 +75,8 @@ const char* kForbiddenWordList[] = {
"class",
};
const std::unordered_set<string>* kReservedNames =
new std::unordered_set<string>({
const std::unordered_set<std::string>* kReservedNames =
new std::unordered_set<std::string>({
"abstract", "assert", "boolean", "break", "byte",
"case", "catch", "char", "class", "const",
"continue", "default", "do", "double", "else",

2
src/google/protobuf/compiler/java/java_name_resolver.cc
View File

@ -91,7 +91,7 @@ std::string ClassNameWithoutPackage(const ServiceDescriptor* descriptor,
}
// Return true if a and b are equals (case insensitive).
NameEquality CheckNameEquality(const string& a, const string& b) {
NameEquality CheckNameEquality(const std::string& a, const std::string& b) {
if (ToUpper(a) == ToUpper(b)) {
if (a == b) {
return NameEquality::EXACT_EQUAL;

4
src/google/protobuf/compiler/plugin.pb.h
View File

@ -106,7 +106,7 @@ inline const std::string& CodeGeneratorResponse_Feature_Name(T enum_t_value) {
CodeGeneratorResponse_Feature_descriptor(), enum_t_value);
}
inline bool CodeGeneratorResponse_Feature_Parse(
const std::string& name, CodeGeneratorResponse_Feature* value) {
::PROTOBUF_NAMESPACE_ID::ConstStringParam name, CodeGeneratorResponse_Feature* value) {
return ::PROTOBUF_NAMESPACE_ID::internal::ParseNamedEnum<CodeGeneratorResponse_Feature>(
CodeGeneratorResponse_Feature_descriptor(), name, value);
}
@ -924,7 +924,7 @@ class PROTOC_EXPORT CodeGeneratorResponse PROTOBUF_FINAL :
"Incorrect type passed to function Feature_Name.");
return CodeGeneratorResponse_Feature_Name(enum_t_value);
}
static inline bool Feature_Parse(const std::string& name,
static inline bool Feature_Parse(::PROTOBUF_NAMESPACE_ID::ConstStringParam name,
Feature* value) {
return CodeGeneratorResponse_Feature_Parse(name, value);
}

2
src/google/protobuf/compiler/python/python_generator.cc
View File

@ -125,7 +125,7 @@ bool ContainsPythonKeyword(const std::string& module_name) {
return false;
}
inline bool IsPythonKeyword(const string& name) {
inline bool IsPythonKeyword(const std::string& name) {
return (std::find(kKeywords, kKeywordsEnd, name) != kKeywordsEnd);
}

43
src/google/protobuf/descriptor.cc
View File

@ -383,19 +383,8 @@ class PrefixRemover {
// hash-maps for each object.
//
// The keys to these hash-maps are (parent, name) or (parent, number) pairs.
//
// TODO(kenton): Use StringPiece rather than const char* in keys? It would
// be a lot cleaner but we'd just have to convert it back to const char*
// for the open source release.
typedef std::pair<const void*, const char*> PointerStringPair;
struct PointerStringPairEqual {
inline bool operator()(const PointerStringPair& a,
const PointerStringPair& b) const {
return a.first == b.first && strcmp(a.second, b.second) == 0;
}
};
typedef std::pair<const void*, StringPiece> PointerStringPair;
typedef std::pair<const Descriptor*, int> DescriptorIntPair;
typedef std::pair<const EnumDescriptor*, int> EnumIntPair;
@ -419,16 +408,16 @@ struct PointerIntegerPairHash {
static const size_t min_buckets = 8;
#endif
inline bool operator()(const PairType& a, const PairType& b) const {
return a.first < b.first || (a.first == b.first && a.second < b.second);
return a < b;
}
};
struct PointerStringPairHash {
size_t operator()(const PointerStringPair& p) const {
static const size_t prime = 16777619;
hash<const char*> cstring_hash;
hash<StringPiece> string_hash;
return reinterpret_cast<size_t>(p.first) * prime ^
static_cast<size_t>(cstring_hash(p.second));
static_cast<size_t>(string_hash(p.second));
}
#ifdef _MSC_VER
@ -438,9 +427,7 @@ struct PointerStringPairHash {
#endif
inline bool operator()(const PointerStringPair& a,
const PointerStringPair& b) const {
if (a.first < b.first) return true;
if (a.first > b.first) return false;
return strcmp(a.second, b.second) < 0;
return a < b;
}
};
@ -450,8 +437,7 @@ const Symbol kNullSymbol;
typedef HASH_MAP<const char*, Symbol, HASH_FXN<const char*>, streq>
SymbolsByNameMap;
typedef HASH_MAP<PointerStringPair, Symbol, PointerStringPairHash,
PointerStringPairEqual>
typedef HASH_MAP<PointerStringPair, Symbol, PointerStringPairHash>
SymbolsByParentMap;
typedef HASH_MAP<const char*, const FileDescriptor*, HASH_FXN<const char*>,
@ -459,7 +445,7 @@ typedef HASH_MAP<const char*, const FileDescriptor*, HASH_FXN<const char*>,
FilesByNameMap;
typedef HASH_MAP<PointerStringPair, const FieldDescriptor*,
PointerStringPairHash, PointerStringPairEqual>
PointerStringPairHash>
FieldsByNameMap;
typedef HASH_MAP<DescriptorIntPair, const FieldDescriptor*,
@ -720,9 +706,9 @@ class FileDescriptorTables {
// Find symbols. These return a null Symbol (symbol.IsNull() is true)
// if not found.
inline Symbol FindNestedSymbol(const void* parent,
const std::string& name) const;
StringPiece name) const;
inline Symbol FindNestedSymbolOfType(const void* parent,
const std::string& name,
StringPiece name,
const Symbol::Type type) const;
// These return nullptr if not found.
@ -908,9 +894,9 @@ inline Symbol DescriptorPool::Tables::FindSymbol(const std::string& key) const {
}
inline Symbol FileDescriptorTables::FindNestedSymbol(
const void* parent, const std::string& name) const {
const Symbol* result = FindOrNull(
symbols_by_parent_, PointerStringPair(parent, name.c_str()));
const void* parent, StringPiece name) const {
const Symbol* result =
FindOrNull(symbols_by_parent_, PointerStringPair(parent, name));
if (result == nullptr) {
return kNullSymbol;
} else {
@ -919,8 +905,7 @@ inline Symbol FileDescriptorTables::FindNestedSymbol(
}
inline Symbol FileDescriptorTables::FindNestedSymbolOfType(
const void* parent, const std::string& name,
const Symbol::Type type) const {
const void* parent, StringPiece name, const Symbol::Type type) const {
Symbol result = FindNestedSymbol(parent, name);
if (result.type != type) return kNullSymbol;
return result;
@ -1737,7 +1722,7 @@ const FieldDescriptor* Descriptor::map_value() const {
}
const EnumValueDescriptor* EnumDescriptor::FindValueByName(
const std::string& key) const {
ConstStringParam key) const {
Symbol result =
file()->tables_->FindNestedSymbolOfType(this, key, Symbol::ENUM_VALUE);
if (!result.IsNull()) {

3
src/google/protobuf/descriptor.h
View File

@ -63,6 +63,7 @@
#include <google/protobuf/stubs/common.h>
#include <google/protobuf/stubs/mutex.h>
#include <google/protobuf/stubs/once.h>
#include <google/protobuf/port.h>
#include <google/protobuf/port_def.inc>
// TYPE_BOOL is defined in the MacOS's ConditionalMacros.h.
@ -1029,7 +1030,7 @@ class PROTOBUF_EXPORT EnumDescriptor {
const EnumValueDescriptor* value(int index) const;
// Looks up a value by name. Returns nullptr if no such value exists.
const EnumValueDescriptor* FindValueByName(const std::string& name) const;
const EnumValueDescriptor* FindValueByName(ConstStringParam name) const;
// Looks up a value by number. Returns nullptr if no such value exists. If
// multiple values have this number, the first one defined is returned.
const EnumValueDescriptor* FindValueByNumber(int number) const;

24
src/google/protobuf/descriptor.pb.h
View File

@ -204,7 +204,7 @@ inline const std::string& FieldDescriptorProto_Type_Name(T enum_t_value) {
FieldDescriptorProto_Type_descriptor(), enum_t_value);
}
inline bool FieldDescriptorProto_Type_Parse(
const std::string& name, FieldDescriptorProto_Type* value) {
::PROTOBUF_NAMESPACE_ID::ConstStringParam name, FieldDescriptorProto_Type* value) {
return ::PROTOBUF_NAMESPACE_ID::internal::ParseNamedEnum<FieldDescriptorProto_Type>(
FieldDescriptorProto_Type_descriptor(), name, value);
}
@ -228,7 +228,7 @@ inline const std::string& FieldDescriptorProto_Label_Name(T enum_t_value) {
FieldDescriptorProto_Label_descriptor(), enum_t_value);
}
inline bool FieldDescriptorProto_Label_Parse(
const std::string& name, FieldDescriptorProto_Label* value) {
::PROTOBUF_NAMESPACE_ID::ConstStringParam name, FieldDescriptorProto_Label* value) {
return ::PROTOBUF_NAMESPACE_ID::internal::ParseNamedEnum<FieldDescriptorProto_Label>(
FieldDescriptorProto_Label_descriptor(), name, value);
}
@ -252,7 +252,7 @@ inline const std::string& FileOptions_OptimizeMode_Name(T enum_t_value) {
FileOptions_OptimizeMode_descriptor(), enum_t_value);
}
inline bool FileOptions_OptimizeMode_Parse(
const std::string& name, FileOptions_OptimizeMode* value) {
::PROTOBUF_NAMESPACE_ID::ConstStringParam name, FileOptions_OptimizeMode* value) {
return ::PROTOBUF_NAMESPACE_ID::internal::ParseNamedEnum<FileOptions_OptimizeMode>(
FileOptions_OptimizeMode_descriptor(), name, value);
}
@ -276,7 +276,7 @@ inline const std::string& FieldOptions_CType_Name(T enum_t_value) {
FieldOptions_CType_descriptor(), enum_t_value);
}
inline bool FieldOptions_CType_Parse(
const std::string& name, FieldOptions_CType* value) {
::PROTOBUF_NAMESPACE_ID::ConstStringParam name, FieldOptions_CType* value) {
return ::PROTOBUF_NAMESPACE_ID::internal::ParseNamedEnum<FieldOptions_CType>(
FieldOptions_CType_descriptor(), name, value);
}
@ -300,7 +300,7 @@ inline const std::string& FieldOptions_JSType_Name(T enum_t_value) {
FieldOptions_JSType_descriptor(), enum_t_value);
}
inline bool FieldOptions_JSType_Parse(
const std::string& name, FieldOptions_JSType* value) {
::PROTOBUF_NAMESPACE_ID::ConstStringParam name, FieldOptions_JSType* value) {
return ::PROTOBUF_NAMESPACE_ID::internal::ParseNamedEnum<FieldOptions_JSType>(
FieldOptions_JSType_descriptor(), name, value);
}
@ -324,7 +324,7 @@ inline const std::string& MethodOptions_IdempotencyLevel_Name(T enum_t_value) {
MethodOptions_IdempotencyLevel_descriptor(), enum_t_value);
}
inline bool MethodOptions_IdempotencyLevel_Parse(
const std::string& name, MethodOptions_IdempotencyLevel* value) {
::PROTOBUF_NAMESPACE_ID::ConstStringParam name, MethodOptions_IdempotencyLevel* value) {
return ::PROTOBUF_NAMESPACE_ID::internal::ParseNamedEnum<MethodOptions_IdempotencyLevel>(
MethodOptions_IdempotencyLevel_descriptor(), name, value);
}
@ -1936,7 +1936,7 @@ class PROTOBUF_EXPORT FieldDescriptorProto PROTOBUF_FINAL :
"Incorrect type passed to function Type_Name.");
return FieldDescriptorProto_Type_Name(enum_t_value);
}
static inline bool Type_Parse(const std::string& name,
static inline bool Type_Parse(::PROTOBUF_NAMESPACE_ID::ConstStringParam name,
Type* value) {
return FieldDescriptorProto_Type_Parse(name, value);
}
@ -1968,7 +1968,7 @@ class PROTOBUF_EXPORT FieldDescriptorProto PROTOBUF_FINAL :
"Incorrect type passed to function Label_Name.");
return FieldDescriptorProto_Label_Name(enum_t_value);
}
static inline bool Label_Parse(const std::string& name,
static inline bool Label_Parse(::PROTOBUF_NAMESPACE_ID::ConstStringParam name,
Label* value) {
return FieldDescriptorProto_Label_Parse(name, value);
}
@ -3668,7 +3668,7 @@ class PROTOBUF_EXPORT FileOptions PROTOBUF_FINAL :
"Incorrect type passed to function OptimizeMode_Name.");
return FileOptions_OptimizeMode_Name(enum_t_value);
}
static inline bool OptimizeMode_Parse(const std::string& name,
static inline bool OptimizeMode_Parse(::PROTOBUF_NAMESPACE_ID::ConstStringParam name,
OptimizeMode* value) {
return FileOptions_OptimizeMode_Parse(name, value);
}
@ -4534,7 +4534,7 @@ class PROTOBUF_EXPORT FieldOptions PROTOBUF_FINAL :
"Incorrect type passed to function CType_Name.");
return FieldOptions_CType_Name(enum_t_value);
}
static inline bool CType_Parse(const std::string& name,
static inline bool CType_Parse(::PROTOBUF_NAMESPACE_ID::ConstStringParam name,
CType* value) {
return FieldOptions_CType_Parse(name, value);
}
@ -4566,7 +4566,7 @@ class PROTOBUF_EXPORT FieldOptions PROTOBUF_FINAL :
"Incorrect type passed to function JSType_Name.");
return FieldOptions_JSType_Name(enum_t_value);
}
static inline bool JSType_Parse(const std::string& name,
static inline bool JSType_Parse(::PROTOBUF_NAMESPACE_ID::ConstStringParam name,
JSType* value) {
return FieldOptions_JSType_Parse(name, value);
}
@ -5532,7 +5532,7 @@ class PROTOBUF_EXPORT MethodOptions PROTOBUF_FINAL :
"Incorrect type passed to function IdempotencyLevel_Name.");
return MethodOptions_IdempotencyLevel_Name(enum_t_value);
}
static inline bool IdempotencyLevel_Parse(const std::string& name,
static inline bool IdempotencyLevel_Parse(::PROTOBUF_NAMESPACE_ID::ConstStringParam name,
IdempotencyLevel* value) {
return MethodOptions_IdempotencyLevel_Parse(name, value);
}

153
src/google/protobuf/descriptor_database.cc
View File

@ -41,6 +41,7 @@
#include <google/protobuf/stubs/map_util.h>
#include <google/protobuf/stubs/stl_util.h>
namespace google {
namespace protobuf {
@ -389,11 +390,14 @@ bool SimpleDescriptorDatabase::MaybeCopy(const FileDescriptorProto* file,
// -------------------------------------------------------------------
class EncodedDescriptorDatabase::DescriptorIndex {
using String = std::string;
public:
using Value = std::pair<const void*, int>;
// Helpers to recursively add particular descriptors and all their contents
// to the index.
bool AddFile(const FileDescriptorProto& file, Value value);
template <typename FileProto>
bool AddFile(const FileProto& file, Value value);
Value FindFile(StringPiece filename);
Value FindSymbol(StringPiece name);
@ -406,11 +410,15 @@ class EncodedDescriptorDatabase::DescriptorIndex {
private:
friend class EncodedDescriptorDatabase;
bool AddSymbol(StringPiece name, Value value);
bool AddSymbol(StringPiece package, StringPiece symbol,
Value value);
template <typename DescProto>
bool AddNestedExtensions(StringPiece filename,
const DescriptorProto& message_type, Value value);
bool AddExtension(StringPiece filename,
const FieldDescriptorProto& field, Value value);
const DescProto& message_type, Value value);
template <typename FieldProto>
bool AddExtension(StringPiece filename, const FieldProto& field,
Value value);
// All the maps below have two representations:
// - a std::set<> where we insert initially.
@ -421,27 +429,72 @@ class EncodedDescriptorDatabase::DescriptorIndex {
void EnsureFlat();
struct Entry {
std::string name;
struct FileEntry {
String name;
Value data;
};
struct Compare {
bool operator()(const Entry& a, const Entry& b) const {
struct FileCompare {
bool operator()(const FileEntry& a, const FileEntry& b) const {
return a.name < b.name;
}
bool operator()(const Entry& a, StringPiece b) const {
bool operator()(const FileEntry& a, StringPiece b) const {
return a.name < b;
}
bool operator()(StringPiece a, const Entry& b) const {
bool operator()(StringPiece a, const FileEntry& b) const {
return a < b.name;
}
};
std::set<Entry, Compare> by_name_;
std::vector<Entry> by_name_flat_;
std::set<Entry, Compare> by_symbol_;
std::vector<Entry> by_symbol_flat_;
std::set<FileEntry, FileCompare> by_name_;
std::vector<FileEntry> by_name_flat_;
struct SymbolEntry {
String package;
String symbol;
Value data;
std::string AsString() const {
return StrCat(package, package.empty() ? "" : ".", symbol);
}
};
struct SymbolCompare {
static std::string AsString(const SymbolEntry& entry) {
return entry.AsString();
}
static StringPiece AsString(StringPiece str) { return str; }
static std::pair<StringPiece, StringPiece> GetParts(
const SymbolEntry& entry) {
if (entry.package.empty()) return {entry.symbol, StringPiece{}};
return {entry.package, entry.symbol};
}
static std::pair<StringPiece, StringPiece> GetParts(
StringPiece str) {
return {str, {}};
}
template <typename T, typename U>
bool operator()(const T& lhs, const U& rhs) const {
auto lhs_parts = GetParts(lhs);
auto rhs_parts = GetParts(rhs);
// Fast path to avoid making the whole string for common cases.
if (int res =
lhs_parts.first.substr(0, rhs_parts.first.size())
.compare(rhs_parts.first.substr(0, lhs_parts.first.size()))) {
// If the packages already differ, exit early.
return res < 0;
} else if (lhs_parts.first.size() == rhs_parts.first.size()) {
return lhs_parts.second < rhs_parts.second;
}
return AsString(lhs) < AsString(rhs);
}
};
std::set<SymbolEntry, SymbolCompare> by_symbol_;
std::vector<SymbolEntry> by_symbol_flat_;
struct ExtensionEntry {
std::string extendee;
String extendee;
int extension_number;
Value data;
};
@ -535,34 +588,30 @@ bool EncodedDescriptorDatabase::FindAllExtensionNumbers(
return index_->FindAllExtensionNumbers(extendee_type, output);
}
bool EncodedDescriptorDatabase::DescriptorIndex::AddFile(
const FileDescriptorProto& file, Value value) {
if (!InsertIfNotPresent(&by_name_, Entry{file.name(), value}) ||
template <typename FileProto>
bool EncodedDescriptorDatabase::DescriptorIndex::AddFile(const FileProto& file,
Value value) {
if (!InsertIfNotPresent(&by_name_, FileEntry{file.name(), value}) ||
std::binary_search(by_name_flat_.begin(), by_name_flat_.end(),
file.name(), by_name_.key_comp())) {
GOOGLE_LOG(ERROR) << "File already exists in database: " << file.name();
return false;
}
// We must be careful here -- calling file.package() if file.has_package() is
// false could access an uninitialized static-storage variable if we are being
// run at startup time.
std::string path = file.has_package() ? file.package() : std::string();
if (!path.empty()) path += '.';
StringPiece package = file.package();
for (const auto& message_type : file.message_type()) {
if (!AddSymbol(path + message_type.name(), value)) return false;
if (!AddSymbol(package, message_type.name(), value)) return false;
if (!AddNestedExtensions(file.name(), message_type, value)) return false;
}
for (const auto& enum_type : file.enum_type()) {
if (!AddSymbol(path + enum_type.name(), value)) return false;
if (!AddSymbol(package, enum_type.name(), value)) return false;
}
for (const auto& extension : file.extension()) {
if (!AddSymbol(path + extension.name(), value)) return false;
if (!AddSymbol(package, extension.name(), value)) return false;
if (!AddExtension(file.name(), extension, value)) return false;
}
for (const auto& service : file.service()) {
if (!AddSymbol(path + service.name(), value)) return false;
if (!AddSymbol(package, service.name(), value)) return false;
}
return true;
@ -572,10 +621,10 @@ template <typename Iter, typename Iter2>
static bool CheckForMutualSubsymbols(StringPiece symbol_name, Iter* iter,
Iter2 end) {
if (*iter != end) {
if (IsSubSymbol((*iter)->name, symbol_name)) {
if (IsSubSymbol((*iter)->AsString(), symbol_name)) {
GOOGLE_LOG(ERROR) << "Symbol name \"" << symbol_name
<< "\" conflicts with the existing symbol \"" << (*iter)->name
<< "\".";
<< "\" conflicts with the existing symbol \""
<< (*iter)->AsString() << "\".";
return false;
}
@ -586,10 +635,10 @@ static bool CheckForMutualSubsymbols(StringPiece symbol_name, Iter* iter,
// to increment it.
++*iter;
if (*iter != end && IsSubSymbol(symbol_name, (*iter)->name)) {
if (*iter != end && IsSubSymbol(symbol_name, (*iter)->AsString())) {
GOOGLE_LOG(ERROR) << "Symbol name \"" << symbol_name
<< "\" conflicts with the existing symbol \"" << (*iter)->name
<< "\".";
<< "\" conflicts with the existing symbol \""
<< (*iter)->AsString() << "\".";
return false;
}
}
@ -597,28 +646,30 @@ static bool CheckForMutualSubsymbols(StringPiece symbol_name, Iter* iter,
}
bool EncodedDescriptorDatabase::DescriptorIndex::AddSymbol(
StringPiece name, Value value) {
StringPiece package, StringPiece symbol, Value value) {
SymbolEntry entry = {String(package), String(symbol), value};
std::string entry_as_string = entry.AsString();
// We need to make sure not to violate our map invariant.
// If the symbol name is invalid it could break our lookup algorithm (which
// relies on the fact that '.' sorts before all other characters that are
// valid in symbol names).
if (!ValidateSymbolName(name)) {
GOOGLE_LOG(ERROR) << "Invalid symbol name: " << name;
if (!ValidateSymbolName(package) || !ValidateSymbolName(symbol)) {
GOOGLE_LOG(ERROR) << "Invalid symbol name: " << entry_as_string;
return false;
}
Entry entry = {std::string(name), value};
auto iter = FindLastLessOrEqual(&by_symbol_, entry);
if (!CheckForMutualSubsymbols(name, &iter, by_symbol_.end())) {
if (!CheckForMutualSubsymbols(entry_as_string, &iter, by_symbol_.end())) {
return false;
}
// Same, but on by_symbol_flat_
auto flat_iter =
FindLastLessOrEqual(&by_symbol_flat_, name, by_symbol_.key_comp());
if (!CheckForMutualSubsymbols(name, &flat_iter, by_symbol_flat_.end())) {
FindLastLessOrEqual(&by_symbol_flat_, entry, by_symbol_.key_comp());
if (!CheckForMutualSubsymbols(entry_as_string, &flat_iter,
by_symbol_flat_.end())) {
return false;
}
@ -626,14 +677,14 @@ bool EncodedDescriptorDatabase::DescriptorIndex::AddSymbol(
// Insert the new symbol using the iterator as a hint, the new entry will
// appear immediately before the one the iterator is pointing at.
by_symbol_.insert(iter, std::move(entry));
by_symbol_.insert(iter, entry);
return true;
}
template <typename DescProto>
bool EncodedDescriptorDatabase::DescriptorIndex::AddNestedExtensions(
StringPiece filename, const DescriptorProto& message_type,
Value value) {
StringPiece filename, const DescProto& message_type, Value value) {
for (const auto& nested_type : message_type.nested_type()) {
if (!AddNestedExtensions(filename, nested_type, value)) return false;
}
@ -643,9 +694,9 @@ bool EncodedDescriptorDatabase::DescriptorIndex::AddNestedExtensions(
return true;
}
template <typename FieldProto>
bool EncodedDescriptorDatabase::DescriptorIndex::AddExtension(
StringPiece filename, const FieldDescriptorProto& field,
Value value) {
StringPiece filename, const FieldProto& field, Value value) {
if (!field.extendee().empty() && field.extendee()[0] == '.') {
// The extension is fully-qualified. We can use it as a lookup key in
// the by_symbol_ table.
@ -682,7 +733,7 @@ EncodedDescriptorDatabase::DescriptorIndex::FindSymbolOnlyFlat(
auto iter =
FindLastLessOrEqual(&by_symbol_flat_, name, by_symbol_.key_comp());
return iter != by_symbol_flat_.end() && IsSubSymbol(iter->name, name)
return iter != by_symbol_flat_.end() && IsSubSymbol(iter->AsString(), name)
? iter->data
: Value();
}
@ -740,11 +791,11 @@ void EncodedDescriptorDatabase::DescriptorIndex::FindAllFileNames(
output->resize(by_name_.size() + by_name_flat_.size());
int i = 0;
for (const auto& entry : by_name_) {
(*output)[i] = entry.name;
(*output)[i] = std::string(entry.name);
i++;
}
for (const auto& entry : by_name_flat_) {
(*output)[i] = entry.name;
(*output)[i] = std::string(entry.name);
i++;
}
}

4
src/google/protobuf/descriptor_database_unittest.cc
View File

@ -547,7 +547,7 @@ TEST(SimpleDescriptorDatabaseExtraTest, FindAllPackageNames) {
db.Add(f);
db.Add(b);
std::vector<string> packages;
std::vector<std::string> packages;
EXPECT_TRUE(db.FindAllPackageNames(&packages));
EXPECT_THAT(packages, ::testing::UnorderedElementsAre("foo", ""));
}
@ -567,7 +567,7 @@ TEST(SimpleDescriptorDatabaseExtraTest, FindAllMessageNames) {
db.Add(f);
db.Add(b);
std::vector<string> messages;
std::vector<std::string> messages;
EXPECT_TRUE(db.FindAllMessageNames(&messages));
EXPECT_THAT(messages, ::testing::UnorderedElementsAre("foo.Foo", "Bar"));
}

25
src/google/protobuf/dynamic_message.cc
View File

@ -336,26 +336,24 @@ class DynamicMessage : public Message {
}
const TypeInfo* type_info_;
Arena* const arena_;
mutable std::atomic<int> cached_byte_size_;
GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(DynamicMessage);
};
DynamicMessage::DynamicMessage(const TypeInfo* type_info)
: type_info_(type_info), arena_(NULL), cached_byte_size_(0) {
: type_info_(type_info), cached_byte_size_(0) {
SharedCtor(true);
}
DynamicMessage::DynamicMessage(const TypeInfo* type_info, Arena* arena)
: Message(arena),
type_info_(type_info),
arena_(arena),
cached_byte_size_(0) {
SharedCtor(true);
}
DynamicMessage::DynamicMessage(TypeInfo* type_info, bool lock_factory)
: type_info_(type_info), arena_(NULL), cached_byte_size_(0) {
: type_info_(type_info), cached_byte_size_(0) {
// The prototype in type_info has to be set before creating the prototype
// instance on memory. e.g., message Foo { map<int32, Foo> a = 1; }. When
// creating prototype for Foo, prototype of the map entry will also be
@ -386,7 +384,8 @@ void DynamicMessage::SharedCtor(bool lock_factory) {
}
if (type_info_->extensions_offset != -1) {
new (OffsetToPointer(type_info_->extensions_offset)) ExtensionSet(arena_);
new (OffsetToPointer(type_info_->extensions_offset))
ExtensionSet(GetArena());
}
for (int i = 0; i < descriptor->field_count(); i++) {
const FieldDescriptor* field = descriptor->field(i);
@ -400,7 +399,7 @@ void DynamicMessage::SharedCtor(bool lock_factory) {
if (!field->is_repeated()) { \
new (field_ptr) TYPE(field->default_value_##TYPE()); \
} else { \
new (field_ptr) RepeatedField<TYPE>(arena_); \
new (field_ptr) RepeatedField<TYPE>(GetArena()); \
} \
break;
@ -417,7 +416,7 @@ void DynamicMessage::SharedCtor(bool lock_factory) {
if (!field->is_repeated()) {
new (field_ptr) int(field->default_value_enum()->number());
} else {
new (field_ptr) RepeatedField<int>(arena_);
new (field_ptr) RepeatedField<int>(GetArena());
}
break;
@ -438,7 +437,7 @@ void DynamicMessage::SharedCtor(bool lock_factory) {
ArenaStringPtr* asp = new (field_ptr) ArenaStringPtr();
asp->UnsafeSetDefault(default_value);
} else {
new (field_ptr) RepeatedPtrField<std::string>(arena_);
new (field_ptr) RepeatedPtrField<std::string>(GetArena());
}
break;
}
@ -453,20 +452,20 @@ void DynamicMessage::SharedCtor(bool lock_factory) {
// when the constructor is called inside GetPrototype(), in which
// case we have already locked the factory.
if (lock_factory) {
if (arena_ != NULL) {
if (GetArena() != nullptr) {
new (field_ptr) DynamicMapField(
type_info_->factory->GetPrototype(field->message_type()),
arena_);
GetArena());
} else {
new (field_ptr) DynamicMapField(
type_info_->factory->GetPrototype(field->message_type()));
}
} else {
if (arena_ != NULL) {
if (GetArena() != nullptr) {
new (field_ptr)
DynamicMapField(type_info_->factory->GetPrototypeNoLock(
field->message_type()),
arena_);
GetArena());
} else {
new (field_ptr)
DynamicMapField(type_info_->factory->GetPrototypeNoLock(
@ -474,7 +473,7 @@ void DynamicMessage::SharedCtor(bool lock_factory) {
}
}
} else {
new (field_ptr) RepeatedPtrField<Message>(arena_);
new (field_ptr) RepeatedPtrField<Message>(GetArena());
}
}
break;

6
src/google/protobuf/dynamic_message_unittest.cc
View File

@ -252,7 +252,7 @@ TEST_P(DynamicMessageTest, Oneof) {
}
TEST_P(DynamicMessageTest, SpaceUsed) {
// Test that SpaceUsed() works properly
// Test that SpaceUsedLong() works properly
// Since we share the implementation with generated messages, we don't need
// to test very much here. Just make sure it appears to be working.
@ -261,10 +261,10 @@ TEST_P(DynamicMessageTest, SpaceUsed) {
Message* message = prototype_->New(GetParam() ? &arena : NULL);
TestUtil::ReflectionTester reflection_tester(descriptor_);
int initial_space_used = message->SpaceUsed();
size_t initial_space_used = message->SpaceUsedLong();
reflection_tester.SetAllFieldsViaReflection(message);
EXPECT_LT(initial_space_used, message->SpaceUsed());
EXPECT_LT(initial_space_used, message->SpaceUsedLong());
if (!GetParam()) {
delete message;

57
src/google/protobuf/extension_set_unittest.cc
View File

@ -514,7 +514,7 @@ TEST(ExtensionSetTest, SerializationToArray) {
unittest::TestAllExtensions source;
unittest::TestAllTypes destination;
TestUtil::SetAllExtensions(&source);
int size = source.ByteSize();
size_t size = source.ByteSizeLong();
std::string data;
data.resize(size);
uint8* target = reinterpret_cast<uint8*>(::google::protobuf::string_as_array(&data));
@ -535,7 +535,7 @@ TEST(ExtensionSetTest, SerializationToStream) {
unittest::TestAllExtensions source;
unittest::TestAllTypes destination;
TestUtil::SetAllExtensions(&source);
int size = source.ByteSize();
size_t size = source.ByteSizeLong();
std::string data;
data.resize(size);
{
@ -558,7 +558,7 @@ TEST(ExtensionSetTest, PackedSerializationToArray) {
unittest::TestPackedExtensions source;
unittest::TestPackedTypes destination;
TestUtil::SetPackedExtensions(&source);
int size = source.ByteSize();
size_t size = source.ByteSizeLong();
std::string data;
data.resize(size);
uint8* target = reinterpret_cast<uint8*>(::google::protobuf::string_as_array(&data));
@ -579,7 +579,7 @@ TEST(ExtensionSetTest, PackedSerializationToStream) {
unittest::TestPackedExtensions source;
unittest::TestPackedTypes destination;
TestUtil::SetPackedExtensions(&source);
int size = source.ByteSize();
size_t size = source.ByteSizeLong();
std::string data;
data.resize(size);
{
@ -739,12 +739,12 @@ TEST(ExtensionSetTest, SpaceUsedExcludingSelf) {
#define TEST_SCALAR_EXTENSIONS_SPACE_USED(type, value) \
do { \
unittest::TestAllExtensions message; \
const int base_size = message.SpaceUsed(); \
const int base_size = message.SpaceUsedLong(); \
message.SetExtension(unittest::optional_##type##_extension, value); \
int min_expected_size = \
base_size + \
sizeof(message.GetExtension(unittest::optional_##type##_extension)); \
EXPECT_LE(min_expected_size, message.SpaceUsed()); \
EXPECT_LE(min_expected_size, message.SpaceUsedLong()); \
} while (0)
TEST_SCALAR_EXTENSIONS_SPACE_USED(int32, 101);
@ -763,36 +763,36 @@ TEST(ExtensionSetTest, SpaceUsedExcludingSelf) {
#undef TEST_SCALAR_EXTENSIONS_SPACE_USED
{
unittest::TestAllExtensions message;
const int base_size = message.SpaceUsed();
const int base_size = message.SpaceUsedLong();
message.SetExtension(unittest::optional_nested_enum_extension,
unittest::TestAllTypes::FOO);
int min_expected_size =
base_size +
sizeof(message.GetExtension(unittest::optional_nested_enum_extension));
EXPECT_LE(min_expected_size, message.SpaceUsed());
EXPECT_LE(min_expected_size, message.SpaceUsedLong());
}
{
// Strings may cause extra allocations depending on their length; ensure
// that gets included as well.
unittest::TestAllExtensions message;
const int base_size = message.SpaceUsed();
const int base_size = message.SpaceUsedLong();
const std::string s(
"this is a fairly large string that will cause some "
"allocation in order to store it in the extension");
message.SetExtension(unittest::optional_string_extension, s);
int min_expected_size = base_size + s.length();
EXPECT_LE(min_expected_size, message.SpaceUsed());
EXPECT_LE(min_expected_size, message.SpaceUsedLong());
}
{
// Messages also have additional allocation that need to be counted.
unittest::TestAllExtensions message;
const int base_size = message.SpaceUsed();
const int base_size = message.SpaceUsedLong();
unittest::ForeignMessage foreign;
foreign.set_c(42);
message.MutableExtension(unittest::optional_foreign_message_extension)
->CopyFrom(foreign);
int min_expected_size = base_size + foreign.SpaceUsed();
EXPECT_LE(min_expected_size, message.SpaceUsed());
int min_expected_size = base_size + foreign.SpaceUsedLong();
EXPECT_LE(min_expected_size, message.SpaceUsedLong());
}
// Repeated primitive extensions will increase space used by at least a
@ -802,23 +802,23 @@ TEST(ExtensionSetTest, SpaceUsedExcludingSelf) {
// - Adds a value to the repeated extension, then clears it, establishing
// the base size.
// - Adds a small number of values, testing that it doesn't increase the
// SpaceUsed()
// SpaceUsedLong()
// - Adds a large number of values (requiring allocation in the repeated
// field), and ensures that that allocation is included in SpaceUsed()
// field), and ensures that that allocation is included in SpaceUsedLong()
#define TEST_REPEATED_EXTENSIONS_SPACE_USED(type, cpptype, value) \
do { \
unittest::TestAllExtensions message; \
const int base_size = message.SpaceUsed(); \
int min_expected_size = sizeof(RepeatedField<cpptype>) + base_size; \
const size_t base_size = message.SpaceUsedLong(); \
size_t min_expected_size = sizeof(RepeatedField<cpptype>) + base_size; \
message.AddExtension(unittest::repeated_##type##_extension, value); \
message.ClearExtension(unittest::repeated_##type##_extension); \
const int empty_repeated_field_size = message.SpaceUsed(); \
const size_t empty_repeated_field_size = message.SpaceUsedLong(); \
EXPECT_LE(min_expected_size, empty_repeated_field_size) << #type; \
message.AddExtension(unittest::repeated_##type##_extension, value); \
message.AddExtension(unittest::repeated_##type##_extension, value); \
EXPECT_EQ(empty_repeated_field_size, message.SpaceUsed()) << #type; \
EXPECT_EQ(empty_repeated_field_size, message.SpaceUsedLong()) << #type; \
message.ClearExtension(unittest::repeated_##type##_extension); \
const int old_capacity = \
const size_t old_capacity = \
message.GetRepeatedExtension(unittest::repeated_##type##_extension) \
.Capacity(); \
EXPECT_GE(old_capacity, kRepeatedFieldLowerClampLimit); \
@ -832,7 +832,7 @@ TEST(ExtensionSetTest, SpaceUsedExcludingSelf) {
.Capacity() - \
old_capacity) + \
empty_repeated_field_size; \
EXPECT_LE(expected_size, message.SpaceUsed()) << #type; \
EXPECT_LE(expected_size, message.SpaceUsedLong()) << #type; \
} while (0)
TEST_REPEATED_EXTENSIONS_SPACE_USED(int32, int32, 101);
@ -854,8 +854,9 @@ TEST(ExtensionSetTest, SpaceUsedExcludingSelf) {
// Repeated strings
{
unittest::TestAllExtensions message;
const int base_size = message.SpaceUsed();
int min_expected_size = sizeof(RepeatedPtrField<std::string>) + base_size;
const size_t base_size = message.SpaceUsedLong();
size_t min_expected_size =
sizeof(RepeatedPtrField<std::string>) + base_size;
const std::string value(256, 'x');
// Once items are allocated, they may stick around even when cleared so
// without the hardcore memory management accessors there isn't a notion of
@ -865,13 +866,13 @@ TEST(ExtensionSetTest, SpaceUsedExcludingSelf) {
}
min_expected_size +=
(sizeof(value) + value.size()) * (16 - kRepeatedFieldLowerClampLimit);
EXPECT_LE(min_expected_size, message.SpaceUsed());
EXPECT_LE(min_expected_size, message.SpaceUsedLong());
}
// Repeated messages
{
unittest::TestAllExtensions message;
const int base_size = message.SpaceUsed();
int min_expected_size =
const size_t base_size = message.SpaceUsedLong();
size_t min_expected_size =
sizeof(RepeatedPtrField<unittest::ForeignMessage>) + base_size;
unittest::ForeignMessage prototype;
prototype.set_c(2);
@ -880,8 +881,8 @@ TEST(ExtensionSetTest, SpaceUsedExcludingSelf) {
->CopyFrom(prototype);
}
min_expected_size +=
(16 - kRepeatedFieldLowerClampLimit) * prototype.SpaceUsed();
EXPECT_LE(min_expected_size, message.SpaceUsed());
(16 - kRepeatedFieldLowerClampLimit) * prototype.SpaceUsedLong();
EXPECT_LE(min_expected_size, message.SpaceUsedLong());
}
}

5
src/google/protobuf/generated_enum_reflection.h
View File

@ -43,6 +43,7 @@
#include <google/protobuf/generated_enum_util.h>
#include <google/protobuf/port.h>
#include <google/protobuf/stubs/strutil.h>
#ifdef SWIG
#error "You cannot SWIG proto headers"
@ -71,10 +72,10 @@ namespace internal {
// an enum name of the given type, returning true and filling in value on
// success, or returning false and leaving value unchanged on failure.
PROTOBUF_EXPORT bool ParseNamedEnum(const EnumDescriptor* descriptor,
const std::string& name, int* value);
StringPiece name, int* value);
template <typename EnumType>
bool ParseNamedEnum(const EnumDescriptor* descriptor, const std::string& name,
bool ParseNamedEnum(const EnumDescriptor* descriptor, StringPiece name,
EnumType* value) {
int tmp;
if (!ParseNamedEnum(descriptor, name, &tmp)) return false;

3
src/google/protobuf/generated_message_reflection.cc
View File

@ -50,6 +50,7 @@
#include <google/protobuf/repeated_field.h>
#include <google/protobuf/unknown_field_set.h>
#include <google/protobuf/wire_format.h>
#include <google/protobuf/stubs/strutil.h>
#include <google/protobuf/port_def.inc>
@ -81,7 +82,7 @@ bool IsMapFieldInApi(const FieldDescriptor* field) { return field->is_map(); }
namespace internal {
bool ParseNamedEnum(const EnumDescriptor* descriptor, const std::string& name,
bool ParseNamedEnum(const EnumDescriptor* descriptor, StringPiece name,
int* value) {
const EnumValueDescriptor* d = descriptor->FindValueByName(name);
if (d == nullptr) return false;

4
src/google/protobuf/lite_unittest.cc
View File

@ -613,7 +613,7 @@ TEST(Lite, AllLite28) {
protobuf_unittest::TestMapLite message1, message2;
std::string data;
MapLiteTestUtil::SetMapFields(&message1);
int size = message1.ByteSize();
size_t size = message1.ByteSizeLong();
data.resize(size);
::google::protobuf::uint8* start = reinterpret_cast<::google::protobuf::uint8*>(::google::protobuf::string_as_array(&data));
::google::protobuf::uint8* end = message1.SerializeWithCachedSizesToArray(start);
@ -630,7 +630,7 @@ TEST(Lite, AllLite29) {
// Test the generated SerializeWithCachedSizes()
protobuf_unittest::TestMapLite message1, message2;
MapLiteTestUtil::SetMapFields(&message1);
int size = message1.ByteSize();
size_t size = message1.ByteSizeLong();
std::string data;
data.resize(size);
{

178
src/google/protobuf/map.h
View File

@ -42,9 +42,14 @@
#include <iterator>
#include <limits> // To support Visual Studio 2008
#include <map>
#include <string>
#include <type_traits>
#include <utility>
#if defined(__cpp_lib_string_view)
#include <string_view>
#endif // defined(__cpp_lib_string_view)
#include <google/protobuf/stubs/common.h>
#include <google/protobuf/arena.h>
#include <google/protobuf/generated_enum_util.h>
@ -186,6 +191,67 @@ using KeyForTree =
typename std::conditional<std::is_scalar<T>::value, T,
std::reference_wrapper<const T>>::type;
// Default case: Not transparent.
// We use std::hash<key_type>/std::less<key_type> and all the lookup functions
// only accept `key_type`.
template <typename key_type>
struct TransparentSupport {
using hash = std::hash<key_type>;
using less = std::less<key_type>;
static bool Equals(const key_type& a, const key_type& b) { return a == b; }
template <typename K>
using key_arg = key_type;
};
#if defined(__cpp_lib_string_view)
// If std::string_view is available, we add transparent support for std::string
// keys. We use std::hash<std::string_view> as it supports the input types we
// care about. The lookup functions accept arbitrary `K`. This will include any
// key type that is convertible to std::string_view.
template <>
struct TransparentSupport<std::string> {
static std::string_view ImplicitConvert(std::string_view str) { return str; }
// If the element is not convertible to std::string_view, try to convert to
// std::string first.
// The template makes this overload lose resolution when both have the same
// rank otherwise.
template <typename = void>
static std::string_view ImplicitConvert(const std::string& str) {
return str;
}
struct hash : private std::hash<std::string_view> {
using is_transparent = void;
template <typename T>
size_t operator()(const T& str) const {
return base()(ImplicitConvert(str));
}
private:
const std::hash<std::string_view>& base() const { return *this; }
};
struct less {
using is_transparent = void;
template <typename T, typename U>
bool operator()(const T& t, const U& u) const {
return ImplicitConvert(t) < ImplicitConvert(u);
}
};
template <typename T, typename U>
static bool Equals(const T& t, const U& u) {
return ImplicitConvert(t) == ImplicitConvert(u);
}
template <typename K>
using key_arg = K;
};
#endif // defined(__cpp_lib_string_view)
} // namespace internal
// This is the class for Map's internal value_type. Instead of using
@ -240,7 +306,7 @@ class Map {
using const_reference = const value_type&;
using size_type = size_t;
using hasher = std::hash<Key>;
using hasher = typename internal::TransparentSupport<Key>::hash;
Map() : arena_(nullptr), default_enum_value_(0) { Init(); }
explicit Map(Arena* arena) : arena_(arena), default_enum_value_(0) { Init(); }
@ -357,7 +423,8 @@ class Map {
// class per key type.
using TreeAllocator = typename Allocator::template rebind<
std::pair<const internal::KeyForTree<Key>, void*>>::other;
using Tree = std::map<internal::KeyForTree<Key>, void*, std::less<Key>,
using Tree = std::map<internal::KeyForTree<Key>, void*,
typename internal::TransparentSupport<Key>::less,
TreeAllocator>;
using TreeIterator = typename Tree::iterator;
@ -541,9 +608,10 @@ class Map {
size_type size() const { return num_elements_; }
bool empty() const { return size() == 0; }
iterator find(const Key& k) { return iterator(FindHelper(k).first); }
const_iterator find(const Key& k) const { return find(k, nullptr); }
bool contains(const Key& k) const { return find(k) != end(); }
template <typename K>
iterator find(const K& k) {
return iterator(FindHelper(k).first);
}
// Insert the key into the map, if not present. In that case, the value will
// be value initialized.
@ -611,16 +679,18 @@ class Map {
const_iterator find(const Key& k, TreeIterator* it) const {
return FindHelper(k, it).first;
}
std::pair<const_iterator, size_type> FindHelper(const Key& k) const {
template <typename K>
std::pair<const_iterator, size_type> FindHelper(const K& k) const {
return FindHelper(k, nullptr);
}
std::pair<const_iterator, size_type> FindHelper(const Key& k,
template <typename K>
std::pair<const_iterator, size_type> FindHelper(const K& k,
TreeIterator* it) const {
size_type b = BucketNumber(k);
if (TableEntryIsNonEmptyList(b)) {
Node* node = static_cast<Node*>(table_[b]);
do {
if (IsMatch(node->kv.first, k)) {
if (internal::TransparentSupport<Key>::Equals(node->kv.first, k)) {
return std::make_pair(const_iterator(node, this, b), b);
} else {
node = node->next;
@ -675,9 +745,29 @@ class Map {
return result;
}
// Returns whether we should insert after the head of the list. For
// non-optimized builds, we randomly decide whether to insert right at the
// head of the list or just after the head. This helps add a little bit of
// non-determinism to the map ordering.
bool ShouldInsertAfterHead(void* node) {
#ifdef NDEBUG
return false;
#else
// Doing modulo with a prime mixes the bits more.
return (reinterpret_cast<uintptr_t>(node) ^ seed_) % 13 > 6;
#endif
}
// Helper for InsertUnique. Handles the case where bucket b is a
// not-too-long linked list.
iterator InsertUniqueInList(size_type b, Node* node) {
if (table_[b] != nullptr && ShouldInsertAfterHead(node)) {
Node* first = static_cast<Node*>(table_[b]);
node->next = first->next;
first->next = node;
return iterator(node, this, b);
}
node->next = static_cast<Node*>(table_[b]);
table_[b] = static_cast<void*>(node);
return iterator(node, this, b);
@ -767,7 +857,7 @@ class Map {
Tree* tree = static_cast<Tree*>(table[index]);
typename Tree::iterator tree_it = tree->begin();
do {
InsertUnique(BucketNumber(tree_it->first),
InsertUnique(BucketNumber(std::cref(tree_it->first).get()),
NodeFromTreeIterator(tree_it));
} while (++tree_it != tree->end());
DestroyTree(tree);
@ -848,12 +938,18 @@ class Map {
return count >= kMaxLength;
}
size_type BucketNumber(const Key& k) const {
size_type h = hash_function()(k);
return (h + seed_) & (num_buckets_ - 1);
}
template <typename K>
size_type BucketNumber(const K& k) const {
// We xor the hash value against the random seed so that we effectively
// have a random hash function.
uint64 h = hash_function()(k) ^ seed_;
bool IsMatch(const Key& k0, const Key& k1) const { return k0 == k1; }
// We use the multiplication method to determine the bucket number from
// the hash value. The constant kPhi (suggested by Knuth) is roughly
// (sqrt(5) - 1) / 2 * 2^64.
constexpr uint64 kPhi = uint64{0x9e3779b97f4a7c15};
return ((kPhi * h) >> 32) & (num_buckets_ - 1);
}
// Return a power of two no less than max(kMinTableSize, n).
// Assumes either n < kMinTableSize or n is a power of two.
@ -899,7 +995,10 @@ class Map {
// Return a randomish value.
size_type Seed() const {
size_type s = static_cast<size_type>(reinterpret_cast<uintptr_t>(this));
// We get a little bit of randomness from the address of the map. The
// lower bits are not very random, due to alignment, so we discard them
// and shift the higher bits into their place.
size_type s = reinterpret_cast<uintptr_t>(this) >> 12;
#if defined(__x86_64__) && defined(__GNUC__) && \
!defined(GOOGLE_PROTOBUF_NO_RDTSC)
uint32 hi, lo;
@ -922,6 +1021,10 @@ class Map {
GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(InnerMap);
}; // end of class InnerMap
template <typename LookupKey>
using key_arg = typename internal::TransparentSupport<
key_type>::template key_arg<LookupKey>;
public:
// Iterators
class const_iterator {
@ -1014,30 +1117,44 @@ class Map {
// Element access
T& operator[](const key_type& key) { return (*elements_)[key].second; }
const T& at(const key_type& key) const {
template <typename K = key_type>
const T& at(const key_arg<K>& key) const {
const_iterator it = find(key);
GOOGLE_CHECK(it != end()) << "key not found: " << key;
GOOGLE_CHECK(it != end()) << "key not found: " << static_cast<Key>(key);
return it->second;
}
T& at(const key_type& key) {
template <typename K = key_type>
T& at(const key_arg<K>& key) {
iterator it = find(key);
GOOGLE_CHECK(it != end()) << "key not found: " << key;
GOOGLE_CHECK(it != end()) << "key not found: " << static_cast<Key>(key);
return it->second;
}
// Lookup
size_type count(const key_type& key) const {
const_iterator it = find(key);
GOOGLE_DCHECK(it == end() || key == it->first);
return it == end() ? 0 : 1;
template <typename K = key_type>
size_type count(const key_arg<K>& key) const {
return find(key) == end() ? 0 : 1;
}
const_iterator find(const key_type& key) const {
template <typename K = key_type>
const_iterator find(const key_arg<K>& key) const {
return const_iterator(iterator(elements_->find(key)));
}
iterator find(const key_type& key) { return iterator(elements_->find(key)); }
bool contains(const Key& key) const { return elements_->contains(key); }
template <typename K = key_type>
iterator find(const key_arg<K>& key) {
return iterator(elements_->find(key));
}
template <typename K = key_type>
bool contains(const key_arg<K>& key) const {
return find(key) != end();
}
template <typename K = key_type>
std::pair<const_iterator, const_iterator> equal_range(
const key_type& key) const {
const key_arg<K>& key) const {
const_iterator it = find(key);
if (it == end()) {
return std::pair<const_iterator, const_iterator>(it, it);
@ -1046,7 +1163,9 @@ class Map {
return std::pair<const_iterator, const_iterator>(begin, it);
}
}
std::pair<iterator, iterator> equal_range(const key_type& key) {
template <typename K = key_type>
std::pair<iterator, iterator> equal_range(const key_arg<K>& key) {
iterator it = find(key);
if (it == end()) {
return std::pair<iterator, iterator>(it, it);
@ -1079,7 +1198,8 @@ class Map {
}
// Erase and clear
size_type erase(const key_type& key) {
template <typename K = key_type>
size_type erase(const key_arg<K>& key) {
iterator it = find(key);
if (it == end()) {
return 0;

200
src/google/protobuf/map_test.cc
View File

@ -980,6 +980,107 @@ TEST_F(MapImplTest, CopyAssignMapIterator) {
EXPECT_EQ(it1.GetKey().GetInt32Value(), it2.GetKey().GetInt32Value());
}
// Attempts to verify that a map with keys a and b has a random ordering. This
// function returns true if it succeeds in observing both possible orderings.
bool MapOrderingIsRandom(int a, int b) {
bool saw_a_first = false;
bool saw_b_first = false;
for (int i = 0; i < 50; ++i) {
Map<int32, int32> m;
m[a] = 0;
m[b] = 0;
int32 first_element = m.begin()->first;
if (first_element == a) saw_a_first = true;
if (first_element == b) saw_b_first = true;
if (saw_a_first && saw_b_first) {
return true;
}
}
return false;
}
// This test verifies that the iteration order is reasonably random even for
// small maps. Currently we only have sufficient randomness for debug builds and
// builds where we can use the RDTSC instruction, so we only test for those
// builds.
#if defined(__x86_64__) && defined(__GNUC__) && \
!defined(GOOGLE_PROTOBUF_NO_RDTSC)
TEST_F(MapImplTest, RandomOrdering) {
for (int i = 0; i < 10; ++i) {
for (int j = i + 1; j < 10; ++j) {
EXPECT_TRUE(MapOrderingIsRandom(i, j))
<< "Map with keys " << i << " and " << j
<< " has deterministic ordering";
}
}
}
#endif
template <typename Key>
void TestTransparent(const Key& key, const Key& miss_key) {
Map<std::string, int> m;
const auto& cm = m;
m.insert({"ABC", 1});
const auto abc_it = m.begin();
m.insert({"DEF", 2});
using testing::Pair;
using testing::UnorderedElementsAre;
EXPECT_EQ(m.at(key), 1);
EXPECT_EQ(cm.at(key), 1);
#ifdef PROTOBUF_HAS_DEATH_TEST
EXPECT_DEATH(m.at(miss_key), "");
EXPECT_DEATH(cm.at(miss_key), "");
#endif // PROTOBUF_HAS_DEATH_TEST
EXPECT_EQ(m.count(key), 1);
EXPECT_EQ(cm.count(key), 1);
EXPECT_EQ(m.count(miss_key), 0);
EXPECT_EQ(cm.count(miss_key), 0);
EXPECT_EQ(m.find(key), abc_it);
EXPECT_EQ(cm.find(key), abc_it);
EXPECT_EQ(m.find(miss_key), m.end());
EXPECT_EQ(cm.find(miss_key), cm.end());
EXPECT_TRUE(m.contains(key));
EXPECT_TRUE(cm.contains(key));
EXPECT_FALSE(m.contains(miss_key));
EXPECT_FALSE(cm.contains(miss_key));
EXPECT_THAT(m.equal_range(key), Pair(abc_it, std::next(abc_it)));
EXPECT_THAT(cm.equal_range(key), Pair(abc_it, std::next(abc_it)));
EXPECT_THAT(m.equal_range(miss_key), Pair(m.end(), m.end()));
EXPECT_THAT(cm.equal_range(miss_key), Pair(m.end(), m.end()));
EXPECT_THAT(m, UnorderedElementsAre(Pair("ABC", 1), Pair("DEF", 2)));
EXPECT_EQ(m.erase(key), 1);
EXPECT_THAT(m, UnorderedElementsAre(Pair("DEF", 2)));
EXPECT_EQ(m.erase(key), 0);
EXPECT_EQ(m.erase(miss_key), 0);
EXPECT_THAT(m, UnorderedElementsAre(Pair("DEF", 2)));
}
TEST_F(MapImplTest, TransparentLookupForString) {
TestTransparent("ABC", "LKJ");
TestTransparent(std::string("ABC"), std::string("LKJ"));
#if defined(__cpp_lib_string_view)
TestTransparent(std::string_view("ABC"), std::string_view("LKJ"));
#endif // defined(__cpp_lib_string_view)
// std::reference_wrapper
std::string abc = "ABC", lkj = "LKJ";
TestTransparent(std::ref(abc), std::ref(lkj));
TestTransparent(std::cref(abc), std::cref(lkj));
}
// Map Field Reflection Test ========================================
static int Func(int i, int j) { return i * j; }
@ -2408,45 +2509,45 @@ TEST(GeneratedMapFieldTest, IsInitialized) {
TEST(GeneratedMapFieldTest, MessagesMustMerge) {
unittest::TestRequiredMessageMap map_message;
unittest::TestRequired with_dummy4;
with_dummy4.set_a(97);
with_dummy4.set_b(0);
with_dummy4.set_c(0);
with_dummy4.set_b(91);
with_dummy4.set_dummy4(98);
EXPECT_TRUE(with_dummy4.IsInitialized());
EXPECT_FALSE(with_dummy4.IsInitialized());
(*map_message.mutable_map_field())[0] = with_dummy4;
EXPECT_TRUE(map_message.IsInitialized());
std::string s = map_message.SerializeAsString();
EXPECT_FALSE(map_message.IsInitialized());
// Modify s so that there are two values in the entry for key 0.
// The first will have no value for c. The second will have no value for a.
// Those are required fields. Also, make some other little changes, to
// ensure we are merging the two values (because they're messages).
ASSERT_EQ(s.size() - 2, s[1]); // encoding of the length of what follows
std::string encoded_val(s.data() + 4, s.data() + s.size());
// In s, change the encoding of c to an encoding of dummy32.
s[s.size() - 3] -= 8;
// Make encoded_val slightly different from what's in s.
encoded_val[encoded_val.size() - 1] += 33; // Encode c = 33.
for (int i = 0; i < encoded_val.size(); i++) {
if (encoded_val[i] == 97) {
// Encode b = 91 instead of a = 97. But this won't matter, because
// we also encode b = 0 right after this. The point is to leave out
// a required field, and make sure the parser doesn't complain, because
// every required field is set after the merge of the two values.
encoded_val[i - 1] += 16;
encoded_val[i] = 91;
} else if (encoded_val[i] == 98) {
// Encode dummy5 = 99 instead of dummy4 = 98.
encoded_val[i - 1] += 8; // The tag for dummy5 is 8 more.
encoded_val[i]++;
break;
}
}
unittest::TestRequired with_dummy5;
with_dummy5.set_b(0);
with_dummy5.set_c(33);
with_dummy5.set_dummy5(99);
EXPECT_FALSE(with_dummy5.IsInitialized());
(*map_message.mutable_map_field())[0] = with_dummy5;
EXPECT_FALSE(map_message.IsInitialized());
s += encoded_val; // Add the second message.
s[1] += encoded_val.size(); // Adjust encoded size.
// The wire format of MapEntry is straightforward (*) and can be manually
// constructed to force merging of two uninitialized messages that would
// result in an initialized message.
//
// (*) http://google3/net/proto2/internal/map_test.cc?l=2433&rcl=310012028
std::string dummy4_s = with_dummy4.SerializePartialAsString();
std::string dummy5_s = with_dummy5.SerializePartialAsString();
int payload_size = dummy4_s.size() + dummy5_s.size();
// Makes sure the payload size fits into one byte.
ASSERT_LT(payload_size, 128);
std::string s(6, 0);
char* p = &s[0];
*p++ = WireFormatLite::MakeTag(1, WireFormatLite::WIRETYPE_LENGTH_DELIMITED);
// Length: 2B for key tag & val and 2B for val tag and length of the following
// payload.
*p++ = 4 + payload_size;
*p++ = WireFormatLite::MakeTag(1, WireFormatLite::WIRETYPE_VARINT);
*p++ = 0;
*p++ = WireFormatLite::MakeTag(2, WireFormatLite::WIRETYPE_LENGTH_DELIMITED);
*p++ = payload_size;
StrAppend(&s, dummy4_s, dummy5_s);
// Test key then value then value.
int key = 0;
@ -2983,10 +3084,9 @@ TEST(WireFormatForMapFieldTest, SerializeMap) {
ASSERT_FALSE(output.HadError());
}
// Should be the same.
// Don't use EXPECT_EQ here because we're comparing raw binary data and
// we really don't want it dumped to stdout on failure.
EXPECT_TRUE(dynamic_data == generated_data);
// Should parse to the same message.
EXPECT_TRUE(TestUtil::EqualsToSerialized(message, generated_data));
EXPECT_TRUE(TestUtil::EqualsToSerialized(message, dynamic_data));
}
TEST(WireFormatForMapFieldTest, SerializeMapDynamicMessage) {
@ -3161,20 +3261,6 @@ static std::string DeterministicSerialization(const T& t) {
return result;
}
// Helper to test the serialization of the first arg against a golden file.
static void TestDeterministicSerialization(const protobuf_unittest::TestMaps& t,
const std::string& filename) {
std::string expected;
GOOGLE_CHECK_OK(File::GetContents(
TestUtil::GetTestDataPath("net/proto2/internal/testdata/" + filename),
&expected, true));
const std::string actual = DeterministicSerialization(t);
EXPECT_EQ(expected, actual);
protobuf_unittest::TestMaps u;
EXPECT_TRUE(u.ParseFromString(actual));
EXPECT_TRUE(util::MessageDifferencer::Equals(u, t));
}
// Helper for MapSerializationTest. Return a 7-bit ASCII string.
static std::string ConstructKey(uint64 n) {
std::string s(n % static_cast<uint64>(9), '\0');
@ -3219,7 +3305,17 @@ TEST(MapSerializationTest, Deterministic) {
frog = frog * multiplier + i;
frog ^= (frog >> 41);
}
TestDeterministicSerialization(t, "golden_message_maps");
// Verifies if two consecutive calls to deterministic serialization produce
// the same bytes. Deterministic serialization means the same serialization
// bytes in the same binary.
const std::string s1 = DeterministicSerialization(t);
const std::string s2 = DeterministicSerialization(t);
EXPECT_EQ(s1, s2);
protobuf_unittest::TestMaps u;
EXPECT_TRUE(u.ParseFromString(s1));
EXPECT_TRUE(util::MessageDifferencer::Equals(u, t));
}
TEST(MapSerializationTest, DeterministicSubmessage) {

2
src/google/protobuf/map_test_util_impl.h
View File

@ -405,7 +405,7 @@ void MapTestUtilImpl::ExpectMapFieldsSetInitialized(const MapMessage& message) {
EXPECT_EQ("", message.map_string_string().at("0"));
EXPECT_EQ("", message.map_int32_bytes().at(0));
EXPECT_EQ(enum_value, message.map_int32_enum().at(0));
EXPECT_EQ(0, message.map_int32_foreign_message().at(0).ByteSize());
EXPECT_EQ(0, message.map_int32_foreign_message().at(0).ByteSizeLong());
}
template <typename EnumType, EnumType enum_value0, EnumType enum_value1,

2
src/google/protobuf/no_field_presence_test.cc
View File

@ -567,7 +567,7 @@ TEST(NoFieldPresenceTest, OneofPresence) {
message.Clear();
message.set_oneof_string("test");
message.clear_oneof_string();
EXPECT_EQ(0, message.ByteSize());
EXPECT_EQ(0, message.ByteSizeLong());
}
} // namespace

10
src/google/protobuf/port_def.inc
View File

@ -662,3 +662,13 @@ PROTOBUF_EXPORT_TEMPLATE_TEST(DEFAULT, __declspec(dllimport));
#else
#define PROTOBUF_THREAD_LOCAL __thread
#endif
// For enabling message owned arena, one major blocker is semantic change from
// moving to copying when there is ownership transfer (e.g., move ctor, swap,
// set allocated, release). This change not only causes performance regression
// but also breaks users code (e.g., dangling reference). For top-level
// messages, since it owns the arena, we can mitigate the issue by transferring
// ownership of arena. However, we cannot do that for nested messages. In order
// to tell how many usages of nested messages affected by message owned arena,
// we need to simulate the arena ownership.
#define PROTOBUF_MESSAGE_OWNED_ARENA_EXPERIMENT

1
src/google/protobuf/port_undef.inc
View File

@ -84,6 +84,7 @@
#undef PROTOBUF_EXPORT_TEMPLATE_STYLE_MATCH_DECLSPEC_dllimport
#undef PROTOBUF_FINAL
#undef PROTOBUF_THREAD_LOCAL
#undef PROTOBUF_MESSAGE_OWNED_ARENA_EXPERIMENT
// Restore macro that may have been #undef'd in port_def.inc.
#ifdef _MSC_VER

4
src/google/protobuf/preserve_unknown_enum_test.cc
View File

@ -137,7 +137,7 @@ TEST(PreserveUnknownEnumTest, Proto2HidesUnknownValues) {
// The intermediate message has everything in its "unknown fields".
proto2_preserve_unknown_enum_unittest::MyMessage message2 = message;
message2.DiscardUnknownFields();
EXPECT_EQ(0, message2.ByteSize());
EXPECT_EQ(0, message2.ByteSizeLong());
// But when we pass it to the correct structure, all values are there.
serialized.clear();
@ -165,7 +165,7 @@ TEST(PreserveUnknownEnumTest, DynamicProto2HidesUnknownValues) {
proto2_preserve_unknown_enum_unittest::MyMessage message2;
message2.CopyFrom(*message);
message2.DiscardUnknownFields();
EXPECT_EQ(0, message2.ByteSize());
EXPECT_EQ(0, message2.ByteSizeLong());
// But when we pass it to the correct structure, all values are there.
serialized.clear();

6
src/google/protobuf/proto3_arena_unittest.cc
View File

@ -138,7 +138,7 @@ TEST(Proto3ArenaTest, UnknownFields) {
// We can modify this UnknownFieldSet.
unknown_fields->AddVarint(1, 2);
// And the unknown fields should be changed.
ASSERT_NE(original.ByteSize(), arena_message->ByteSize());
ASSERT_NE(original.ByteSizeLong(), arena_message->ByteSizeLong());
ASSERT_FALSE(
arena_message->GetReflection()->GetUnknownFields(*arena_message).empty());
}
@ -203,7 +203,7 @@ TEST(Proto3OptionalTest, OptionalFields) {
msg.set_optional_int32(0);
EXPECT_TRUE(msg.has_optional_int32());
string serialized;
std::string serialized;
msg.SerializeToString(&serialized);
EXPECT_GT(serialized.size(), 0);
@ -236,7 +236,7 @@ TEST(Proto3OptionalTest, OptionalField) {
msg.set_optional_int32(0);
EXPECT_TRUE(msg.has_optional_int32());
string serialized;
std::string serialized;
msg.SerializeToString(&serialized);
EXPECT_GT(serialized.size(), 0);

2
src/google/protobuf/struct.pb.h
View File

@ -100,7 +100,7 @@ inline const std::string& NullValue_Name(T enum_t_value) {
NullValue_descriptor(), enum_t_value);
}
inline bool NullValue_Parse(
const std::string& name, NullValue* value) {
::PROTOBUF_NAMESPACE_ID::ConstStringParam name, NullValue* value) {
return ::PROTOBUF_NAMESPACE_ID::internal::ParseNamedEnum<NullValue>(
NullValue_descriptor(), name, value);
}

2
src/google/protobuf/stubs/port.h
View File

@ -117,6 +117,8 @@
namespace google {
namespace protobuf {
using ConstStringParam = const std::string &;
typedef unsigned int uint;
typedef int8_t int8;

2
src/google/protobuf/test_util_lite.cc
View File

@ -1351,7 +1351,7 @@ void TestUtilLite::ExpectExtensionsClear(
std::string serialized;
ASSERT_TRUE(message.SerializeToString(&serialized));
EXPECT_EQ("", serialized);
EXPECT_EQ(0, message.ByteSize());
EXPECT_EQ(0, message.ByteSizeLong());
// has_blah() should initially be false for all optional fields.
EXPECT_FALSE(message.HasExtension(unittest::optional_int32_extension_lite));

16
src/google/protobuf/text_format_unittest.cc
View File

@ -55,6 +55,7 @@
#include <google/protobuf/io/tokenizer.h>
#include <google/protobuf/io/zero_copy_stream_impl.h>
#include <google/protobuf/stubs/strutil.h>
#include <gmock/gmock.h>
#include <google/protobuf/testing/googletest.h>
#include <gtest/gtest.h>
#include <google/protobuf/stubs/substitute.h>
@ -346,14 +347,13 @@ TEST_F(TextFormatTest, PrintUnknownMessage) {
UnknownFieldSet unknown_fields;
EXPECT_TRUE(unknown_fields.ParseFromString(data));
EXPECT_TRUE(TextFormat::PrintUnknownFieldsToString(unknown_fields, &text));
EXPECT_EQ(
"44: \"abc\"\n"
"44: \"def\"\n"
"44: \"\"\n"
"48 {\n"
" 1: 123\n"
"}\n",
text);
// Field 44 and 48 can be printed in any order.
EXPECT_THAT(text, testing::HasSubstr("44: \"abc\"\n"
"44: \"def\"\n"
"44: \"\"\n"));
EXPECT_THAT(text, testing::HasSubstr("48 {\n"
" 1: 123\n"
"}\n"));
}
TEST_F(TextFormatTest, PrintDeeplyNestedUnknownMessage) {

10
src/google/protobuf/type.pb.h
View File

@ -121,7 +121,7 @@ inline const std::string& Field_Kind_Name(T enum_t_value) {
Field_Kind_descriptor(), enum_t_value);
}
inline bool Field_Kind_Parse(
const std::string& name, Field_Kind* value) {
::PROTOBUF_NAMESPACE_ID::ConstStringParam name, Field_Kind* value) {
return ::PROTOBUF_NAMESPACE_ID::internal::ParseNamedEnum<Field_Kind>(
Field_Kind_descriptor(), name, value);
}
@ -148,7 +148,7 @@ inline const std::string& Field_Cardinality_Name(T enum_t_value) {
Field_Cardinality_descriptor(), enum_t_value);
}
inline bool Field_Cardinality_Parse(
const std::string& name, Field_Cardinality* value) {
::PROTOBUF_NAMESPACE_ID::ConstStringParam name, Field_Cardinality* value) {
return ::PROTOBUF_NAMESPACE_ID::internal::ParseNamedEnum<Field_Cardinality>(
Field_Cardinality_descriptor(), name, value);
}
@ -173,7 +173,7 @@ inline const std::string& Syntax_Name(T enum_t_value) {
Syntax_descriptor(), enum_t_value);
}
inline bool Syntax_Parse(
const std::string& name, Syntax* value) {
::PROTOBUF_NAMESPACE_ID::ConstStringParam name, Syntax* value) {
return ::PROTOBUF_NAMESPACE_ID::internal::ParseNamedEnum<Syntax>(
Syntax_descriptor(), name, value);
}
@ -598,7 +598,7 @@ class PROTOBUF_EXPORT Field PROTOBUF_FINAL :
"Incorrect type passed to function Kind_Name.");
return Field_Kind_Name(enum_t_value);
}
static inline bool Kind_Parse(const std::string& name,
static inline bool Kind_Parse(::PROTOBUF_NAMESPACE_ID::ConstStringParam name,
Kind* value) {
return Field_Kind_Parse(name, value);
}
@ -632,7 +632,7 @@ class PROTOBUF_EXPORT Field PROTOBUF_FINAL :
"Incorrect type passed to function Cardinality_Name.");
return Field_Cardinality_Name(enum_t_value);
}
static inline bool Cardinality_Parse(const std::string& name,
static inline bool Cardinality_Parse(::PROTOBUF_NAMESPACE_ID::ConstStringParam name,
Cardinality* value) {
return Field_Cardinality_Parse(name, value);
}

72
src/google/protobuf/unknown_field_set_unittest.cc
View File

@ -37,6 +37,8 @@
#include <google/protobuf/unknown_field_set.h>
#include <unordered_set>
#include <google/protobuf/stubs/callback.h>
#include <google/protobuf/stubs/common.h>
#include <google/protobuf/stubs/logging.h>
@ -113,30 +115,32 @@ class UnknownFieldSetTest : public testing::Test {
namespace {
TEST_F(UnknownFieldSetTest, AllFieldsPresent) {
// All fields of TestAllTypes should be present, in numeric order (because
// that's the order we parsed them in). Fields that are not valid field
// numbers of TestAllTypes should NOT be present.
// Verifies the following:
// --all unknown tags belong to TestAllTypes.
// --all fields in TestAllTypes is present in UnknownFieldSet except unset
// oneof fields.
//
// Should handle repeated fields that may appear multiple times in
// UnknownFieldSet.
int pos = 0;
for (int i = 0; i < 1000; i++) {
const FieldDescriptor* field = descriptor_->FindFieldByNumber(i);
if (field != NULL) {
ASSERT_LT(pos, unknown_fields_->field_count());
// Do not check oneof field if it is not set.
if (field->containing_oneof() == NULL) {
EXPECT_EQ(i, unknown_fields_->field(pos++).number());
} else if (i == unknown_fields_->field(pos).number()) {
pos++;
}
if (field->is_repeated()) {
// Should have a second instance.
ASSERT_LT(pos, unknown_fields_->field_count());
EXPECT_EQ(i, unknown_fields_->field(pos++).number());
}
int non_oneof_count = 0;
for (int i = 0; i < descriptor_->field_count(); i++) {
if (!descriptor_->field(i)->containing_oneof()) {
non_oneof_count++;
}
}
EXPECT_EQ(unknown_fields_->field_count(), pos);
std::unordered_set<uint32> unknown_tags;
for (int i = 0; i < unknown_fields_->field_count(); i++) {
unknown_tags.insert(unknown_fields_->field(i).number());
}
for (uint32 t : unknown_tags) {
EXPECT_NE(descriptor_->FindFieldByNumber(t), nullptr);
}
EXPECT_EQ(non_oneof_count + descriptor_->oneof_decl_count(),
unknown_tags.size());
}
TEST_F(UnknownFieldSetTest, Varint) {
@ -246,7 +250,7 @@ TEST_F(UnknownFieldSetTest, SerializeViaReflection) {
{
io::StringOutputStream raw_output(&data);
io::CodedOutputStream output(&raw_output);
int size = WireFormat::ByteSize(empty_message_);
size_t size = WireFormat::ByteSize(empty_message_);
WireFormat::SerializeWithCachedSizes(empty_message_, size, &output);
ASSERT_FALSE(output.HadError());
}
@ -536,29 +540,29 @@ TEST_F(UnknownFieldSetTest, SpaceUsed) {
// Make sure an unknown field set has zero space used until a field is
// actually added.
int base_size = empty_message.SpaceUsed();
size_t base_size = empty_message.SpaceUsedLong();
UnknownFieldSet* unknown_fields = empty_message.mutable_unknown_fields();
EXPECT_EQ(base_size, empty_message.SpaceUsed());
EXPECT_EQ(base_size, empty_message.SpaceUsedLong());
// Make sure each thing we add to the set increases the SpaceUsed().
// Make sure each thing we add to the set increases the SpaceUsedLong().
unknown_fields->AddVarint(1, 0);
EXPECT_LT(base_size, empty_message.SpaceUsed());
base_size = empty_message.SpaceUsed();
EXPECT_LT(base_size, empty_message.SpaceUsedLong());
base_size = empty_message.SpaceUsedLong();
std::string* str = unknown_fields->AddLengthDelimited(1);
EXPECT_LT(base_size, empty_message.SpaceUsed());
base_size = empty_message.SpaceUsed();
EXPECT_LT(base_size, empty_message.SpaceUsedLong());
base_size = empty_message.SpaceUsedLong();
str->assign(sizeof(std::string) + 1, 'x');
EXPECT_LT(base_size, empty_message.SpaceUsed());
base_size = empty_message.SpaceUsed();
EXPECT_LT(base_size, empty_message.SpaceUsedLong());
base_size = empty_message.SpaceUsedLong();
UnknownFieldSet* group = unknown_fields->AddGroup(1);
EXPECT_LT(base_size, empty_message.SpaceUsed());
base_size = empty_message.SpaceUsed();
EXPECT_LT(base_size, empty_message.SpaceUsedLong());
base_size = empty_message.SpaceUsedLong();
group->AddVarint(1, 0);
EXPECT_LT(base_size, empty_message.SpaceUsed());
EXPECT_LT(base_size, empty_message.SpaceUsedLong());
}

4
src/google/protobuf/util/field_mask_util_test.cc
View File

@ -46,7 +46,7 @@ namespace util {
class SnakeCaseCamelCaseTest : public ::testing::Test {
protected:
string SnakeCaseToCamelCase(const std::string& input) {
std::string SnakeCaseToCamelCase(const std::string& input) {
std::string output;
if (FieldMaskUtil::SnakeCaseToCamelCase(input, &output)) {
return output;
@ -55,7 +55,7 @@ class SnakeCaseCamelCaseTest : public ::testing::Test {
}
}
string CamelCaseToSnakeCase(const std::string& input) {
std::string CamelCaseToSnakeCase(const std::string& input) {
std::string output;
if (FieldMaskUtil::CamelCaseToSnakeCase(input, &output)) {
return output;

4
src/google/protobuf/util/json_util.cc
View File

@ -145,8 +145,8 @@ class StatusErrorListener : public converter::ErrorListener {
StringPiece value) override {
status_ = util::Status(
util::error::INVALID_ARGUMENT,
StrCat(GetLocString(loc), ": invalid value ", string(value),
" for type ", string(type_name)));
StrCat(GetLocString(loc), ": invalid value ", std::string(value),
" for type ", std::string(type_name)));
}
void MissingField(const converter::LocationTrackerInterface& loc,

16
src/google/protobuf/util/message_differencer_unittest.cc
View File

@ -1205,7 +1205,7 @@ TEST(MessageDifferencerTest, RepeatedFieldSmartSetTest_PreviouslyMatch) {
*msg2.add_rm() = elem1_2;
*msg2.add_rm() = elem2_2;
string diff_report;
std::string diff_report;
util::MessageDifferencer differencer;
differencer.ReportDifferencesToString(&diff_report);
differencer.set_repeated_field_comparison(
@ -2385,7 +2385,7 @@ class ComparisonTest : public testing::Test {
void field_as_set(const std::string& field) { set_field_ = field; }
void field_as_map(const string& field, const std::string& key) {
void field_as_map(const std::string& field, const std::string& key) {
map_field_ = field;
map_key_ = key;
}
@ -3198,11 +3198,13 @@ TEST_F(ComparisonTest, EquivalentIgnoresUnknown) {
}
TEST_F(ComparisonTest, MapTest) {
Map<string, std::string>& map1 = *map_proto1_.mutable_map_string_string();
Map<std::string, std::string>& map1 =
*map_proto1_.mutable_map_string_string();
map1["key1"] = "1";
map1["key2"] = "2";
map1["key3"] = "3";
Map<string, std::string>& map2 = *map_proto2_.mutable_map_string_string();
Map<std::string, std::string>& map2 =
*map_proto2_.mutable_map_string_string();
map2["key3"] = "0";
map2["key2"] = "2";
map2["key1"] = "1";
@ -3212,11 +3214,13 @@ TEST_F(ComparisonTest, MapTest) {
}
TEST_F(ComparisonTest, MapIgnoreKeyTest) {
Map<string, std::string>& map1 = *map_proto1_.mutable_map_string_string();
Map<std::string, std::string>& map1 =
*map_proto1_.mutable_map_string_string();
map1["key1"] = "1";
map1["key2"] = "2";
map1["key3"] = "3";
Map<string, std::string>& map2 = *map_proto2_.mutable_map_string_string();
Map<std::string, std::string>& map2 =
*map_proto2_.mutable_map_string_string();
map2["key4"] = "2";
map2["key5"] = "3";
map2["key6"] = "1";

2
src/google/protobuf/util/type_resolver_util.cc
View File

@ -305,7 +305,7 @@ class DescriptorPoolTypeResolver : public TypeResolver {
return url_prefix_ + "/" + descriptor->full_name();
}
Status ParseTypeUrl(const string& type_url, std::string* type_name) {
Status ParseTypeUrl(const std::string& type_url, std::string* type_name) {
if (type_url.substr(0, url_prefix_.size() + 1) != url_prefix_ + "/") {
return Status(
util::error::INVALID_ARGUMENT,

2
src/google/protobuf/util/type_resolver_util_test.cc
View File

@ -158,7 +158,7 @@ class DescriptorPoolTypeResolverTest : public testing::Test {
return false;
}
string GetTypeUrl(std::string full_name) {
std::string GetTypeUrl(std::string full_name) {
return kUrlPrefix + std::string("/") + full_name;
}

84
src/google/protobuf/wire_format_unittest.cc
View File

@ -38,6 +38,7 @@
#include <google/protobuf/stubs/common.h>
#include <google/protobuf/stubs/logging.h>
#include <google/protobuf/test_util.h>
#include <google/protobuf/test_util2.h>
#include <google/protobuf/unittest.pb.h>
#include <google/protobuf/unittest_mset.pb.h>
#include <google/protobuf/unittest_mset_wire_format.pb.h>
@ -48,12 +49,15 @@
#include <google/protobuf/descriptor.h>
#include <google/protobuf/wire_format_lite.h>
#include <google/protobuf/testing/googletest.h>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <google/protobuf/stubs/casts.h>
#include <google/protobuf/stubs/strutil.h>
#include <google/protobuf/stubs/stl_util.h>
// clang-format off
#include <google/protobuf/port_def.inc>
// clang-format on
namespace google {
namespace protobuf {
@ -209,8 +213,16 @@ TEST(WireFormatTest, OneofOnlySetLast) {
source.set_foo_int(100);
source.set_foo_string("101");
// Serialize and parse to oneof message.
source.SerializeToString(&data);
// Serialize and parse to oneof message. Generated serializer may not order
// fields in tag order. Use WireFormat::SerializeWithCachedSizes instead as
// it sorts fields beforehand.
{
io::StringOutputStream raw_output(&data);
io::CodedOutputStream output(&raw_output);
WireFormat::SerializeWithCachedSizes(source, source.ByteSizeLong(),
&output);
ASSERT_FALSE(output.HadError());
}
io::ArrayInputStream raw_input(data.data(), data.size());
io::CodedInputStream input(&raw_input);
WireFormat::ParseAndMergePartial(&input, &oneof_dest);
@ -224,9 +236,9 @@ TEST(WireFormatTest, ByteSize) {
unittest::TestAllTypes message;
TestUtil::SetAllFields(&message);
EXPECT_EQ(message.ByteSize(), WireFormat::ByteSize(message));
EXPECT_EQ(message.ByteSizeLong(), WireFormat::ByteSize(message));
message.Clear();
EXPECT_EQ(0, message.ByteSize());
EXPECT_EQ(0, message.ByteSizeLong());
EXPECT_EQ(0, WireFormat::ByteSize(message));
}
@ -234,9 +246,9 @@ TEST(WireFormatTest, ByteSizeExtensions) {
unittest::TestAllExtensions message;
TestUtil::SetAllExtensions(&message);
EXPECT_EQ(message.ByteSize(), WireFormat::ByteSize(message));
EXPECT_EQ(message.ByteSizeLong(), WireFormat::ByteSize(message));
message.Clear();
EXPECT_EQ(0, message.ByteSize());
EXPECT_EQ(0, message.ByteSizeLong());
EXPECT_EQ(0, WireFormat::ByteSize(message));
}
@ -244,9 +256,9 @@ TEST(WireFormatTest, ByteSizePacked) {
unittest::TestPackedTypes message;
TestUtil::SetPackedFields(&message);
EXPECT_EQ(message.ByteSize(), WireFormat::ByteSize(message));
EXPECT_EQ(message.ByteSizeLong(), WireFormat::ByteSize(message));
message.Clear();
EXPECT_EQ(0, message.ByteSize());
EXPECT_EQ(0, message.ByteSizeLong());
EXPECT_EQ(0, WireFormat::ByteSize(message));
}
@ -254,9 +266,9 @@ TEST(WireFormatTest, ByteSizePackedExtensions) {
unittest::TestPackedExtensions message;
TestUtil::SetPackedExtensions(&message);
EXPECT_EQ(message.ByteSize(), WireFormat::ByteSize(message));
EXPECT_EQ(message.ByteSizeLong(), WireFormat::ByteSize(message));
message.Clear();
EXPECT_EQ(0, message.ByteSize());
EXPECT_EQ(0, message.ByteSizeLong());
EXPECT_EQ(0, WireFormat::ByteSize(message));
}
@ -264,10 +276,10 @@ TEST(WireFormatTest, ByteSizeOneof) {
unittest::TestOneof2 message;
TestUtil::SetOneof1(&message);
EXPECT_EQ(message.ByteSize(), WireFormat::ByteSize(message));
EXPECT_EQ(message.ByteSizeLong(), WireFormat::ByteSize(message));
message.Clear();
EXPECT_EQ(0, message.ByteSize());
EXPECT_EQ(0, message.ByteSizeLong());
EXPECT_EQ(0, WireFormat::ByteSize(message));
}
@ -277,7 +289,7 @@ TEST(WireFormatTest, Serialize) {
std::string dynamic_data;
TestUtil::SetAllFields(&message);
int size = message.ByteSize();
size_t size = message.ByteSizeLong();
// Serialize using the generated code.
{
@ -295,10 +307,9 @@ TEST(WireFormatTest, Serialize) {
ASSERT_FALSE(output.HadError());
}
// Should be the same.
// Don't use EXPECT_EQ here because we're comparing raw binary data and
// we really don't want it dumped to stdout on failure.
EXPECT_TRUE(dynamic_data == generated_data);
// Should parse to the same message.
EXPECT_TRUE(TestUtil::EqualsToSerialized(message, generated_data));
EXPECT_TRUE(TestUtil::EqualsToSerialized(message, dynamic_data));
}
TEST(WireFormatTest, SerializeExtensions) {
@ -307,7 +318,7 @@ TEST(WireFormatTest, SerializeExtensions) {
std::string dynamic_data;
TestUtil::SetAllExtensions(&message);
int size = message.ByteSize();
size_t size = message.ByteSizeLong();
// Serialize using the generated code.
{
@ -325,10 +336,9 @@ TEST(WireFormatTest, SerializeExtensions) {
ASSERT_FALSE(output.HadError());
}
// Should be the same.
// Don't use EXPECT_EQ here because we're comparing raw binary data and
// we really don't want it dumped to stdout on failure.
EXPECT_TRUE(dynamic_data == generated_data);
// Should parse to the same message.
EXPECT_TRUE(TestUtil::EqualsToSerialized(message, generated_data));
EXPECT_TRUE(TestUtil::EqualsToSerialized(message, dynamic_data));
}
TEST(WireFormatTest, SerializeFieldsAndExtensions) {
@ -337,7 +347,7 @@ TEST(WireFormatTest, SerializeFieldsAndExtensions) {
std::string dynamic_data;
TestUtil::SetAllFieldsAndExtensions(&message);
int size = message.ByteSize();
size_t size = message.ByteSizeLong();
// Serialize using the generated code.
{
@ -355,14 +365,9 @@ TEST(WireFormatTest, SerializeFieldsAndExtensions) {
ASSERT_FALSE(output.HadError());
}
// Should be the same.
// Don't use EXPECT_EQ here because we're comparing raw binary data and
// we really don't want it dumped to stdout on failure.
EXPECT_TRUE(dynamic_data == generated_data);
// Should output in canonical order.
TestUtil::ExpectAllFieldsAndExtensionsInOrder(dynamic_data);
TestUtil::ExpectAllFieldsAndExtensionsInOrder(generated_data);
// Should parse to the same message.
EXPECT_TRUE(TestUtil::EqualsToSerialized(message, generated_data));
EXPECT_TRUE(TestUtil::EqualsToSerialized(message, dynamic_data));
}
TEST(WireFormatTest, SerializeOneof) {
@ -371,7 +376,7 @@ TEST(WireFormatTest, SerializeOneof) {
std::string dynamic_data;
TestUtil::SetOneof1(&message);
int size = message.ByteSize();
size_t size = message.ByteSizeLong();
// Serialize using the generated code.
{
@ -389,10 +394,9 @@ TEST(WireFormatTest, SerializeOneof) {
ASSERT_FALSE(output.HadError());
}
// Should be the same.
// Don't use EXPECT_EQ here because we're comparing raw binary data and
// we really don't want it dumped to stdout on failure.
EXPECT_TRUE(dynamic_data == generated_data);
// Should parse to the same message.
EXPECT_TRUE(TestUtil::EqualsToSerialized(message, generated_data));
EXPECT_TRUE(TestUtil::EqualsToSerialized(message, dynamic_data));
}
TEST(WireFormatTest, ParseMultipleExtensionRanges) {
@ -482,7 +486,7 @@ TEST(WireFormatTest, SerializeMessageSetVariousWaysAreEqual) {
message_set.mutable_unknown_fields()->AddLengthDelimited(kUnknownTypeId,
"bar");
int size = message_set.ByteSize();
size_t size = message_set.ByteSizeLong();
EXPECT_EQ(size, message_set.GetCachedSize());
ASSERT_EQ(size, WireFormat::ByteSize(message_set));
@ -595,7 +599,7 @@ TEST(WireFormatTest, ParseMessageSetWithReverseTagOrder) {
WireFormatLite::WriteTag(WireFormatLite::kMessageSetMessageNumber,
WireFormatLite::WIRETYPE_LENGTH_DELIMITED,
&coded_output);
coded_output.WriteVarint32(message.ByteSize());
coded_output.WriteVarint32(message.ByteSizeLong());
message.SerializeWithCachedSizes(&coded_output);
// Write the type id.
uint32 type_id = message.GetDescriptor()->extension(0)->number();
@ -989,7 +993,7 @@ class Proto3PrimitiveRepeatedWireFormatTest : public ::testing::Test {
void TestSerialization(Proto* message, const std::string& expected) {
SetProto3PrimitiveRepeatedFields(message);
int size = message->ByteSize();
size_t size = message->ByteSizeLong();
// Serialize using the generated code.
std::string generated_data;
@ -999,7 +1003,7 @@ class Proto3PrimitiveRepeatedWireFormatTest : public ::testing::Test {
message->SerializeWithCachedSizes(&output);
ASSERT_FALSE(output.HadError());
}
EXPECT_TRUE(expected == generated_data);
EXPECT_TRUE(TestUtil::EqualsToSerialized(*message, generated_data));
// Serialize using the dynamic code.
std::string dynamic_data;