AuroraMiddleware/protobuf
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

Merge pull request #7925 from haberman/sync-stage

Browse Source 
Integrate from Piper for C++, Java, and Python
This commit is contained in:
Joshua Haberman 2020-10-28 12:02:24 -07:00 committed by GitHub
commit fdc35840b9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
208 changed files with 4480 additions and 3655 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
BUILD
View File

@ -164,6 +164,7 @@ cc_library(
# AUTOGEN(protobuf_lite_srcs)
"src/google/protobuf/any_lite.cc",
"src/google/protobuf/arena.cc",
"src/google/protobuf/arenastring.cc",
"src/google/protobuf/extension_set.cc",
"src/google/protobuf/generated_enum_util.cc",
"src/google/protobuf/generated_message_table_driven_lite.cc",

17
CHANGES.txt
View File

@ -3,10 +3,13 @@ Unreleased Changes
Protocol Compiler
* The proto compiler no longer requires a .proto filename when it is not
generating code.
* Added flag `--deterministic_output` to `protoc --encode=...`.
* Fixed deadlock when using google.protobuf.Any embedded in aggregate options.
C++
* Arenas are now unconditionally enabled. cc_enable_arenas no longer has
any effect.
* Removed inlined string support, which is incompatible with arenas.
* Fix a memory corruption bug in reflection when mixing optional and
non-optional fields.
* Make SpaceUsed() calculation more thorough for map fields.
@ -20,9 +23,23 @@ Unreleased Changes
* When running under ASAN, skip a test that makes huge allocations.
* Fixed a crash that could happen when creating more than 256 extensions in
a single message.
* Fix a crash in BuildFile when passing in invalid descriptor proto.
* Parser security fix when operating with CodedInputStream.
* Warn against the use of AllowUnknownExtension.
* Migrated to C++11 for-range loops instead of index-based loops where
possible. This fixes a lot of warnings when compiling with -Wsign-compare.
Java
* Bugfix in mergeFrom() when a oneof has multiple message fields.
* Fix RopeByteString.RopeInputStream.read() returning -1 when told to read
0 bytes when not at EOF.
* Redefine remove(Object) on primitive repeated field Lists to avoid
autoboxing.
* Support "\u" escapes in textformat string literals.
* Trailing empty spaces are no longer ignored for FieldMask.
* Fix FieldMaskUtil.subtract to recursively remove mask.
* Mark enums with `@java.lang.Deprecated` if the proto enum has option
`deprecated = true;`.
Python
* Print google.protobuf.NullValue as null instead of "NULL_VALUE" when it is

2
cmake/extract_includes.bat.in
View File

@ -53,7 +53,6 @@ copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_message_tab
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_message_util.h" include\google\protobuf\generated_message_util.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\has_bits.h" include\google\protobuf\has_bits.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\implicit_weak_message.h" include\google\protobuf\implicit_weak_message.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\inlined_string_field.h" include\google\protobuf\inlined_string_field.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\coded_stream.h" include\google\protobuf\io\coded_stream.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\gzip_stream.h" include\google\protobuf\io\gzip_stream.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\io_win32.h" include\google\protobuf\io\io_win32.h
@ -88,7 +87,6 @@ copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\bytestream.h" i
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\callback.h" include\google\protobuf\stubs\callback.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\casts.h" include\google\protobuf\stubs\casts.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\common.h" include\google\protobuf\stubs\common.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\fastmem.h" include\google\protobuf\stubs\fastmem.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\hash.h" include\google\protobuf\stubs\hash.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\logging.h" include\google\protobuf\stubs\logging.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\macros.h" include\google\protobuf\stubs\macros.h

1
cmake/libprotobuf-lite.cmake
View File

@ -1,6 +1,7 @@
set(libprotobuf_lite_files
${protobuf_source_dir}/src/google/protobuf/any_lite.cc
${protobuf_source_dir}/src/google/protobuf/arena.cc
${protobuf_source_dir}/src/google/protobuf/arenastring.cc
${protobuf_source_dir}/src/google/protobuf/extension_set.cc
${protobuf_source_dir}/src/google/protobuf/generated_enum_util.cc
${protobuf_source_dir}/src/google/protobuf/generated_message_table_driven_lite.cc

20
conformance/Makefile.am
View File

@ -333,7 +333,7 @@ conformance-php-c:
# Targets for actually running tests.
test_cpp: protoc_middleman conformance-test-runner conformance-cpp
./conformance-test-runner --enforce_recommended --failure_list failure_list_cpp.txt ./conformance-cpp
./conformance-test-runner --enforce_recommended --failure_list failure_list_cpp.txt --text_format_failure_list text_format_failure_list_cpp.txt ./conformance-cpp
test_java: protoc_middleman conformance-test-runner conformance-java
./conformance-test-runner --enforce_recommended --failure_list failure_list_java.txt --text_format_failure_list text_format_failure_list_java.txt ./conformance-java
@ -359,10 +359,24 @@ test_php_c_32: protoc_middleman conformance-test-runner conformance-php-c $(othe
# These depend on library paths being properly set up. The easiest way to
# run them is to just use "tox" from the python dir.
test_python: protoc_middleman conformance-test-runner
./conformance-test-runner --enforce_recommended --failure_list failure_list_python.txt --text_format_failure_list text_format_failure_list_python.txt ./conformance_python.py
VERSION="$(shell python --version 2>&1)"; \
if [[ "$$VERSION" == "Python 2.7"* ]]; then \
echo "Using Python 2.7 failure list."; \
./conformance-test-runner --enforce_recommended --failure_list failure_list_python.txt --text_format_failure_list text_format_failure_list_python_2.7.txt ./conformance_python.py; \
else \
echo "Using Python >2.7 failure list."; \
./conformance-test-runner --enforce_recommended --failure_list failure_list_python.txt --text_format_failure_list text_format_failure_list_python.txt ./conformance_python.py; \
fi
test_python_cpp: protoc_middleman conformance-test-runner
./conformance-test-runner --enforce_recommended --failure_list failure_list_python_cpp.txt ./conformance_python.py
VERSION="$(shell python --version 2>&1)"; \
if [[ "$$VERSION" == "Python 2.7"* ]]; then \
echo "Using Python 2.7 failure list."; \
./conformance-test-runner --enforce_recommended --failure_list failure_list_python_cpp.txt --text_format_failure_list text_format_failure_list_python_cpp_2.7.txt ./conformance_python.py; \
else \
echo "Using Python >2.7 failure list."; \
./conformance-test-runner --enforce_recommended --failure_list failure_list_python_cpp.txt --text_format_failure_list text_format_failure_list_python_cpp.txt ./conformance_python.py; \
fi
test_nodejs: protoc_middleman conformance-test-runner $(other_language_protoc_outputs)
NODE_PATH=../js:. ./conformance-test-runner --enforce_recommended --failure_list failure_list_js.txt ./conformance_nodejs.js

1
conformance/failure_list_js.txt
View File

@ -1 +0,0 @@

78
conformance/text_format_conformance_suite.cc
View File

@ -258,6 +258,84 @@ void TextFormatConformanceTestSuite::RunSuiteImpl() {
RunValidTextFormatTest("FloatFieldLargerThanUint64", REQUIRED,
"optional_float: 18446744073709551616");
// String literals x {Strings, Bytes}
for (const auto& field_type : std::vector<std::string>{"String", "Bytes"}) {
const std::string field_name =
field_type == "String" ? "optional_string" : "optional_bytes";
RunValidTextFormatTest(
StrCat("StringLiteralConcat", field_type), REQUIRED,
StrCat(field_name, ": 'first' \"second\"\n'third'"));
RunValidTextFormatTest(
StrCat("StringLiteralBasicEscapes", field_type), REQUIRED,
StrCat(field_name, ": '\\a\\b\\f\\n\\r\\t\\v\\?\\\\\\'\\\"'"));
RunValidTextFormatTest(
StrCat("StringLiteralOctalEscapes", field_type), REQUIRED,
StrCat(field_name, ": '\\341\\210\\264'"));
RunValidTextFormatTest(StrCat("StringLiteralHexEscapes", field_type),
REQUIRED,
StrCat(field_name, ": '\\xe1\\x88\\xb4'"));
RunValidTextFormatTest(
StrCat("StringLiteralShortUnicodeEscape", field_type),
RECOMMENDED, StrCat(field_name, ": '\\u1234'"));
RunValidTextFormatTest(
StrCat("StringLiteralLongUnicodeEscapes", field_type),
RECOMMENDED, StrCat(field_name, ": '\\U00001234\\U00010437'"));
// String literals don't include line feeds.
ExpectParseFailure(StrCat("StringLiteralIncludesLF", field_type),
REQUIRED,
StrCat(field_name, ": 'first line\nsecond line'"));
// Unicode escapes don't include code points that lie beyond the planes
// (> 0x10ffff).
ExpectParseFailure(
StrCat("StringLiteralLongUnicodeEscapeTooLarge", field_type),
REQUIRED, StrCat(field_name, ": '\\U00110000'"));
// Unicode escapes don't include surrogates.
ExpectParseFailure(
StrCat("StringLiteralShortUnicodeEscapeSurrogatePair",
field_type),
RECOMMENDED, StrCat(field_name, ": '\\ud801\\udc37'"));
ExpectParseFailure(
StrCat("StringLiteralShortUnicodeEscapeSurrogateFirstOnly",
field_type),
RECOMMENDED, StrCat(field_name, ": '\\ud800'"));
ExpectParseFailure(
StrCat("StringLiteralShortUnicodeEscapeSurrogateSecondOnly",
field_type),
RECOMMENDED, StrCat(field_name, ": '\\udc00'"));
ExpectParseFailure(
StrCat("StringLiteralLongUnicodeEscapeSurrogateFirstOnly",
field_type),
RECOMMENDED, StrCat(field_name, ": '\\U0000d800'"));
ExpectParseFailure(
StrCat("StringLiteralLongUnicodeEscapeSurrogateSecondOnly",
field_type),
RECOMMENDED, StrCat(field_name, ": '\\U0000dc00'"));
ExpectParseFailure(
StrCat("StringLiteralLongUnicodeEscapeSurrogatePair", field_type),
RECOMMENDED, StrCat(field_name, ": '\\U0000d801\\U00000dc37'"));
ExpectParseFailure(
StrCat("StringLiteralUnicodeEscapeSurrogatePairLongShort",
field_type),
RECOMMENDED, StrCat(field_name, ": '\\U0000d801\\udc37'"));
ExpectParseFailure(
StrCat("StringLiteralUnicodeEscapeSurrogatePairShortLong",
field_type),
RECOMMENDED, StrCat(field_name, ": '\\ud801\\U0000dc37'"));
// The following method depend on the type of field, as strings have extra
// validation.
const auto test_method =
field_type == "String"
? &TextFormatConformanceTestSuite::ExpectParseFailure
: &TextFormatConformanceTestSuite::RunValidTextFormatTest;
// String fields reject invalid UTF-8 byte sequences; bytes fields don't.
(this->*test_method)(StrCat(field_type, "FieldBadUTF8Octal"),
REQUIRED, StrCat(field_name, ": '\\300'"));
(this->*test_method)(StrCat(field_type, "FieldBadUTF8Hex"), REQUIRED,
StrCat(field_name, ": '\\xc0'"));
}
// Group fields
RunValidTextFormatTestProto2("GroupFieldNoColon", REQUIRED,
"Data { group_int32: 1 }");

20
conformance/text_format_failure_list_cpp.txt Normal file
View File

@ -0,0 +1,20 @@
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairBytes
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairString
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairBytes
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairString
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortBytes
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortString
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongBytes
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongString
Required.Proto3.TextFormatInput.StringFieldBadUTF8Hex
Required.Proto3.TextFormatInput.StringFieldBadUTF8Octal
Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeBytes
Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeString

7
conformance/text_format_failure_list_java.txt
View File

@ -4,3 +4,10 @@ Recommended.Proto3.ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput
Recommended.Proto3.ProtobufInput.ScalarUnknownFields_Drop.TextFormatOutput
Required.Proto3.TextFormatInput.AnyField.ProtobufOutput
Required.Proto3.TextFormatInput.AnyField.TextFormatOutput
Required.Proto3.TextFormatInput.StringFieldBadUTF8Hex
Required.Proto3.TextFormatInput.StringFieldBadUTF8Octal
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput

29
conformance/text_format_failure_list_python.txt
View File

@ -3,3 +3,32 @@
# TODO: These should be fixed.
Required.Proto3.TextFormatInput.FloatFieldMaxValue.ProtobufOutput
Required.Proto3.TextFormatInput.FloatFieldMaxValue.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairBytes
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairString
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesBytes.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesBytes.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesString.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesString.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeBytes.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeBytes.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeString.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeString.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairBytes
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairString
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortBytes
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortString
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongBytes
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongString
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput

36
conformance/text_format_failure_list_python_2.7.txt Normal file
View File

@ -0,0 +1,36 @@
# This is the list of text format conformance tests that are known to fail right
# now.
# TODO: These should be fixed.
Required.Proto3.TextFormatInput.FloatFieldMaxValue.ProtobufOutput
Required.Proto3.TextFormatInput.FloatFieldMaxValue.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairBytes
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairString
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesBytes.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesBytes.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesString.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesString.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeBytes.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeBytes.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeString.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeString.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairBytes
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairString
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortBytes
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortString
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongBytes
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongString
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput
Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeBytes
Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeString

28
conformance/text_format_failure_list_python_cpp.txt Normal file
View File

@ -0,0 +1,28 @@
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairBytes
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairString
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesBytes.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesBytes.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesString.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesString.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeBytes.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeBytes.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeString.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeString.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairBytes
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairString
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortBytes
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortString
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongBytes
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongString
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput

30
conformance/text_format_failure_list_python_cpp_2.7.txt Normal file
View File

@ -0,0 +1,30 @@
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairBytes
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairString
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesBytes.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesBytes.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesString.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesString.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeBytes.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeBytes.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeString.ProtobufOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeString.TextFormatOutput
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairBytes
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairString
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyBytes
Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyString
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortBytes
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortString
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongBytes
Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongString
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput
Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput
Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeBytes
Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeString

7
java/core/src/main/java/com/google/protobuf/AbstractProtobufList.java
View File

@ -142,7 +142,12 @@ abstract class AbstractProtobufList<E> extends AbstractList<E> implements Protob
@Override
public boolean remove(Object o) {
ensureIsMutable();
return super.remove(o);
int index = indexOf(o);
if (index == -1) {
return false;
}
remove(index);
return true;
}
@Override

14
java/core/src/main/java/com/google/protobuf/BooleanArrayList.java
View File

@ -267,20 +267,6 @@ final class BooleanArrayList extends AbstractProtobufList<Boolean>
return true;
}
@Override
public boolean remove(Object o) {
ensureIsMutable();
for (int i = 0; i < size; i++) {
if (o.equals(array[i])) {
System.arraycopy(array, i + 1, array, i, size - i - 1);
size--;
modCount++;
return true;
}
}
return false;
}
@Override
public Boolean remove(int index) {
ensureIsMutable();

17
java/core/src/main/java/com/google/protobuf/CodedInputStream.java
View File

@ -41,6 +41,7 @@ import static com.google.protobuf.WireFormat.MAX_VARINT_SIZE;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.Buffer;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
@ -2009,14 +2010,14 @@ public abstract class CodedInputStream {
int prevPos = buffer.position();
int prevLimit = buffer.limit();
try {
buffer.position(bufferPos(begin));
buffer.limit(bufferPos(end));
((Buffer) buffer).position(bufferPos(begin));
((Buffer) buffer).limit(bufferPos(end));
return buffer.slice();
} catch (IllegalArgumentException e) {
throw InvalidProtocolBufferException.truncatedMessage();
} finally {
buffer.position(prevPos);
buffer.limit(prevLimit);
((Buffer) buffer).position(prevPos);
((Buffer) buffer).limit(prevLimit);
}
}
}
@ -3910,14 +3911,14 @@ public abstract class CodedInputStream {
int prevPos = currentByteBuffer.position();
int prevLimit = currentByteBuffer.limit();
try {
currentByteBuffer.position(begin);
currentByteBuffer.limit(end);
((Buffer) currentByteBuffer).position(begin);
((Buffer) currentByteBuffer).limit(end);
return currentByteBuffer.slice();
} catch (IllegalArgumentException e) {
throw InvalidProtocolBufferException.truncatedMessage();
} finally {
currentByteBuffer.position(prevPos);
currentByteBuffer.limit(prevLimit);
((Buffer) currentByteBuffer).position(prevPos);
((Buffer) currentByteBuffer).limit(prevLimit);
}
}
}

5
java/core/src/main/java/com/google/protobuf/Descriptors.java
View File

@ -764,7 +764,10 @@ public final class Descriptors {
/**
* Finds a field by name.
*
* @param name The unqualified name of the field (e.g. "foo").
* @param name The unqualified name of the field (e.g. "foo"). For protocol buffer messages that
* follow <a
* href=https://developers.google.com/protocol-buffers/docs/style#message_and_field_names>Google's
* guidance on naming</a> this will be a snake case string, such as <pre>song_name</pre>.
* @return The field's descriptor, or {@code null} if not found.
*/
public FieldDescriptor findFieldByName(final String name) {

14
java/core/src/main/java/com/google/protobuf/DoubleArrayList.java
View File

@ -267,20 +267,6 @@ final class DoubleArrayList extends AbstractProtobufList<Double>
return true;
}
@Override
public boolean remove(Object o) {
ensureIsMutable();
for (int i = 0; i < size; i++) {
if (o.equals(array[i])) {
System.arraycopy(array, i + 1, array, i, size - i - 1);
size--;
modCount++;
return true;
}
}
return false;
}
@Override
public Double remove(int index) {
ensureIsMutable();

2
java/core/src/main/java/com/google/protobuf/ExtensionRegistryFactory.java
View File

@ -46,7 +46,6 @@ final class ExtensionRegistryFactory {
@Nullable */
static final Class<?> EXTENSION_REGISTRY_CLASS = reflectExtensionRegistry();
/* @Nullable */
static Class<?> reflectExtensionRegistry() {
try {
return Class.forName(FULL_REGISTRY_CLASS_NAME);
@ -77,7 +76,6 @@ final class ExtensionRegistryFactory {
&& EXTENSION_REGISTRY_CLASS.isAssignableFrom(registry.getClass());
}
/* @Nullable */
private static final ExtensionRegistryLite invokeSubclassFactory(String methodName) {
if (EXTENSION_REGISTRY_CLASS == null) {
return null;

2
java/core/src/main/java/com/google/protobuf/ExtensionRegistryLite.java
View File

@ -84,10 +84,8 @@ public class ExtensionRegistryLite {
static final String EXTENSION_CLASS_NAME = "com.google.protobuf.Extension";
private static class ExtensionClassHolder {
/* @Nullable */
static final Class<?> INSTANCE = resolveExtensionClass();
/* @Nullable */
static Class<?> resolveExtensionClass() {
try {
return Class.forName(EXTENSION_CLASS_NAME);

2
java/core/src/main/java/com/google/protobuf/FieldType.java
View File

@ -204,7 +204,6 @@ public enum FieldType {
*
* @return the {@link FieldType} or {@code null} if not found.
*/
/* @Nullable */
public static FieldType forId(int id) {
if (id < 0 || id >= VALUES.length) {
return null;
@ -228,7 +227,6 @@ public enum FieldType {
*
* @return the generic super class/interface, or {@code null} if not found.
*/
/* @Nullable */
private static Type getGenericSuperList(Class<?> clazz) {
// First look at interfaces.
Type[] genericInterfaces = clazz.getGenericInterfaces();

14
java/core/src/main/java/com/google/protobuf/FloatArrayList.java
View File

@ -266,20 +266,6 @@ final class FloatArrayList extends AbstractProtobufList<Float>
return true;
}
@Override
public boolean remove(Object o) {
ensureIsMutable();
for (int i = 0; i < size; i++) {
if (o.equals(array[i])) {
System.arraycopy(array, i + 1, array, i, size - i - 1);
size--;
modCount++;
return true;
}
}
return false;
}
@Override
public Float remove(int index) {
ensureIsMutable();

14
java/core/src/main/java/com/google/protobuf/IntArrayList.java
View File

@ -266,20 +266,6 @@ final class IntArrayList extends AbstractProtobufList<Integer>
return true;
}
@Override
public boolean remove(Object o) {
ensureIsMutable();
for (int i = 0; i < size; i++) {
if (o.equals(array[i])) {
System.arraycopy(array, i + 1, array, i, size - i - 1);
size--;
modCount++;
return true;
}
}
return false;
}
@Override
public Integer remove(int index) {
ensureIsMutable();

1
java/core/src/main/java/com/google/protobuf/Internal.java
View File

@ -30,7 +30,6 @@
package com.google.protobuf;
import java.io.IOException;
import java.lang.reflect.Method;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;

14
java/core/src/main/java/com/google/protobuf/LongArrayList.java
View File

@ -266,20 +266,6 @@ final class LongArrayList extends AbstractProtobufList<Long>
return true;
}
@Override
public boolean remove(Object o) {
ensureIsMutable();
for (int i = 0; i < size; i++) {
if (o.equals(array[i])) {
System.arraycopy(array, i + 1, array, i, size - i - 1);
size--;
modCount++;
return true;
}
}
return false;
}
@Override
public Long remove(int index) {
ensureIsMutable();

2
java/core/src/main/java/com/google/protobuf/MessageSchema.java
View File

@ -2572,7 +2572,7 @@ final class MessageSchema<T> implements Schema<T> {
int presenceMaskAndOffset = 0;
int presenceMask = 0;
if (!proto3 && fieldType <= 17) {
if (fieldType <= 17) {
presenceMaskAndOffset = buffer[pos + 2];
final int presenceFieldOffset = presenceMaskAndOffset & OFFSET_MASK;
if (presenceFieldOffset != currentPresenceFieldOffset) {

7
java/core/src/main/java/com/google/protobuf/NioByteString.java
View File

@ -37,6 +37,7 @@ import java.io.InputStream;
import java.io.InvalidObjectException;
import java.io.ObjectInputStream;
import java.io.OutputStream;
import java.nio.Buffer;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.InvalidMarkException;
@ -109,7 +110,7 @@ final class NioByteString extends ByteString.LeafByteString {
protected void copyToInternal(
byte[] target, int sourceOffset, int targetOffset, int numberToCopy) {
ByteBuffer slice = buffer.slice();
slice.position(sourceOffset);
((Buffer) slice).position(sourceOffset);
slice.get(target, targetOffset, numberToCopy);
}
@ -285,8 +286,8 @@ final class NioByteString extends ByteString.LeafByteString {
}
ByteBuffer slice = buffer.slice();
slice.position(beginIndex - buffer.position());
slice.limit(endIndex - buffer.position());
((Buffer) slice).position(beginIndex - buffer.position());
((Buffer) slice).limit(endIndex - buffer.position());
return slice;
}
}

7
java/core/src/main/java/com/google/protobuf/Protobuf.java
View File

@ -76,11 +76,8 @@ final class Protobuf {
schemaFor(message).makeImmutable(message);
}
/**
* Checks if all required fields are set. TODO(xiaofeng): Make this package private when the tests
* are moved to protobuf package.
*/
public <T> boolean isInitialized(T message) {
/** Checks if all required fields are set. */
<T> boolean isInitialized(T message) {
return schemaFor(message).isInitialized(message);
}

14
java/core/src/main/java/com/google/protobuf/RopeByteString.java
View File

@ -845,7 +845,10 @@ final class RopeByteString extends ByteString {
throw new IndexOutOfBoundsException();
}
int bytesRead = readSkipInternal(b, offset, length);
if (bytesRead == 0) {
if (bytesRead == 0 && (length > 0 || availableInternal() == 0)) {
// Modeling ByteArrayInputStream.read(byte[], int, int) behavior noted above:
// It's ok to read 0 bytes on purpose (length == 0) from a stream that isn't at EOF.
// It's not ok to try to read bytes (even 0 bytes) from a stream that is at EOF.
return -1;
} else {
return bytesRead;
@ -905,8 +908,7 @@ final class RopeByteString extends ByteString {
@Override
public int available() throws IOException {
int bytesRead = currentPieceOffsetInRope + currentPieceIndex;
return RopeByteString.this.size() - bytesRead;
return availableInternal();
}
@Override
@ -955,5 +957,11 @@ final class RopeByteString extends ByteString {
}
}
}
/** Computes the number of bytes still available to read. */
private int availableInternal() {
int bytesRead = currentPieceOffsetInRope + currentPieceIndex;
return RopeByteString.this.size() - bytesRead;
}
}
}

69
java/core/src/main/java/com/google/protobuf/TextFormat.java
View File

@ -30,6 +30,8 @@
package com.google.protobuf;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.protobuf.Descriptors.Descriptor;
import com.google.protobuf.Descriptors.EnumDescriptor;
import com.google.protobuf.Descriptors.EnumValueDescriptor;
@ -2375,6 +2377,73 @@ public final class TextFormat {
result[pos++] = (byte) code;
break;
case 'u':
// Unicode escape
++i;
if (i + 3 < input.size()
&& isHex(input.byteAt(i))
&& isHex(input.byteAt(i + 1))
&& isHex(input.byteAt(i + 2))
&& isHex(input.byteAt(i + 3))) {
char ch =
(char)
(digitValue(input.byteAt(i)) << 12
| digitValue(input.byteAt(i + 1)) << 8
| digitValue(input.byteAt(i + 2)) << 4
| digitValue(input.byteAt(i + 3)));
if (Character.isSurrogate(ch)) {
throw new InvalidEscapeSequenceException(
"Invalid escape sequence: '\\u' refers to a surrogate");
}
byte[] chUtf8 = Character.toString(ch).getBytes(UTF_8);
System.arraycopy(chUtf8, 0, result, pos, chUtf8.length);
pos += chUtf8.length;
i += 3;
} else {
throw new InvalidEscapeSequenceException(
"Invalid escape sequence: '\\u' with too few hex chars");
}
break;
case 'U':
// Unicode escape
++i;
if (i + 7 >= input.size()) {
throw new InvalidEscapeSequenceException(
"Invalid escape sequence: '\\U' with too few hex chars");
}
int codepoint = 0;
for (int offset = i; offset < i + 8; offset++) {
byte b = input.byteAt(offset);
if (!isHex(b)) {
throw new InvalidEscapeSequenceException(
"Invalid escape sequence: '\\U' with too few hex chars");
}
codepoint = (codepoint << 4) | digitValue(b);
}
if (!Character.isValidCodePoint(codepoint)) {
throw new InvalidEscapeSequenceException(
"Invalid escape sequence: '\\U"
+ input.substring(i, i + 8).toStringUtf8()
+ "' is not a valid code point value");
}
Character.UnicodeBlock unicodeBlock = Character.UnicodeBlock.of(codepoint);
if (unicodeBlock.equals(Character.UnicodeBlock.LOW_SURROGATES)
|| unicodeBlock.equals(Character.UnicodeBlock.HIGH_SURROGATES)
|| unicodeBlock.equals(Character.UnicodeBlock.HIGH_PRIVATE_USE_SURROGATES)) {
throw new InvalidEscapeSequenceException(
"Invalid escape sequence: '\\U"
+ input.substring(i, i + 8).toStringUtf8()
+ "' refers to a surrogate code unit");
}
int[] codepoints = new int[1];
codepoints[0] = codepoint;
byte[] chUtf8 = new String(codepoints, 0, 1).getBytes(UTF_8);
System.arraycopy(chUtf8, 0, result, pos, chUtf8.length);
pos += chUtf8.length;
i += 7;
break;
default:
throw new InvalidEscapeSequenceException(
"Invalid escape sequence: '\\" + (char) c + '\'');

1
java/core/src/main/java/com/google/protobuf/TypeRegistry.java
View File

@ -70,7 +70,6 @@ public class TypeRegistry {
/**
* Find a type by its typeUrl. Returns null if it cannot be found in this {@link TypeRegistry}.
*/
/* @Nullable */
public final Descriptor getDescriptorForTypeUrl(String typeUrl)
throws InvalidProtocolBufferException {
return find(getTypeName(typeUrl));

21
java/core/src/main/java/com/google/protobuf/UnsafeUtil.java
View File

@ -41,7 +41,6 @@ import java.util.logging.Logger;
/** Utility class for working with unsafe operations. */
final class UnsafeUtil {
private static final Logger logger = Logger.getLogger(UnsafeUtil.class.getName());
private static final sun.misc.Unsafe UNSAFE = getUnsafe();
private static final Class<?> MEMORY_CLASS = Android.getMemoryClass();
private static final boolean IS_ANDROID_64 = determineAndroidSupportByAddressSize(long.class);
@ -363,9 +362,13 @@ final class UnsafeUtil {
return true;
} catch (Throwable e) {
logger.log(
Level.WARNING,
"platform method missing - proto runtime falling back to safer methods: " + e);
// Because log statements are fairly sparse in this class, this logger is initialized
// non-statically. Static initialization adds undue runtime costs to the first client to
// initialize this class.
Logger.getLogger(UnsafeUtil.class.getName())
.log(
Level.WARNING,
"platform method missing - proto runtime falling back to safer methods: " + e);
}
return false;
}
@ -397,9 +400,13 @@ final class UnsafeUtil {
clazz.getMethod("copyMemory", Object.class, long.class, Object.class, long.class, long.class);
return true;
} catch (Throwable e) {
logger.log(
Level.WARNING,
"platform method missing - proto runtime falling back to safer methods: " + e);
// Because log statements are fairly sparse in this class, this logger is initialized
// non-statically. Static initialization adds undue runtime costs to the first client to
// initialize this class.
Logger.getLogger(UnsafeUtil.class.getName())
.log(
Level.WARNING,
"platform method missing - proto runtime falling back to safer methods: " + e);
}
return false;
}

14
java/core/src/test/java/com/google/protobuf/LiteralByteStringTest.java
View File

@ -514,6 +514,20 @@ public class LiteralByteStringTest extends TestCase {
assertEquals(classUnderTest + " InputStream must now be exhausted", -1, input.read());
}
public void testNewInput_readZeroBytes() throws IOException {
InputStream input = stringUnderTest.newInput();
assertEquals(
classUnderTest + " InputStream.read() returns 0 when told to read 0 bytes and not at EOF",
0,
input.read(new byte[0]));
input.skip(input.available());
assertEquals(
classUnderTest + " InputStream.read() returns -1 when told to read 0 bytes at EOF",
-1,
input.read(new byte[0]));
}
public void testNewInput_skip() throws IOException {
InputStream input = stringUnderTest.newInput();
int stringSize = stringUnderTest.size();

165
java/core/src/test/java/com/google/protobuf/MapForProto2LiteTest.java
View File

@ -77,44 +77,44 @@ public final class MapForProto2LiteTest extends TestCase {
private void copyMapValues(TestMap source, TestMap.Builder destination) {
destination
.putAllInt32ToInt32Field(source.getInt32ToInt32Field())
.putAllInt32ToStringField(source.getInt32ToStringField())
.putAllInt32ToBytesField(source.getInt32ToBytesField())
.putAllInt32ToEnumField(source.getInt32ToEnumField())
.putAllInt32ToMessageField(source.getInt32ToMessageField())
.putAllStringToInt32Field(source.getStringToInt32Field());
.putAllInt32ToInt32Field(source.getInt32ToInt32FieldMap())
.putAllInt32ToStringField(source.getInt32ToStringFieldMap())
.putAllInt32ToBytesField(source.getInt32ToBytesFieldMap())
.putAllInt32ToEnumField(source.getInt32ToEnumFieldMap())
.putAllInt32ToMessageField(source.getInt32ToMessageFieldMap())
.putAllStringToInt32Field(source.getStringToInt32FieldMap());
}
private void assertMapValuesSet(TestMap message) {
assertEquals(3, message.getInt32ToInt32Field().size());
assertEquals(11, message.getInt32ToInt32Field().get(1).intValue());
assertEquals(22, message.getInt32ToInt32Field().get(2).intValue());
assertEquals(33, message.getInt32ToInt32Field().get(3).intValue());
assertEquals(3, message.getInt32ToInt32FieldMap().size());
assertEquals(11, message.getInt32ToInt32FieldMap().get(1).intValue());
assertEquals(22, message.getInt32ToInt32FieldMap().get(2).intValue());
assertEquals(33, message.getInt32ToInt32FieldMap().get(3).intValue());
assertEquals(3, message.getInt32ToStringField().size());
assertEquals("11", message.getInt32ToStringField().get(1));
assertEquals("22", message.getInt32ToStringField().get(2));
assertEquals("33", message.getInt32ToStringField().get(3));
assertEquals(3, message.getInt32ToStringFieldMap().size());
assertEquals("11", message.getInt32ToStringFieldMap().get(1));
assertEquals("22", message.getInt32ToStringFieldMap().get(2));
assertEquals("33", message.getInt32ToStringFieldMap().get(3));
assertEquals(3, message.getInt32ToBytesField().size());
assertEquals(TestUtil.toBytes("11"), message.getInt32ToBytesField().get(1));
assertEquals(TestUtil.toBytes("22"), message.getInt32ToBytesField().get(2));
assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesField().get(3));
assertEquals(3, message.getInt32ToBytesFieldMap().size());
assertEquals(TestUtil.toBytes("11"), message.getInt32ToBytesFieldMap().get(1));
assertEquals(TestUtil.toBytes("22"), message.getInt32ToBytesFieldMap().get(2));
assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesFieldMap().get(3));
assertEquals(3, message.getInt32ToEnumField().size());
assertEquals(TestMap.EnumValue.FOO, message.getInt32ToEnumField().get(1));
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(2));
assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumField().get(3));
assertEquals(3, message.getInt32ToEnumFieldMap().size());
assertEquals(TestMap.EnumValue.FOO, message.getInt32ToEnumFieldMap().get(1));
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumFieldMap().get(2));
assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumFieldMap().get(3));
assertEquals(3, message.getInt32ToMessageField().size());
assertEquals(11, message.getInt32ToMessageField().get(1).getValue());
assertEquals(22, message.getInt32ToMessageField().get(2).getValue());
assertEquals(33, message.getInt32ToMessageField().get(3).getValue());
assertEquals(3, message.getInt32ToMessageFieldMap().size());
assertEquals(11, message.getInt32ToMessageFieldMap().get(1).getValue());
assertEquals(22, message.getInt32ToMessageFieldMap().get(2).getValue());
assertEquals(33, message.getInt32ToMessageFieldMap().get(3).getValue());
assertEquals(3, message.getStringToInt32Field().size());
assertEquals(11, message.getStringToInt32Field().get("1").intValue());
assertEquals(22, message.getStringToInt32Field().get("2").intValue());
assertEquals(33, message.getStringToInt32Field().get("3").intValue());
assertEquals(3, message.getStringToInt32FieldMap().size());
assertEquals(11, message.getStringToInt32FieldMap().get("1").intValue());
assertEquals(22, message.getStringToInt32FieldMap().get("2").intValue());
assertEquals(33, message.getStringToInt32FieldMap().get("3").intValue());
}
private void updateMapValues(TestMap.Builder builder) {
@ -152,35 +152,35 @@ public final class MapForProto2LiteTest extends TestCase {
}
private void assertMapValuesUpdated(TestMap message) {
assertEquals(3, message.getInt32ToInt32Field().size());
assertEquals(111, message.getInt32ToInt32Field().get(1).intValue());
assertEquals(33, message.getInt32ToInt32Field().get(3).intValue());
assertEquals(44, message.getInt32ToInt32Field().get(4).intValue());
assertEquals(3, message.getInt32ToInt32FieldMap().size());
assertEquals(111, message.getInt32ToInt32FieldMap().get(1).intValue());
assertEquals(33, message.getInt32ToInt32FieldMap().get(3).intValue());
assertEquals(44, message.getInt32ToInt32FieldMap().get(4).intValue());
assertEquals(3, message.getInt32ToStringField().size());
assertEquals("111", message.getInt32ToStringField().get(1));
assertEquals("33", message.getInt32ToStringField().get(3));
assertEquals("44", message.getInt32ToStringField().get(4));
assertEquals(3, message.getInt32ToStringFieldMap().size());
assertEquals("111", message.getInt32ToStringFieldMap().get(1));
assertEquals("33", message.getInt32ToStringFieldMap().get(3));
assertEquals("44", message.getInt32ToStringFieldMap().get(4));
assertEquals(3, message.getInt32ToBytesField().size());
assertEquals(TestUtil.toBytes("111"), message.getInt32ToBytesField().get(1));
assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesField().get(3));
assertEquals(TestUtil.toBytes("44"), message.getInt32ToBytesField().get(4));
assertEquals(3, message.getInt32ToBytesFieldMap().size());
assertEquals(TestUtil.toBytes("111"), message.getInt32ToBytesFieldMap().get(1));
assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesFieldMap().get(3));
assertEquals(TestUtil.toBytes("44"), message.getInt32ToBytesFieldMap().get(4));
assertEquals(3, message.getInt32ToEnumField().size());
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(1));
assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumField().get(3));
assertEquals(TestMap.EnumValue.QUX, message.getInt32ToEnumField().get(4));
assertEquals(3, message.getInt32ToEnumFieldMap().size());
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumFieldMap().get(1));
assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumFieldMap().get(3));
assertEquals(TestMap.EnumValue.QUX, message.getInt32ToEnumFieldMap().get(4));
assertEquals(3, message.getInt32ToMessageField().size());
assertEquals(111, message.getInt32ToMessageField().get(1).getValue());
assertEquals(33, message.getInt32ToMessageField().get(3).getValue());
assertEquals(44, message.getInt32ToMessageField().get(4).getValue());
assertEquals(3, message.getInt32ToMessageFieldMap().size());
assertEquals(111, message.getInt32ToMessageFieldMap().get(1).getValue());
assertEquals(33, message.getInt32ToMessageFieldMap().get(3).getValue());
assertEquals(44, message.getInt32ToMessageFieldMap().get(4).getValue());
assertEquals(3, message.getStringToInt32Field().size());
assertEquals(111, message.getStringToInt32Field().get("1").intValue());
assertEquals(33, message.getStringToInt32Field().get("3").intValue());
assertEquals(44, message.getStringToInt32Field().get("4").intValue());
assertEquals(3, message.getStringToInt32FieldMap().size());
assertEquals(111, message.getStringToInt32FieldMap().get("1").intValue());
assertEquals(33, message.getStringToInt32FieldMap().get("3").intValue());
assertEquals(44, message.getStringToInt32FieldMap().get("4").intValue());
}
private void assertMapValuesCleared(TestMapOrBuilder testMapOrBuilder) {
@ -206,13 +206,13 @@ public final class MapForProto2LiteTest extends TestCase {
TestMap.Builder builder = TestMap.newBuilder();
TestMap message = builder.build();
builder.putInt32ToInt32Field(1, 2);
assertTrue(message.getInt32ToInt32Field().isEmpty());
assertTrue(message.getInt32ToInt32FieldMap().isEmpty());
message = builder.build();
assertEquals(newMap(1, 2), message.getInt32ToInt32Field());
assertEquals(newMap(1, 2), builder.getInt32ToInt32Field());
assertEquals(newMap(1, 2), message.getInt32ToInt32FieldMap());
assertEquals(newMap(1, 2), builder.getInt32ToInt32FieldMap());
builder.putInt32ToInt32Field(2, 3);
assertEquals(newMap(1, 2), message.getInt32ToInt32Field());
assertEquals(newMap(1, 2, 2, 3), builder.getInt32ToInt32Field());
assertEquals(newMap(1, 2), message.getInt32ToInt32FieldMap());
assertEquals(newMap(1, 2, 2, 3), builder.getInt32ToInt32FieldMap());
}
public void testGetMapIsImmutable() {
@ -254,30 +254,31 @@ public final class MapForProto2LiteTest extends TestCase {
public void testMutableMapLifecycle() {
TestMap.Builder builder = TestMap.newBuilder().putInt32ToInt32Field(1, 2);
assertEquals(newMap(1, 2), builder.build().getInt32ToInt32Field());
assertEquals(newMap(1, 2), builder.getInt32ToInt32Field());
assertEquals(newMap(1, 2), builder.build().getInt32ToInt32FieldMap());
assertEquals(newMap(1, 2), builder.getInt32ToInt32FieldMap());
builder.putInt32ToInt32Field(2, 3);
assertEquals(newMap(1, 2, 2, 3), builder.getInt32ToInt32Field());
assertEquals(newMap(1, 2, 2, 3), builder.getInt32ToInt32FieldMap());
builder.putInt32ToEnumField(1, TestMap.EnumValue.BAR);
assertEquals(newMap(1, TestMap.EnumValue.BAR), builder.build().getInt32ToEnumField());
assertEquals(newMap(1, TestMap.EnumValue.BAR), builder.getInt32ToEnumField());
assertEquals(newMap(1, TestMap.EnumValue.BAR), builder.build().getInt32ToEnumFieldMap());
assertEquals(newMap(1, TestMap.EnumValue.BAR), builder.getInt32ToEnumFieldMap());
builder.putInt32ToEnumField(2, TestMap.EnumValue.FOO);
assertEquals(
newMap(1, TestMap.EnumValue.BAR, 2, TestMap.EnumValue.FOO), builder.getInt32ToEnumField());
newMap(1, TestMap.EnumValue.BAR, 2, TestMap.EnumValue.FOO),
builder.getInt32ToEnumFieldMap());
builder.putInt32ToStringField(1, "1");
assertEquals(newMap(1, "1"), builder.build().getInt32ToStringField());
assertEquals(newMap(1, "1"), builder.getInt32ToStringField());
assertEquals(newMap(1, "1"), builder.build().getInt32ToStringFieldMap());
assertEquals(newMap(1, "1"), builder.getInt32ToStringFieldMap());
builder.putInt32ToStringField(2, "2");
assertEquals(newMap(1, "1", 2, "2"), builder.getInt32ToStringField());
assertEquals(newMap(1, "1", 2, "2"), builder.getInt32ToStringFieldMap());
builder.putInt32ToMessageField(1, TestMap.MessageValue.getDefaultInstance());
assertEquals(
newMap(1, TestMap.MessageValue.getDefaultInstance()),
builder.build().getInt32ToMessageField());
builder.build().getInt32ToMessageFieldMap());
assertEquals(
newMap(1, TestMap.MessageValue.getDefaultInstance()), builder.getInt32ToMessageField());
newMap(1, TestMap.MessageValue.getDefaultInstance()), builder.getInt32ToMessageFieldMap());
builder.putInt32ToMessageField(2, TestMap.MessageValue.getDefaultInstance());
assertEquals(
newMap(
@ -285,7 +286,7 @@ public final class MapForProto2LiteTest extends TestCase {
TestMap.MessageValue.getDefaultInstance(),
2,
TestMap.MessageValue.getDefaultInstance()),
builder.getInt32ToMessageField());
builder.getInt32ToMessageFieldMap());
}
public void testGettersAndSetters() throws Exception {
@ -415,7 +416,7 @@ public final class MapForProto2LiteTest extends TestCase {
} catch (InvalidProtocolBufferException expected) {
assertTrue(expected.getUnfinishedMessage() instanceof TestMap);
map = (TestMap) expected.getUnfinishedMessage();
assertTrue(map.getInt32ToMessageField().isEmpty());
assertTrue(map.getInt32ToMessageFieldMap().isEmpty());
}
map =
@ -476,14 +477,14 @@ public final class MapForProto2LiteTest extends TestCase {
TestMap message = TestMap.parseFrom(data);
// Entries with unknown enum values will be stored into UnknownFieldSet so
// there is only one entry in the map.
assertEquals(1, message.getInt32ToEnumField().size());
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(1));
assertEquals(1, message.getInt32ToEnumFieldMap().size());
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumFieldMap().get(1));
// Serializing and parsing should preserve the unknown entry.
data = message.toByteString();
TestUnknownEnumValue messageWithUnknownEnums = TestUnknownEnumValue.parseFrom(data);
assertEquals(2, messageWithUnknownEnums.getInt32ToInt32Field().size());
assertEquals(1, messageWithUnknownEnums.getInt32ToInt32Field().get(1).intValue());
assertEquals(54321, messageWithUnknownEnums.getInt32ToInt32Field().get(2).intValue());
assertEquals(2, messageWithUnknownEnums.getInt32ToInt32FieldMap().size());
assertEquals(1, messageWithUnknownEnums.getInt32ToInt32FieldMap().get(1).intValue());
assertEquals(54321, messageWithUnknownEnums.getInt32ToInt32FieldMap().get(2).intValue());
}
public void testIterationOrder() throws Exception {
@ -493,7 +494,7 @@ public final class MapForProto2LiteTest extends TestCase {
assertEquals(
Arrays.asList("1", "2", "3"),
new ArrayList<String>(message.getStringToInt32Field().keySet()));
new ArrayList<String>(message.getStringToInt32FieldMap().keySet()));
}
private static <K, V> Map<K, V> newMap(K key1, V value1) {
@ -513,10 +514,10 @@ public final class MapForProto2LiteTest extends TestCase {
TestMap.Builder builder = TestMap.newBuilder();
setMapValues(builder);
TestMap message = builder.build();
assertEquals(message.getStringToInt32Field(), message.getStringToInt32FieldMap());
assertEquals(message.getInt32ToBytesField(), message.getInt32ToBytesFieldMap());
assertEquals(message.getInt32ToEnumField(), message.getInt32ToEnumFieldMap());
assertEquals(message.getInt32ToMessageField(), message.getInt32ToMessageFieldMap());
assertEquals(message.getStringToInt32FieldMap(), message.getStringToInt32FieldMap());
assertEquals(message.getInt32ToBytesFieldMap(), message.getInt32ToBytesFieldMap());
assertEquals(message.getInt32ToEnumFieldMap(), message.getInt32ToEnumFieldMap());
assertEquals(message.getInt32ToMessageFieldMap(), message.getInt32ToMessageFieldMap());
}
public void testContains() {

108
java/core/src/test/java/com/google/protobuf/MapForProto2Test.java
View File

@ -119,12 +119,12 @@ public class MapForProto2Test extends TestCase {
private void copyMapValues(TestMap source, TestMap.Builder destination) {
destination
.putAllInt32ToInt32Field(source.getInt32ToInt32Field())
.putAllInt32ToStringField(source.getInt32ToStringField())
.putAllInt32ToBytesField(source.getInt32ToBytesField())
.putAllInt32ToEnumField(source.getInt32ToEnumField())
.putAllInt32ToMessageField(source.getInt32ToMessageField())
.putAllStringToInt32Field(source.getStringToInt32Field());
.putAllInt32ToInt32Field(source.getInt32ToInt32FieldMap())
.putAllInt32ToStringField(source.getInt32ToStringFieldMap())
.putAllInt32ToBytesField(source.getInt32ToBytesFieldMap())
.putAllInt32ToEnumField(source.getInt32ToEnumFieldMap())
.putAllInt32ToMessageField(source.getInt32ToMessageFieldMap())
.putAllStringToInt32Field(source.getStringToInt32FieldMap());
}
private void assertMapValuesSet(TestMapOrBuilder message) {
@ -236,35 +236,35 @@ public class MapForProto2Test extends TestCase {
}
private void assertMapValuesUpdated(TestMap message) {
assertEquals(3, message.getInt32ToInt32Field().size());
assertEquals(111, message.getInt32ToInt32Field().get(1).intValue());
assertEquals(33, message.getInt32ToInt32Field().get(3).intValue());
assertEquals(44, message.getInt32ToInt32Field().get(4).intValue());
assertEquals(3, message.getInt32ToInt32FieldMap().size());
assertEquals(111, message.getInt32ToInt32FieldMap().get(1).intValue());
assertEquals(33, message.getInt32ToInt32FieldMap().get(3).intValue());
assertEquals(44, message.getInt32ToInt32FieldMap().get(4).intValue());
assertEquals(3, message.getInt32ToStringField().size());
assertEquals("111", message.getInt32ToStringField().get(1));
assertEquals("33", message.getInt32ToStringField().get(3));
assertEquals("44", message.getInt32ToStringField().get(4));
assertEquals(3, message.getInt32ToStringFieldMap().size());
assertEquals("111", message.getInt32ToStringFieldMap().get(1));
assertEquals("33", message.getInt32ToStringFieldMap().get(3));
assertEquals("44", message.getInt32ToStringFieldMap().get(4));
assertEquals(3, message.getInt32ToBytesField().size());
assertEquals(TestUtil.toBytes("111"), message.getInt32ToBytesField().get(1));
assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesField().get(3));
assertEquals(TestUtil.toBytes("44"), message.getInt32ToBytesField().get(4));
assertEquals(3, message.getInt32ToBytesFieldMap().size());
assertEquals(TestUtil.toBytes("111"), message.getInt32ToBytesFieldMap().get(1));
assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesFieldMap().get(3));
assertEquals(TestUtil.toBytes("44"), message.getInt32ToBytesFieldMap().get(4));
assertEquals(3, message.getInt32ToEnumField().size());
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(1));
assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumField().get(3));
assertEquals(TestMap.EnumValue.QUX, message.getInt32ToEnumField().get(4));
assertEquals(3, message.getInt32ToEnumFieldMap().size());
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumFieldMap().get(1));
assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumFieldMap().get(3));
assertEquals(TestMap.EnumValue.QUX, message.getInt32ToEnumFieldMap().get(4));
assertEquals(3, message.getInt32ToMessageField().size());
assertEquals(111, message.getInt32ToMessageField().get(1).getValue());
assertEquals(33, message.getInt32ToMessageField().get(3).getValue());
assertEquals(44, message.getInt32ToMessageField().get(4).getValue());
assertEquals(3, message.getInt32ToMessageFieldMap().size());
assertEquals(111, message.getInt32ToMessageFieldMap().get(1).getValue());
assertEquals(33, message.getInt32ToMessageFieldMap().get(3).getValue());
assertEquals(44, message.getInt32ToMessageFieldMap().get(4).getValue());
assertEquals(3, message.getStringToInt32Field().size());
assertEquals(111, message.getStringToInt32Field().get("1").intValue());
assertEquals(33, message.getStringToInt32Field().get("3").intValue());
assertEquals(44, message.getStringToInt32Field().get("4").intValue());
assertEquals(3, message.getStringToInt32FieldMap().size());
assertEquals(111, message.getStringToInt32FieldMap().get("1").intValue());
assertEquals(33, message.getStringToInt32FieldMap().get("3").intValue());
assertEquals(44, message.getStringToInt32FieldMap().get("4").intValue());
}
private void assertMapValuesCleared(TestMapOrBuilder testMapOrBuilder) {
@ -563,7 +563,7 @@ public class MapForProto2Test extends TestCase {
} catch (InvalidProtocolBufferException expected) {
assertTrue(expected.getUnfinishedMessage() instanceof TestMap);
map = (TestMap) expected.getUnfinishedMessage();
assertTrue(map.getInt32ToMessageField().isEmpty());
assertTrue(map.getInt32ToMessageFieldMap().isEmpty());
}
map =
@ -698,8 +698,8 @@ public class MapForProto2Test extends TestCase {
builder.clearField(f("int32_to_int32_field"));
builder.clearField(f("int32_to_message_field"));
message = builder.build();
assertEquals(0, message.getInt32ToInt32Field().size());
assertEquals(0, message.getInt32ToMessageField().size());
assertEquals(0, message.getInt32ToInt32FieldMap().size());
assertEquals(0, message.getInt32ToMessageFieldMap().size());
// Test setField()
setMapValues(builder, "int32_to_int32_field", mapForValues(11, 22, 33, 44));
@ -710,10 +710,10 @@ public class MapForProto2Test extends TestCase {
111, MessageValue.newBuilder().setValue(222).build(),
333, MessageValue.newBuilder().setValue(444).build()));
message = builder.build();
assertEquals(22, message.getInt32ToInt32Field().get(11).intValue());
assertEquals(44, message.getInt32ToInt32Field().get(33).intValue());
assertEquals(222, message.getInt32ToMessageField().get(111).getValue());
assertEquals(444, message.getInt32ToMessageField().get(333).getValue());
assertEquals(22, message.getInt32ToInt32FieldMap().get(11).intValue());
assertEquals(44, message.getInt32ToInt32FieldMap().get(33).intValue());
assertEquals(222, message.getInt32ToMessageFieldMap().get(111).getValue());
assertEquals(444, message.getInt32ToMessageFieldMap().get(333).getValue());
// Test addRepeatedField
builder.addRepeatedField(
@ -726,8 +726,8 @@ public class MapForProto2Test extends TestCase {
555,
MessageValue.newBuilder().setValue(666).build()));
message = builder.build();
assertEquals(66, message.getInt32ToInt32Field().get(55).intValue());
assertEquals(666, message.getInt32ToMessageField().get(555).getValue());
assertEquals(66, message.getInt32ToInt32FieldMap().get(55).intValue());
assertEquals(666, message.getInt32ToMessageFieldMap().get(555).getValue());
// Test addRepeatedField (overriding existing values)
builder.addRepeatedField(
@ -740,8 +740,8 @@ public class MapForProto2Test extends TestCase {
555,
MessageValue.newBuilder().setValue(555).build()));
message = builder.build();
assertEquals(55, message.getInt32ToInt32Field().get(55).intValue());
assertEquals(555, message.getInt32ToMessageField().get(555).getValue());
assertEquals(55, message.getInt32ToInt32FieldMap().get(55).intValue());
assertEquals(555, message.getInt32ToMessageFieldMap().get(555).getValue());
// Test setRepeatedField
for (int i = 0; i < builder.getRepeatedFieldCount(f("int32_to_int32_field")); i++) {
@ -755,9 +755,9 @@ public class MapForProto2Test extends TestCase {
builder.setRepeatedField(f("int32_to_int32_field"), i, mapEntryBuilder.build());
}
message = builder.build();
assertEquals(11, message.getInt32ToInt32Field().get(22).intValue());
assertEquals(33, message.getInt32ToInt32Field().get(44).intValue());
assertEquals(55, message.getInt32ToInt32Field().get(55).intValue());
assertEquals(11, message.getInt32ToInt32FieldMap().get(22).intValue());
assertEquals(33, message.getInt32ToInt32FieldMap().get(44).intValue());
assertEquals(55, message.getInt32ToInt32FieldMap().get(55).intValue());
}
// See additional coverage in TextFormatTest.java.
@ -844,16 +844,16 @@ public class MapForProto2Test extends TestCase {
TestMap message = TestMap.parseFrom(data);
// Entries with unknown enum values will be stored into UnknownFieldSet so
// there is only one entry in the map.
assertEquals(1, message.getInt32ToEnumField().size());
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(1));
assertEquals(1, message.getInt32ToEnumFieldMap().size());
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumFieldMap().get(1));
// UnknownFieldSet should not be empty.
assertFalse(message.getUnknownFields().asMap().isEmpty());
// Serializing and parsing should preserve the unknown entry.
data = message.toByteString();
TestUnknownEnumValue messageWithUnknownEnums = TestUnknownEnumValue.parseFrom(data);
assertEquals(2, messageWithUnknownEnums.getInt32ToInt32Field().size());
assertEquals(1, messageWithUnknownEnums.getInt32ToInt32Field().get(1).intValue());
assertEquals(54321, messageWithUnknownEnums.getInt32ToInt32Field().get(2).intValue());
assertEquals(2, messageWithUnknownEnums.getInt32ToInt32FieldMap().size());
assertEquals(1, messageWithUnknownEnums.getInt32ToInt32FieldMap().get(1).intValue());
assertEquals(54321, messageWithUnknownEnums.getInt32ToInt32FieldMap().get(2).intValue());
}
public void testRequiredMessage() throws Exception {
@ -885,7 +885,7 @@ public class MapForProto2Test extends TestCase {
assertEquals(
Arrays.asList("1", "2", "3"),
new ArrayList<String>(message.getStringToInt32Field().keySet()));
new ArrayList<String>(message.getStringToInt32FieldMap().keySet()));
}
public void testContains() {
@ -1172,9 +1172,9 @@ public class MapForProto2Test extends TestCase {
setMapValuesUsingAccessors(builder);
assertMapValuesSet(builder);
TestMap message = builder.build();
assertEquals(message.getStringToInt32Field(), message.getStringToInt32FieldMap());
assertEquals(message.getInt32ToBytesField(), message.getInt32ToBytesFieldMap());
assertEquals(message.getInt32ToEnumField(), message.getInt32ToEnumFieldMap());
assertEquals(message.getInt32ToMessageField(), message.getInt32ToMessageFieldMap());
assertEquals(message.getStringToInt32FieldMap(), message.getStringToInt32FieldMap());
assertEquals(message.getInt32ToBytesFieldMap(), message.getInt32ToBytesFieldMap());
assertEquals(message.getInt32ToEnumFieldMap(), message.getInt32ToEnumFieldMap());
assertEquals(message.getInt32ToMessageFieldMap(), message.getInt32ToMessageFieldMap());
}
}

179
java/core/src/test/java/com/google/protobuf/MapLiteTest.java
View File

@ -84,44 +84,44 @@ public final class MapLiteTest extends TestCase {
private void copyMapValues(TestMap source, TestMap.Builder destination) {
destination
.putAllInt32ToInt32Field(source.getInt32ToInt32Field())
.putAllInt32ToStringField(source.getInt32ToStringField())
.putAllInt32ToBytesField(source.getInt32ToBytesField())
.putAllInt32ToEnumField(source.getInt32ToEnumField())
.putAllInt32ToMessageField(source.getInt32ToMessageField())
.putAllStringToInt32Field(source.getStringToInt32Field());
.putAllInt32ToInt32Field(source.getInt32ToInt32FieldMap())
.putAllInt32ToStringField(source.getInt32ToStringFieldMap())
.putAllInt32ToBytesField(source.getInt32ToBytesFieldMap())
.putAllInt32ToEnumField(source.getInt32ToEnumFieldMap())
.putAllInt32ToMessageField(source.getInt32ToMessageFieldMap())
.putAllStringToInt32Field(source.getStringToInt32FieldMap());
}
private void assertMapValuesSet(TestMap message) {
assertEquals(3, message.getInt32ToInt32Field().size());
assertEquals(11, message.getInt32ToInt32Field().get(1).intValue());
assertEquals(22, message.getInt32ToInt32Field().get(2).intValue());
assertEquals(33, message.getInt32ToInt32Field().get(3).intValue());
assertEquals(3, message.getInt32ToInt32FieldMap().size());
assertEquals(11, message.getInt32ToInt32FieldMap().get(1).intValue());
assertEquals(22, message.getInt32ToInt32FieldMap().get(2).intValue());
assertEquals(33, message.getInt32ToInt32FieldMap().get(3).intValue());
assertEquals(3, message.getInt32ToStringField().size());
assertEquals("11", message.getInt32ToStringField().get(1));
assertEquals("22", message.getInt32ToStringField().get(2));
assertEquals("33", message.getInt32ToStringField().get(3));
assertEquals(3, message.getInt32ToStringFieldMap().size());
assertEquals("11", message.getInt32ToStringFieldMap().get(1));
assertEquals("22", message.getInt32ToStringFieldMap().get(2));
assertEquals("33", message.getInt32ToStringFieldMap().get(3));
assertEquals(3, message.getInt32ToBytesField().size());
assertEquals(TestUtil.toBytes("11"), message.getInt32ToBytesField().get(1));
assertEquals(TestUtil.toBytes("22"), message.getInt32ToBytesField().get(2));
assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesField().get(3));
assertEquals(3, message.getInt32ToBytesFieldMap().size());
assertEquals(TestUtil.toBytes("11"), message.getInt32ToBytesFieldMap().get(1));
assertEquals(TestUtil.toBytes("22"), message.getInt32ToBytesFieldMap().get(2));
assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesFieldMap().get(3));
assertEquals(3, message.getInt32ToEnumField().size());
assertEquals(TestMap.EnumValue.FOO, message.getInt32ToEnumField().get(1));
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(2));
assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumField().get(3));
assertEquals(3, message.getInt32ToEnumFieldMap().size());
assertEquals(TestMap.EnumValue.FOO, message.getInt32ToEnumFieldMap().get(1));
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumFieldMap().get(2));
assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumFieldMap().get(3));
assertEquals(3, message.getInt32ToMessageField().size());
assertEquals(11, message.getInt32ToMessageField().get(1).getValue());
assertEquals(22, message.getInt32ToMessageField().get(2).getValue());
assertEquals(33, message.getInt32ToMessageField().get(3).getValue());
assertEquals(3, message.getInt32ToMessageFieldMap().size());
assertEquals(11, message.getInt32ToMessageFieldMap().get(1).getValue());
assertEquals(22, message.getInt32ToMessageFieldMap().get(2).getValue());
assertEquals(33, message.getInt32ToMessageFieldMap().get(3).getValue());
assertEquals(3, message.getStringToInt32Field().size());
assertEquals(11, message.getStringToInt32Field().get("1").intValue());
assertEquals(22, message.getStringToInt32Field().get("2").intValue());
assertEquals(33, message.getStringToInt32Field().get("3").intValue());
assertEquals(3, message.getStringToInt32FieldMap().size());
assertEquals(11, message.getStringToInt32FieldMap().get("1").intValue());
assertEquals(22, message.getStringToInt32FieldMap().get("2").intValue());
assertEquals(33, message.getStringToInt32FieldMap().get("3").intValue());
}
private void updateMapValues(TestMap.Builder builder) {
@ -159,35 +159,35 @@ public final class MapLiteTest extends TestCase {
}
private void assertMapValuesUpdated(TestMap message) {
assertEquals(3, message.getInt32ToInt32Field().size());
assertEquals(111, message.getInt32ToInt32Field().get(1).intValue());
assertEquals(33, message.getInt32ToInt32Field().get(3).intValue());
assertEquals(44, message.getInt32ToInt32Field().get(4).intValue());
assertEquals(3, message.getInt32ToInt32FieldMap().size());
assertEquals(111, message.getInt32ToInt32FieldMap().get(1).intValue());
assertEquals(33, message.getInt32ToInt32FieldMap().get(3).intValue());
assertEquals(44, message.getInt32ToInt32FieldMap().get(4).intValue());
assertEquals(3, message.getInt32ToStringField().size());
assertEquals("111", message.getInt32ToStringField().get(1));
assertEquals("33", message.getInt32ToStringField().get(3));
assertEquals("44", message.getInt32ToStringField().get(4));
assertEquals(3, message.getInt32ToStringFieldMap().size());
assertEquals("111", message.getInt32ToStringFieldMap().get(1));
assertEquals("33", message.getInt32ToStringFieldMap().get(3));
assertEquals("44", message.getInt32ToStringFieldMap().get(4));
assertEquals(3, message.getInt32ToBytesField().size());
assertEquals(TestUtil.toBytes("111"), message.getInt32ToBytesField().get(1));
assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesField().get(3));
assertEquals(TestUtil.toBytes("44"), message.getInt32ToBytesField().get(4));
assertEquals(3, message.getInt32ToBytesFieldMap().size());
assertEquals(TestUtil.toBytes("111"), message.getInt32ToBytesFieldMap().get(1));
assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesFieldMap().get(3));
assertEquals(TestUtil.toBytes("44"), message.getInt32ToBytesFieldMap().get(4));
assertEquals(3, message.getInt32ToEnumField().size());
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(1));
assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumField().get(3));
assertEquals(TestMap.EnumValue.QUX, message.getInt32ToEnumField().get(4));
assertEquals(3, message.getInt32ToEnumFieldMap().size());
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumFieldMap().get(1));
assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumFieldMap().get(3));
assertEquals(TestMap.EnumValue.QUX, message.getInt32ToEnumFieldMap().get(4));
assertEquals(3, message.getInt32ToMessageField().size());
assertEquals(111, message.getInt32ToMessageField().get(1).getValue());
assertEquals(33, message.getInt32ToMessageField().get(3).getValue());
assertEquals(44, message.getInt32ToMessageField().get(4).getValue());
assertEquals(3, message.getInt32ToMessageFieldMap().size());
assertEquals(111, message.getInt32ToMessageFieldMap().get(1).getValue());
assertEquals(33, message.getInt32ToMessageFieldMap().get(3).getValue());
assertEquals(44, message.getInt32ToMessageFieldMap().get(4).getValue());
assertEquals(3, message.getStringToInt32Field().size());
assertEquals(111, message.getStringToInt32Field().get("1").intValue());
assertEquals(33, message.getStringToInt32Field().get("3").intValue());
assertEquals(44, message.getStringToInt32Field().get("4").intValue());
assertEquals(3, message.getStringToInt32FieldMap().size());
assertEquals(111, message.getStringToInt32FieldMap().get("1").intValue());
assertEquals(33, message.getStringToInt32FieldMap().get("3").intValue());
assertEquals(44, message.getStringToInt32FieldMap().get("4").intValue());
}
private void assertMapValuesCleared(TestMapOrBuilder testMapOrBuilder) {
@ -213,12 +213,12 @@ public final class MapLiteTest extends TestCase {
TestMap.Builder builder = TestMap.newBuilder();
TestMap message = builder.build();
builder.putInt32ToInt32Field(1, 2);
assertTrue(message.getInt32ToInt32Field().isEmpty());
assertEquals(newMap(1, 2), builder.getInt32ToInt32Field());
assertTrue(message.getInt32ToInt32FieldMap().isEmpty());
assertEquals(newMap(1, 2), builder.getInt32ToInt32FieldMap());
message = builder.build();
builder.putInt32ToInt32Field(2, 3);
assertEquals(newMap(1, 2), message.getInt32ToInt32Field());
assertEquals(newMap(1, 2, 2, 3), builder.getInt32ToInt32Field());
assertEquals(newMap(1, 2), message.getInt32ToInt32FieldMap());
assertEquals(newMap(1, 2, 2, 3), builder.getInt32ToInt32FieldMap());
}
public void testGetMapIsImmutable() {
@ -266,30 +266,31 @@ public final class MapLiteTest extends TestCase {
public void testMutableMapLifecycle() {
TestMap.Builder builder = TestMap.newBuilder().putInt32ToInt32Field(1, 2);
assertEquals(newMap(1, 2), builder.build().getInt32ToInt32Field());
assertEquals(newMap(1, 2), builder.getInt32ToInt32Field());
assertEquals(newMap(1, 2), builder.build().getInt32ToInt32FieldMap());
assertEquals(newMap(1, 2), builder.getInt32ToInt32FieldMap());
builder.putInt32ToInt32Field(2, 3);
assertEquals(newMap(1, 2, 2, 3), builder.getInt32ToInt32Field());
assertEquals(newMap(1, 2, 2, 3), builder.getInt32ToInt32FieldMap());
builder.putInt32ToEnumField(1, TestMap.EnumValue.BAR);
assertEquals(newMap(1, TestMap.EnumValue.BAR), builder.build().getInt32ToEnumField());
assertEquals(newMap(1, TestMap.EnumValue.BAR), builder.getInt32ToEnumField());
assertEquals(newMap(1, TestMap.EnumValue.BAR), builder.build().getInt32ToEnumFieldMap());
assertEquals(newMap(1, TestMap.EnumValue.BAR), builder.getInt32ToEnumFieldMap());
builder.putInt32ToEnumField(2, TestMap.EnumValue.FOO);
assertEquals(
newMap(1, TestMap.EnumValue.BAR, 2, TestMap.EnumValue.FOO), builder.getInt32ToEnumField());
newMap(1, TestMap.EnumValue.BAR, 2, TestMap.EnumValue.FOO),
builder.getInt32ToEnumFieldMap());
builder.putInt32ToStringField(1, "1");
assertEquals(newMap(1, "1"), builder.build().getInt32ToStringField());
assertEquals(newMap(1, "1"), builder.getInt32ToStringField());
assertEquals(newMap(1, "1"), builder.build().getInt32ToStringFieldMap());
assertEquals(newMap(1, "1"), builder.getInt32ToStringFieldMap());
builder.putInt32ToStringField(2, "2");
assertEquals(newMap(1, "1", 2, "2"), builder.getInt32ToStringField());
assertEquals(newMap(1, "1", 2, "2"), builder.getInt32ToStringFieldMap());
builder.putInt32ToMessageField(1, TestMap.MessageValue.getDefaultInstance());
assertEquals(
newMap(1, TestMap.MessageValue.getDefaultInstance()),
builder.build().getInt32ToMessageField());
builder.build().getInt32ToMessageFieldMap());
assertEquals(
newMap(1, TestMap.MessageValue.getDefaultInstance()), builder.getInt32ToMessageField());
newMap(1, TestMap.MessageValue.getDefaultInstance()), builder.getInt32ToMessageFieldMap());
builder.putInt32ToMessageField(2, TestMap.MessageValue.getDefaultInstance());
assertEquals(
newMap(
@ -297,7 +298,7 @@ public final class MapLiteTest extends TestCase {
TestMap.MessageValue.getDefaultInstance(),
2,
TestMap.MessageValue.getDefaultInstance()),
builder.getInt32ToMessageField());
builder.getInt32ToMessageFieldMap());
}
public void testGettersAndSetters() throws Exception {
@ -342,12 +343,12 @@ public final class MapLiteTest extends TestCase {
TestMap source = sourceBuilder.build();
TestMap.Builder destinationBuilder = TestMap.newBuilder();
destinationBuilder.putAllInt32ToEnumFieldValue(source.getInt32ToEnumFieldValue());
destinationBuilder.putAllInt32ToEnumFieldValue(source.getInt32ToEnumFieldValueMap());
TestMap destination = destinationBuilder.build();
assertEquals(0, destination.getInt32ToEnumFieldValue().get(0).intValue());
assertEquals(1, destination.getInt32ToEnumFieldValue().get(1).intValue());
assertEquals(1000, destination.getInt32ToEnumFieldValue().get(2).intValue());
assertEquals(0, destination.getInt32ToEnumFieldValueMap().get(0).intValue());
assertEquals(1, destination.getInt32ToEnumFieldValueMap().get(1).intValue());
assertEquals(1000, destination.getInt32ToEnumFieldValueMap().get(2).intValue());
assertEquals(3, destination.getInt32ToEnumFieldCount());
}
@ -459,7 +460,7 @@ public final class MapLiteTest extends TestCase {
} catch (InvalidProtocolBufferException expected) {
assertTrue(expected.getUnfinishedMessage() instanceof TestMap);
map = (TestMap) expected.getUnfinishedMessage();
assertTrue(map.getInt32ToMessageField().isEmpty());
assertTrue(map.getInt32ToMessageFieldMap().isEmpty());
}
map =
@ -524,24 +525,24 @@ public final class MapLiteTest extends TestCase {
.putInt32ToEnumFieldValue(2, 1000); // unknown value.
TestMap message = builder.build();
assertEquals(TestMap.EnumValue.FOO, message.getInt32ToEnumField().get(0));
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(1));
assertEquals(TestMap.EnumValue.UNRECOGNIZED, message.getInt32ToEnumField().get(2));
assertEquals(TestMap.EnumValue.FOO, message.getInt32ToEnumFieldMap().get(0));
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumFieldMap().get(1));
assertEquals(TestMap.EnumValue.UNRECOGNIZED, message.getInt32ToEnumFieldMap().get(2));
builder.putAllInt32ToEnumFieldValue(newMap(2, 1000)); // unknown value.
message = builder.build();
assertEquals(TestMap.EnumValue.UNRECOGNIZED, message.getInt32ToEnumField().get(2));
assertEquals(TestMap.EnumValue.UNRECOGNIZED, message.getInt32ToEnumFieldMap().get(2));
// Unknown enum values should be preserved after:
// 1. Serialization and parsing.
// 2. toBuild().
// 3. mergeFrom().
message = TestMap.parseFrom(message.toByteString());
assertEquals(1000, message.getInt32ToEnumFieldValue().get(2).intValue());
assertEquals(1000, message.getInt32ToEnumFieldValueMap().get(2).intValue());
builder = message.toBuilder();
assertEquals(1000, builder.getInt32ToEnumFieldValue().get(2).intValue());
assertEquals(1000, builder.getInt32ToEnumFieldValueMap().get(2).intValue());
builder = TestMap.newBuilder().mergeFrom(message);
assertEquals(1000, builder.getInt32ToEnumFieldValue().get(2).intValue());
assertEquals(1000, builder.getInt32ToEnumFieldValueMap().get(2).intValue());
// hashCode()/equals() should take unknown enum values into account.
builder.putAllInt32ToEnumFieldValue(newMap(2, 1001));
@ -550,7 +551,7 @@ public final class MapLiteTest extends TestCase {
assertFalse(message.equals(message2));
// Unknown values will be converted to UNRECOGNIZED so the resulted enum map
// should be the same.
assertEquals(message2.getInt32ToEnumField(), message.getInt32ToEnumField());
assertEquals(message2.getInt32ToEnumFieldMap(), message.getInt32ToEnumFieldMap());
}
public void testIterationOrder() throws Exception {
@ -559,18 +560,18 @@ public final class MapLiteTest extends TestCase {
TestMap message = builder.build();
assertEquals(
Arrays.asList("1", "2", "3"), new ArrayList<>(message.getStringToInt32Field().keySet()));
Arrays.asList("1", "2", "3"), new ArrayList<>(message.getStringToInt32FieldMap().keySet()));
}
public void testGetMap() {
TestMap.Builder builder = TestMap.newBuilder();
setMapValues(builder);
TestMap message = builder.build();
assertEquals(message.getStringToInt32Field(), message.getStringToInt32FieldMap());
assertEquals(message.getInt32ToBytesField(), message.getInt32ToBytesFieldMap());
assertEquals(message.getInt32ToEnumField(), message.getInt32ToEnumFieldMap());
assertEquals(message.getInt32ToEnumFieldValue(), message.getInt32ToEnumFieldValueMap());
assertEquals(message.getInt32ToMessageField(), message.getInt32ToMessageFieldMap());
assertEquals(message.getStringToInt32FieldMap(), message.getStringToInt32FieldMap());
assertEquals(message.getInt32ToBytesFieldMap(), message.getInt32ToBytesFieldMap());
assertEquals(message.getInt32ToEnumFieldMap(), message.getInt32ToEnumFieldMap());
assertEquals(message.getInt32ToEnumFieldValueMap(), message.getInt32ToEnumFieldValueMap());
assertEquals(message.getInt32ToMessageFieldMap(), message.getInt32ToMessageFieldMap());
}
public void testContains() {

190
java/core/src/test/java/com/google/protobuf/MapTest.java
View File

@ -122,44 +122,44 @@ public class MapTest extends TestCase {
private void copyMapValues(TestMap source, TestMap.Builder destination) {
destination
.putAllInt32ToInt32Field(source.getInt32ToInt32Field())
.putAllInt32ToStringField(source.getInt32ToStringField())
.putAllInt32ToBytesField(source.getInt32ToBytesField())
.putAllInt32ToEnumField(source.getInt32ToEnumField())
.putAllInt32ToMessageField(source.getInt32ToMessageField())
.putAllStringToInt32Field(source.getStringToInt32Field());
.putAllInt32ToInt32Field(source.getInt32ToInt32FieldMap())
.putAllInt32ToStringField(source.getInt32ToStringFieldMap())
.putAllInt32ToBytesField(source.getInt32ToBytesFieldMap())
.putAllInt32ToEnumField(source.getInt32ToEnumFieldMap())
.putAllInt32ToMessageField(source.getInt32ToMessageFieldMap())
.putAllStringToInt32Field(source.getStringToInt32FieldMap());
}
private void assertMapValuesSet(TestMap message) {
assertEquals(3, message.getInt32ToInt32Field().size());
assertEquals(11, message.getInt32ToInt32Field().get(1).intValue());
assertEquals(22, message.getInt32ToInt32Field().get(2).intValue());
assertEquals(33, message.getInt32ToInt32Field().get(3).intValue());
assertEquals(3, message.getInt32ToInt32FieldMap().size());
assertEquals(11, message.getInt32ToInt32FieldMap().get(1).intValue());
assertEquals(22, message.getInt32ToInt32FieldMap().get(2).intValue());
assertEquals(33, message.getInt32ToInt32FieldMap().get(3).intValue());
assertEquals(3, message.getInt32ToStringField().size());
assertEquals("11", message.getInt32ToStringField().get(1));
assertEquals("22", message.getInt32ToStringField().get(2));
assertEquals("33", message.getInt32ToStringField().get(3));
assertEquals(3, message.getInt32ToStringFieldMap().size());
assertEquals("11", message.getInt32ToStringFieldMap().get(1));
assertEquals("22", message.getInt32ToStringFieldMap().get(2));
assertEquals("33", message.getInt32ToStringFieldMap().get(3));
assertEquals(3, message.getInt32ToBytesField().size());
assertEquals(TestUtil.toBytes("11"), message.getInt32ToBytesField().get(1));
assertEquals(TestUtil.toBytes("22"), message.getInt32ToBytesField().get(2));
assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesField().get(3));
assertEquals(3, message.getInt32ToBytesFieldMap().size());
assertEquals(TestUtil.toBytes("11"), message.getInt32ToBytesFieldMap().get(1));
assertEquals(TestUtil.toBytes("22"), message.getInt32ToBytesFieldMap().get(2));
assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesFieldMap().get(3));
assertEquals(3, message.getInt32ToEnumField().size());
assertEquals(TestMap.EnumValue.FOO, message.getInt32ToEnumField().get(1));
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(2));
assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumField().get(3));
assertEquals(3, message.getInt32ToEnumFieldMap().size());
assertEquals(TestMap.EnumValue.FOO, message.getInt32ToEnumFieldMap().get(1));
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumFieldMap().get(2));
assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumFieldMap().get(3));
assertEquals(3, message.getInt32ToMessageField().size());
assertEquals(11, message.getInt32ToMessageField().get(1).getValue());
assertEquals(22, message.getInt32ToMessageField().get(2).getValue());
assertEquals(33, message.getInt32ToMessageField().get(3).getValue());
assertEquals(3, message.getInt32ToMessageFieldMap().size());
assertEquals(11, message.getInt32ToMessageFieldMap().get(1).getValue());
assertEquals(22, message.getInt32ToMessageFieldMap().get(2).getValue());
assertEquals(33, message.getInt32ToMessageFieldMap().get(3).getValue());
assertEquals(3, message.getStringToInt32Field().size());
assertEquals(11, message.getStringToInt32Field().get("1").intValue());
assertEquals(22, message.getStringToInt32Field().get("2").intValue());
assertEquals(33, message.getStringToInt32Field().get("3").intValue());
assertEquals(3, message.getStringToInt32FieldMap().size());
assertEquals(11, message.getStringToInt32FieldMap().get("1").intValue());
assertEquals(22, message.getStringToInt32FieldMap().get("2").intValue());
assertEquals(33, message.getStringToInt32FieldMap().get("3").intValue());
}
private void updateMapValuesUsingMutableMap(TestMap.Builder builder) {
@ -239,35 +239,35 @@ public class MapTest extends TestCase {
}
private void assertMapValuesUpdated(TestMap message) {
assertEquals(3, message.getInt32ToInt32Field().size());
assertEquals(111, message.getInt32ToInt32Field().get(1).intValue());
assertEquals(33, message.getInt32ToInt32Field().get(3).intValue());
assertEquals(44, message.getInt32ToInt32Field().get(4).intValue());
assertEquals(3, message.getInt32ToInt32FieldMap().size());
assertEquals(111, message.getInt32ToInt32FieldMap().get(1).intValue());
assertEquals(33, message.getInt32ToInt32FieldMap().get(3).intValue());
assertEquals(44, message.getInt32ToInt32FieldMap().get(4).intValue());
assertEquals(3, message.getInt32ToStringField().size());
assertEquals("111", message.getInt32ToStringField().get(1));
assertEquals("33", message.getInt32ToStringField().get(3));
assertEquals("44", message.getInt32ToStringField().get(4));
assertEquals(3, message.getInt32ToStringFieldMap().size());
assertEquals("111", message.getInt32ToStringFieldMap().get(1));
assertEquals("33", message.getInt32ToStringFieldMap().get(3));
assertEquals("44", message.getInt32ToStringFieldMap().get(4));
assertEquals(3, message.getInt32ToBytesField().size());
assertEquals(TestUtil.toBytes("111"), message.getInt32ToBytesField().get(1));
assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesField().get(3));
assertEquals(TestUtil.toBytes("44"), message.getInt32ToBytesField().get(4));
assertEquals(3, message.getInt32ToBytesFieldMap().size());
assertEquals(TestUtil.toBytes("111"), message.getInt32ToBytesFieldMap().get(1));
assertEquals(TestUtil.toBytes("33"), message.getInt32ToBytesFieldMap().get(3));
assertEquals(TestUtil.toBytes("44"), message.getInt32ToBytesFieldMap().get(4));
assertEquals(3, message.getInt32ToEnumField().size());
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(1));
assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumField().get(3));
assertEquals(TestMap.EnumValue.QUX, message.getInt32ToEnumField().get(4));
assertEquals(3, message.getInt32ToEnumFieldMap().size());
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumFieldMap().get(1));
assertEquals(TestMap.EnumValue.BAZ, message.getInt32ToEnumFieldMap().get(3));
assertEquals(TestMap.EnumValue.QUX, message.getInt32ToEnumFieldMap().get(4));
assertEquals(3, message.getInt32ToMessageField().size());
assertEquals(111, message.getInt32ToMessageField().get(1).getValue());
assertEquals(33, message.getInt32ToMessageField().get(3).getValue());
assertEquals(44, message.getInt32ToMessageField().get(4).getValue());
assertEquals(3, message.getInt32ToMessageFieldMap().size());
assertEquals(111, message.getInt32ToMessageFieldMap().get(1).getValue());
assertEquals(33, message.getInt32ToMessageFieldMap().get(3).getValue());
assertEquals(44, message.getInt32ToMessageFieldMap().get(4).getValue());
assertEquals(3, message.getStringToInt32Field().size());
assertEquals(111, message.getStringToInt32Field().get("1").intValue());
assertEquals(33, message.getStringToInt32Field().get("3").intValue());
assertEquals(44, message.getStringToInt32Field().get("4").intValue());
assertEquals(3, message.getStringToInt32FieldMap().size());
assertEquals(111, message.getStringToInt32FieldMap().get("1").intValue());
assertEquals(33, message.getStringToInt32FieldMap().get("3").intValue());
assertEquals(44, message.getStringToInt32FieldMap().get("4").intValue());
}
private void assertMapValuesCleared(TestMapOrBuilder testMapOrBuilder) {
@ -468,11 +468,13 @@ public class MapTest extends TestCase {
.build();
TestMap destination =
TestMap.newBuilder().putAllInt32ToEnumFieldValue(source.getInt32ToEnumFieldValue()).build();
TestMap.newBuilder()
.putAllInt32ToEnumFieldValue(source.getInt32ToEnumFieldValueMap())
.build();
assertEquals(0, destination.getInt32ToEnumFieldValue().get(0).intValue());
assertEquals(1, destination.getInt32ToEnumFieldValue().get(1).intValue());
assertEquals(1000, destination.getInt32ToEnumFieldValue().get(2).intValue());
assertEquals(0, destination.getInt32ToEnumFieldValueMap().get(0).intValue());
assertEquals(1, destination.getInt32ToEnumFieldValueMap().get(1).intValue());
assertEquals(1000, destination.getInt32ToEnumFieldValueMap().get(2).intValue());
assertEquals(3, destination.getInt32ToEnumFieldCount());
}
@ -583,7 +585,7 @@ public class MapTest extends TestCase {
} catch (InvalidProtocolBufferException expected) {
assertTrue(expected.getUnfinishedMessage() instanceof TestMap);
map = (TestMap) expected.getUnfinishedMessage();
assertTrue(map.getInt32ToMessageField().isEmpty());
assertTrue(map.getInt32ToMessageFieldMap().isEmpty());
}
map =
@ -644,14 +646,14 @@ public class MapTest extends TestCase {
TestOnChangeEventPropagation.Builder parent = TestOnChangeEventPropagation.newBuilder();
parent.getOptionalMessageBuilder().putInt32ToInt32Field(1, 2);
TestOnChangeEventPropagation message = parent.build();
assertEquals(2, message.getOptionalMessage().getInt32ToInt32Field().get(1).intValue());
assertEquals(2, message.getOptionalMessage().getInt32ToInt32FieldMap().get(1).intValue());
// Make a change using nested builder.
parent.getOptionalMessageBuilder().putInt32ToInt32Field(1, 3);
// Should be able to observe the change.
message = parent.build();
assertEquals(3, message.getOptionalMessage().getInt32ToInt32Field().get(1).intValue());
assertEquals(3, message.getOptionalMessage().getInt32ToInt32FieldMap().get(1).intValue());
// Make another change using mergeFrom()
TestMap other = TestMap.newBuilder().putInt32ToInt32Field(1, 4).build();
@ -659,14 +661,14 @@ public class MapTest extends TestCase {
// Should be able to observe the change.
message = parent.build();
assertEquals(4, message.getOptionalMessage().getInt32ToInt32Field().get(1).intValue());
assertEquals(4, message.getOptionalMessage().getInt32ToInt32FieldMap().get(1).intValue());
// Make yet another change by clearing the nested builder.
parent.getOptionalMessageBuilder().clear();
// Should be able to observe the change.
message = parent.build();
assertEquals(0, message.getOptionalMessage().getInt32ToInt32Field().size());
assertEquals(0, message.getOptionalMessage().getInt32ToInt32FieldMap().size());
}
public void testNestedBuilderOnChangeEventPropagationReflection() {
@ -683,7 +685,7 @@ public class MapTest extends TestCase {
// Should be able to observe the change.
TestOnChangeEventPropagation message = parentBuilder.build();
assertEquals(1, message.getOptionalMessage().getInt32ToInt32Field().size());
assertEquals(1, message.getOptionalMessage().getInt32ToInt32FieldMap().size());
// Change the entry value.
entryBuilder.putInt32ToInt32Field(1, 4);
@ -692,7 +694,7 @@ public class MapTest extends TestCase {
// Should be able to observe the change.
message = parentBuilder.build();
assertEquals(4, message.getOptionalMessage().getInt32ToInt32Field().get(1).intValue());
assertEquals(4, message.getOptionalMessage().getInt32ToInt32FieldMap().get(1).intValue());
// Clear the nested builder.
testMapBuilder = parentBuilder.getOptionalMessageBuilder();
@ -700,7 +702,7 @@ public class MapTest extends TestCase {
// Should be able to observe the change.
message = parentBuilder.build();
assertEquals(0, message.getOptionalMessage().getInt32ToInt32Field().size());
assertEquals(0, message.getOptionalMessage().getInt32ToInt32FieldMap().size());
}
// The following methods are used to test reflection API.
@ -789,8 +791,8 @@ public class MapTest extends TestCase {
builder.clearField(f("int32_to_int32_field"));
builder.clearField(f("int32_to_message_field"));
message = builder.build();
assertEquals(0, message.getInt32ToInt32Field().size());
assertEquals(0, message.getInt32ToMessageField().size());
assertEquals(0, message.getInt32ToInt32FieldMap().size());
assertEquals(0, message.getInt32ToMessageFieldMap().size());
// Test setField()
setMapValues(builder, "int32_to_int32_field", mapForValues(11, 22, 33, 44));
@ -801,10 +803,10 @@ public class MapTest extends TestCase {
111, MessageValue.newBuilder().setValue(222).build(),
333, MessageValue.newBuilder().setValue(444).build()));
message = builder.build();
assertEquals(22, message.getInt32ToInt32Field().get(11).intValue());
assertEquals(44, message.getInt32ToInt32Field().get(33).intValue());
assertEquals(222, message.getInt32ToMessageField().get(111).getValue());
assertEquals(444, message.getInt32ToMessageField().get(333).getValue());
assertEquals(22, message.getInt32ToInt32FieldMap().get(11).intValue());
assertEquals(44, message.getInt32ToInt32FieldMap().get(33).intValue());
assertEquals(222, message.getInt32ToMessageFieldMap().get(111).getValue());
assertEquals(444, message.getInt32ToMessageFieldMap().get(333).getValue());
// Test addRepeatedField
builder.addRepeatedField(
@ -817,8 +819,8 @@ public class MapTest extends TestCase {
555,
MessageValue.newBuilder().setValue(666).build()));
message = builder.build();
assertEquals(66, message.getInt32ToInt32Field().get(55).intValue());
assertEquals(666, message.getInt32ToMessageField().get(555).getValue());
assertEquals(66, message.getInt32ToInt32FieldMap().get(55).intValue());
assertEquals(666, message.getInt32ToMessageFieldMap().get(555).getValue());
// Test addRepeatedField (overriding existing values)
builder.addRepeatedField(
@ -831,8 +833,8 @@ public class MapTest extends TestCase {
555,
MessageValue.newBuilder().setValue(555).build()));
message = builder.build();
assertEquals(55, message.getInt32ToInt32Field().get(55).intValue());
assertEquals(555, message.getInt32ToMessageField().get(555).getValue());
assertEquals(55, message.getInt32ToInt32FieldMap().get(55).intValue());
assertEquals(555, message.getInt32ToMessageFieldMap().get(555).getValue());
// Test setRepeatedField
for (int i = 0; i < builder.getRepeatedFieldCount(f("int32_to_int32_field")); i++) {
@ -846,9 +848,9 @@ public class MapTest extends TestCase {
builder.setRepeatedField(f("int32_to_int32_field"), i, mapEntryBuilder.build());
}
message = builder.build();
assertEquals(11, message.getInt32ToInt32Field().get(22).intValue());
assertEquals(33, message.getInt32ToInt32Field().get(44).intValue());
assertEquals(55, message.getInt32ToInt32Field().get(55).intValue());
assertEquals(11, message.getInt32ToInt32FieldMap().get(22).intValue());
assertEquals(33, message.getInt32ToInt32FieldMap().get(44).intValue());
assertEquals(55, message.getInt32ToInt32FieldMap().get(55).intValue());
}
// See additional coverage in TextFormatTest.java.
@ -937,21 +939,21 @@ public class MapTest extends TestCase {
2, 1000)); // unknown value.
TestMap message = builder.build();
assertEquals(TestMap.EnumValue.FOO, message.getInt32ToEnumField().get(0));
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumField().get(1));
assertEquals(TestMap.EnumValue.UNRECOGNIZED, message.getInt32ToEnumField().get(2));
assertEquals(1000, message.getInt32ToEnumFieldValue().get(2).intValue());
assertEquals(TestMap.EnumValue.FOO, message.getInt32ToEnumFieldMap().get(0));
assertEquals(TestMap.EnumValue.BAR, message.getInt32ToEnumFieldMap().get(1));
assertEquals(TestMap.EnumValue.UNRECOGNIZED, message.getInt32ToEnumFieldMap().get(2));
assertEquals(1000, message.getInt32ToEnumFieldValueMap().get(2).intValue());
// Unknown enum values should be preserved after:
// 1. Serialization and parsing.
// 2. toBuild().
// 3. mergeFrom().
message = TestMap.parseFrom(message.toByteString());
assertEquals(1000, message.getInt32ToEnumFieldValue().get(2).intValue());
assertEquals(1000, message.getInt32ToEnumFieldValueMap().get(2).intValue());
builder = message.toBuilder();
assertEquals(1000, builder.getInt32ToEnumFieldValue().get(2).intValue());
assertEquals(1000, builder.getInt32ToEnumFieldValueMap().get(2).intValue());
builder = TestMap.newBuilder().mergeFrom(message);
assertEquals(1000, builder.getInt32ToEnumFieldValue().get(2).intValue());
assertEquals(1000, builder.getInt32ToEnumFieldValueMap().get(2).intValue());
// hashCode()/equals() should take unknown enum values into account.
builder.putAllInt32ToEnumFieldValue(newMap(2, 1001));
@ -960,7 +962,7 @@ public class MapTest extends TestCase {
assertFalse(message.equals(message2));
// Unknown values will be converted to UNRECOGNIZED so the resulted enum map
// should be the same.
assertEquals(message2.getInt32ToEnumField(), message.getInt32ToEnumField());
assertEquals(message2.getInt32ToEnumFieldMap(), message.getInt32ToEnumFieldMap());
}
public void testUnknownEnumValuesInReflectionApi() throws Exception {
@ -991,7 +993,7 @@ public class MapTest extends TestCase {
// Verify that enum values have been successfully updated.
TestMap message = builder.build();
for (Map.Entry<Integer, Integer> entry : message.getInt32ToEnumFieldValue().entrySet()) {
for (Map.Entry<Integer, Integer> entry : message.getInt32ToEnumFieldValueMap().entrySet()) {
assertEquals(data.get(entry.getKey()) + 1, entry.getValue().intValue());
}
}
@ -1002,18 +1004,18 @@ public class MapTest extends TestCase {
TestMap message = builder.build();
assertEquals(
Arrays.asList("1", "2", "3"), new ArrayList<>(message.getStringToInt32Field().keySet()));
Arrays.asList("1", "2", "3"), new ArrayList<>(message.getStringToInt32FieldMap().keySet()));
}
public void testGetMap() {
TestMap.Builder builder = TestMap.newBuilder();
setMapValuesUsingAccessors(builder);
TestMap message = builder.build();
assertEquals(message.getStringToInt32Field(), message.getStringToInt32FieldMap());
assertEquals(message.getInt32ToBytesField(), message.getInt32ToBytesFieldMap());
assertEquals(message.getInt32ToEnumField(), message.getInt32ToEnumFieldMap());
assertEquals(message.getInt32ToEnumFieldValue(), message.getInt32ToEnumFieldValueMap());
assertEquals(message.getInt32ToMessageField(), message.getInt32ToMessageFieldMap());
assertEquals(message.getStringToInt32FieldMap(), message.getStringToInt32FieldMap());
assertEquals(message.getInt32ToBytesFieldMap(), message.getInt32ToBytesFieldMap());
assertEquals(message.getInt32ToEnumFieldMap(), message.getInt32ToEnumFieldMap());
assertEquals(message.getInt32ToEnumFieldValueMap(), message.getInt32ToEnumFieldValueMap());
assertEquals(message.getInt32ToMessageFieldMap(), message.getInt32ToMessageFieldMap());
}
public void testContains() {

88
java/core/src/test/java/com/google/protobuf/TextFormatTest.java
View File

@ -30,6 +30,7 @@
package com.google.protobuf;
import static com.google.common.truth.Truth.assertThat;
import static com.google.protobuf.TestUtil.TEST_REQUIRED_INITIALIZED;
import static com.google.protobuf.TestUtil.TEST_REQUIRED_UNINITIALIZED;
@ -866,6 +867,12 @@ public class TextFormatTest extends TestCase {
assertEquals(bytes(0xe1, 0x88, 0xb4), TextFormat.unescapeBytes("\\341\\210\\264"));
assertEquals("\u1234", TextFormat.unescapeText("\\xe1\\x88\\xb4"));
assertEquals(bytes(0xe1, 0x88, 0xb4), TextFormat.unescapeBytes("\\xe1\\x88\\xb4"));
assertEquals("\u1234", TextFormat.unescapeText("\\u1234"));
assertEquals(bytes(0xe1, 0x88, 0xb4), TextFormat.unescapeBytes("\\u1234"));
assertEquals(bytes(0xe1, 0x88, 0xb4), TextFormat.unescapeBytes("\\U00001234"));
assertEquals(
new String(new int[] {0x10437}, 0, 1), TextFormat.unescapeText("\\xf0\\x90\\x90\\xb7"));
assertEquals(bytes(0xf0, 0x90, 0x90, 0xb7), TextFormat.unescapeBytes("\\U00010437"));
// Handling of strings with unescaped Unicode characters > 255.
final String zh = "\u9999\u6e2f\u4e0a\u6d77\ud84f\udf80\u8c50\u9280\u884c";
@ -893,6 +900,87 @@ public class TextFormatTest extends TestCase {
} catch (TextFormat.InvalidEscapeSequenceException e) {
// success
}
try {
TextFormat.unescapeText("\\u");
fail("Should have thrown an exception.");
} catch (TextFormat.InvalidEscapeSequenceException e) {
assertThat(e)
.hasMessageThat()
.isEqualTo("Invalid escape sequence: '\\u' with too few hex chars");
}
try {
TextFormat.unescapeText("\\ud800");
fail("Should have thrown an exception.");
} catch (TextFormat.InvalidEscapeSequenceException e) {
assertThat(e)
.hasMessageThat()
.isEqualTo("Invalid escape sequence: '\\u' refers to a surrogate");
}
try {
TextFormat.unescapeText("\\ud800\\u1234");
fail("Should have thrown an exception.");
} catch (TextFormat.InvalidEscapeSequenceException e) {
assertThat(e)
.hasMessageThat()
.isEqualTo("Invalid escape sequence: '\\u' refers to a surrogate");
}
try {
TextFormat.unescapeText("\\udc00");
fail("Should have thrown an exception.");
} catch (TextFormat.InvalidEscapeSequenceException e) {
assertThat(e)
.hasMessageThat()
.isEqualTo("Invalid escape sequence: '\\u' refers to a surrogate");
}
try {
TextFormat.unescapeText("\\ud801\\udc37");
fail("Should have thrown an exception.");
} catch (TextFormat.InvalidEscapeSequenceException e) {
assertThat(e)
.hasMessageThat()
.isEqualTo("Invalid escape sequence: '\\u' refers to a surrogate");
}
try {
TextFormat.unescapeText("\\U1234");
fail("Should have thrown an exception.");
} catch (TextFormat.InvalidEscapeSequenceException e) {
assertThat(e)
.hasMessageThat()
.isEqualTo("Invalid escape sequence: '\\U' with too few hex chars");
}
try {
TextFormat.unescapeText("\\U1234no more hex");
fail("Should have thrown an exception.");
} catch (TextFormat.InvalidEscapeSequenceException e) {
assertThat(e)
.hasMessageThat()
.isEqualTo("Invalid escape sequence: '\\U' with too few hex chars");
}
try {
TextFormat.unescapeText("\\U00110000");
fail("Should have thrown an exception.");
} catch (TextFormat.InvalidEscapeSequenceException e) {
assertThat(e)
.hasMessageThat()
.isEqualTo("Invalid escape sequence: '\\U00110000' is not a valid code point value");
}
try {
TextFormat.unescapeText("\\U0000d801\\U00000dc37");
fail("Should have thrown an exception.");
} catch (TextFormat.InvalidEscapeSequenceException e) {
assertThat(e)
.hasMessageThat()
.isEqualTo("Invalid escape sequence: '\\U0000d801' refers to a surrogate code unit");
}
}
public void testParseInteger() throws Exception {

5
java/lite/src/test/java/com/google/protobuf/Proto2MessageLiteInfoFactory.java
View File

@ -174,8 +174,9 @@ public final class Proto2MessageLiteInfoFactory implements MessageInfoFactory {
"fieldRequiredSint6487_",
"fieldRequiredGroup88_",
};
// To update this after a proto change, run protoc on proto2_message_lite.proto and copy over
// the content of the generated buildMessageInfo() method here.
// To update this after a proto change, run blaze build on proto2_message_lite.proto and copy
// over the String info from the proto2_message_lite_proto-lite-src.jar file in the
// blaze-genfiles directory.
java.lang.String info =
"\u0001U\u0001\u0002\u0001XU\u0000 \u0015\u0001\u1000\u0000\u0002\u1001\u0001\u0003"
+ "\u1002\u0002\u0004\u1003\u0003\u0005\u1004\u0004\u0006\u1005\u0005\u0007\u1006\u0006\b\u1007\u0007"

68
java/util/src/main/java/com/google/protobuf/util/FieldMaskTree.java
View File

@ -137,11 +137,19 @@ final class FieldMaskTree {
}
/**
* Remove {@code path} from the tree.
* Removes {@code path} from the tree.
*
* <p>When removing a field path from the tree, all sub-paths will be removed. That is, after
* removing "foo.bar" from the tree, "foo.bar.baz" will be removed. Likewise, if the field path to
* remove is a non-exist sub-path, nothing will be changed.
* <ul>
* When removing a field path from the tree:
* <li>All sub-paths will be removed. That is, after removing "foo.bar" from the tree,
* "foo.bar.baz" will be removed.
* <li>If all children of a node has been removed, the node itself will be removed as well.
* That is, if "foo" only has one child "bar" and "foo.bar" only has one child "baz",
* removing "foo.bar.barz" would remove both "foo" and "foo.bar".
* If "foo" has both "bar" and "qux" as children, removing "foo.bar" would leave the path
* "foo.qux" intact.
* <li>If the field path to remove is a non-exist sub-path, nothing will be changed.
* </ul>
*/
@CanIgnoreReturnValue
FieldMaskTree removeFieldPath(String path) {
@ -149,23 +157,35 @@ final class FieldMaskTree {
if (parts.isEmpty()) {
return this;
}
Node node = root;
for (int i = 0; i < parts.size(); i++) {
String key = parts.get(i);
if (!node.children.containsKey(key)) {
// Path does not exist.
return this;
}
if (i == parts.size() - 1) {
// Remove path.
node.children.remove(key);
return this;
}
node = node.children.get(key);
}
removeFieldPath(root, parts, 0);
return this;
}
/**
* Removes {@code parts} from {@code node} recursively.
*
* @return a boolean value indicating whether current {@code node} should be removed.
*/
@CanIgnoreReturnValue
private static boolean removeFieldPath(Node node, List<String> parts, int index) {
String key = parts.get(index);
// Base case 1: path not match.
if (!node.children.containsKey(key)) {
return false;
}
// Base case 2: last element in parts.
if (index == parts.size() - 1) {
node.children.remove(key);
return node.children.isEmpty();
}
// Recursive remove sub-path.
if (removeFieldPath(node.children.get(key), parts, index + 1)) {
node.children.remove(key);
}
return node.children.isEmpty();
}
/** Removes all field paths in {@code mask} from this tree. */
@CanIgnoreReturnValue
FieldMaskTree removeFromFieldMask(FieldMask mask) {
@ -187,10 +207,8 @@ final class FieldMaskTree {
return FieldMask.newBuilder().addAllPaths(paths).build();
}
/**
* Gathers all field paths in a sub-tree.
*/
private void getFieldPaths(Node node, String path, List<String> paths) {
/** Gathers all field paths in a sub-tree. */
private static void getFieldPaths(Node node, String path, List<String> paths) {
if (node.children.isEmpty()) {
paths.add(path);
return;
@ -247,10 +265,8 @@ final class FieldMaskTree {
merge(root, "", source, destination, options);
}
/**
* Merges all fields specified by a sub-tree from {@code source} to {@code destination}.
*/
private void merge(
/** Merges all fields specified by a sub-tree from {@code source} to {@code destination}. */
private static void merge(
Node node,
String path,
Message source,

8
java/util/src/main/java/com/google/protobuf/util/FieldMaskUtil.java
View File

@ -276,7 +276,13 @@ public final class FieldMaskUtil {
return maskTree.toFieldMask();
}
/** Subtracts {@code secondMask} and {@code otherMasks} from {@code firstMask}. */
/**
* Subtracts {@code secondMask} and {@code otherMasks} from {@code firstMask}.
*
* <p>This method disregards proto structure. That is, if {@code firstMask} is "foo" and {@code
* secondMask} is "foo.bar", the response will always be "foo" without considering the internal
* proto structure of message "foo".
*/
public static FieldMask subtract(
FieldMask firstMask, FieldMask secondMask, FieldMask... otherMasks) {
FieldMaskTree maskTree = new FieldMaskTree(firstMask).removeFromFieldMask(secondMask);

1
java/util/src/main/java/com/google/protobuf/util/JsonFormat.java
View File

@ -525,7 +525,6 @@ public class JsonFormat {
return types.get(name);
}
/* @Nullable */
Descriptor getDescriptorForTypeUrl(String typeUrl) throws InvalidProtocolBufferException {
return find(getTypeName(typeUrl));
}

24
java/util/src/test/java/com/google/protobuf/util/FieldMaskTreeTest.java
View File

@ -77,25 +77,37 @@ public class FieldMaskTreeTest extends TestCase {
public void testRemoveFieldPath() throws Exception {
String initialTreeString = "bar.baz,bar.quz.bar,foo";
FieldMaskTree tree = new FieldMaskTree(FieldMaskUtil.fromString(initialTreeString));
FieldMaskTree tree;
// Empty path.
tree = new FieldMaskTree(FieldMaskUtil.fromString(initialTreeString));
tree.removeFieldPath("");
assertEquals(initialTreeString, tree.toString());
// Non-exist sub-path of an existing leaf.
tree = new FieldMaskTree(FieldMaskUtil.fromString(initialTreeString));
tree.removeFieldPath("foo.bar");
assertEquals(initialTreeString, tree.toString());
// Non-exist path.
tree = new FieldMaskTree(FieldMaskUtil.fromString(initialTreeString));
tree.removeFieldPath("bar.foo");
assertEquals(initialTreeString, tree.toString());
// Match an existing leaf node.
// Match an existing leaf node -> remove leaf node.
tree = new FieldMaskTree(FieldMaskUtil.fromString(initialTreeString));
tree.removeFieldPath("foo");
assertEquals("bar.baz,bar.quz.bar", tree.toString());
// Match sub-path of an existing leaf node.
// Match sub-path of an existing leaf node -> recursive removal.
tree = new FieldMaskTree(FieldMaskUtil.fromString(initialTreeString));
tree.removeFieldPath("bar.quz.bar");
assertEquals("bar.baz,bar.quz", tree.toString());
// Match a non-leaf node.
assertEquals("bar.baz,foo", tree.toString());
// Match a non-leaf node -> remove all children.
tree = new FieldMaskTree(FieldMaskUtil.fromString(initialTreeString));
tree.removeFieldPath("bar");
assertThat(tree.toString()).isEmpty();
assertEquals("foo", tree.toString());
}
public void testRemoveFromFieldMask() throws Exception {

2
java/util/src/test/java/com/google/protobuf/util/FieldMaskUtilTest.java
View File

@ -239,7 +239,7 @@ public class FieldMaskUtilTest extends TestCase {
FieldMask mask3 = FieldMaskUtil.fromString("bar.quz");
FieldMask mask4 = FieldMaskUtil.fromString("foo,bar.baz");
FieldMask result = FieldMaskUtil.subtract(mask1, mask2, mask3, mask4);
assertEquals("bar", FieldMaskUtil.toString(result));
assertThat(FieldMaskUtil.toString(result)).isEmpty();
}
public void testIntersection() throws Exception {

4
java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java
View File

@ -722,8 +722,8 @@ public class JsonFormatTest extends TestCase {
mergeFromJson(
"{\n" + " int32ToInt32Map: {1: 2},\n" + " stringToInt32Map: {hello: 3}\n" + "}", builder);
TestMap message = builder.build();
assertEquals(2, message.getInt32ToInt32Map().get(1).intValue());
assertEquals(3, message.getStringToInt32Map().get("hello").intValue());
assertEquals(2, message.getInt32ToInt32MapMap().get(1).intValue());
assertEquals(3, message.getStringToInt32MapMap().get("hello").intValue());
}
public void testWrappers() throws Exception {

1
js/binary/constants.js
View File

@ -31,6 +31,7 @@
/**
* @fileoverview This file contains constants and typedefs used by
* jspb.BinaryReader and BinaryWriter.
* @suppress {missingRequire} TODO(b/152540451): this shouldn't be needed
*
* @author aappleby@google.com (Austin Appleby)
*/

1
js/binary/decoder.js
View File

@ -40,6 +40,7 @@
* intact, you _must_ read them using one of the Hash64 methods, which return
* an 8-character string.
*
* @suppress {missingRequire} TODO(b/152540451): this shouldn't be needed
* @author aappleby@google.com (Austin Appleby)
*/

1
js/binary/reader.js
View File

@ -41,6 +41,7 @@
* using the typed jspb code generator, but if you bypass that you'll need
* to keep things in sync by hand.
*
* @suppress {missingRequire} TODO(b/152540451): this shouldn't be needed
* @author aappleby@google.com (Austin Appleby)
*/

1
js/binary/utils.js
View File

@ -32,6 +32,7 @@
* @fileoverview This file contains helper code used by jspb.BinaryReader
* and BinaryWriter.
*
* @suppress {missingRequire} TODO(b/152540451): this shouldn't be needed
* @author aappleby@google.com (Austin Appleby)
*/

1
js/binary/writer.js
View File

@ -52,6 +52,7 @@
* Major caveat 3 - This class uses typed arrays and must not be used on older
* browsers that do not support them.
*
* @suppress {missingRequire} TODO(b/152540451): this shouldn't be needed
* @author aappleby@google.com (Austin Appleby)
*/

5
js/map.js
View File

@ -1,3 +1,4 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
@ -28,6 +29,10 @@
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
/**
* @fileoverview
* @suppress {missingRequire} TODO(b/152540451): this shouldn't be needed
*/
goog.provide('jspb.Map');
goog.require('goog.asserts');

1
js/message.js
View File

@ -31,6 +31,7 @@
/**
* @fileoverview Definition of jspb.Message.
*
* @suppress {missingRequire} TODO(b/152540451): this shouldn't be needed
* @author mwr@google.com (Mark Rawling)
*/

7
python/google/protobuf/internal/api_implementation.py
View File

@ -60,11 +60,16 @@ if _api_version < 0: # Still unspecified?
raise ImportError('_use_fast_cpp_protos import succeeded but was None')
del _use_fast_cpp_protos
_api_version = 2
# Can not import both use_fast_cpp_protos and use_pure_python.
from google.protobuf import use_pure_python
raise RuntimeError(
'Conflict depend on both use_fast_cpp_protos and use_pure_python')
except ImportError:
try:
# pylint: disable=g-import-not-at-top
from google.protobuf.internal import use_pure_python
from google.protobuf import use_pure_python
del use_pure_python # Avoids a pylint error and namespace pollution.
_api_version = 0
except ImportError:
# TODO(b/74017912): It's unsafe to enable :use_fast_cpp_protos by default;
# it can cause data loss if you have any Python-only extensions to any

3
python/google/protobuf/internal/containers.py
View File

@ -629,7 +629,8 @@ class MessageMap(MutableMapping):
return repr(self._values)
def MergeFrom(self, other):
for key in other:
# pylint: disable=protected-access
for key in other._values:
# According to documentation: "When parsing from the wire or when merging,
# if there are duplicate map keys the last key seen is used".
if key in self:

5
python/google/protobuf/internal/message_test.py
View File

@ -2107,6 +2107,11 @@ class Proto3Test(unittest.TestCase):
self.assertEqual(msg.map_int32_foreign_message[222].d, 20)
self.assertNotEqual(msg.map_int32_foreign_message[222].c, 123)
# Merge a dict to map field is not accepted
with self.assertRaises(AttributeError):
m1.map_int32_all_types.MergeFrom(
{1: unittest_proto3_arena_pb2.TestAllTypes()})
def testMergeFromBadType(self):
msg = map_unittest_pb2.TestMap()
with self.assertRaisesRegexp(

5
python/google/protobuf/pyext/map_container.cc
View File

@ -339,6 +339,11 @@ PyObject* GetEntryClass(PyObject* _self) {
PyObject* MapReflectionFriend::MergeFrom(PyObject* _self, PyObject* arg) {
MapContainer* self = GetMap(_self);
if (!PyObject_TypeCheck(arg, ScalarMapContainer_Type) &&
!PyObject_TypeCheck(arg, MessageMapContainer_Type)) {
PyErr_SetString(PyExc_AttributeError, "Not a map field");
return nullptr;
}
MapContainer* other_map = GetMap(arg);
Message* message = self->GetMutableMessage();
const Message* other_message = other_map->parent->message;

2
python/google/protobuf/pyext/message.cc
View File

@ -809,7 +809,7 @@ bool CheckAndSetString(
return false;
}
string value_string(value, value_len);
std::string value_string(value, value_len);
if (append) {
reflection->AddString(message, descriptor, std::move(value_string));
} else if (index < 0) {

5
python/google/protobuf/text_format.py
View File

@ -882,8 +882,11 @@ class _Parser(object):
raise tokenizer.ParseErrorPreviousToken('Expected "%s".' %
(expanded_any_end_token,))
self._MergeField(tokenizer, expanded_any_sub_message)
deterministic = False
message.Pack(expanded_any_sub_message,
type_url_prefix=type_url_prefix)
type_url_prefix=type_url_prefix,
deterministic=deterministic)
return
if tokenizer.TryConsume('['):

3
src/Makefile.am
View File

@ -68,7 +68,6 @@ nobase_include_HEADERS = \
google/protobuf/stubs/bytestream.h \
google/protobuf/stubs/casts.h \
google/protobuf/stubs/common.h \
google/protobuf/stubs/fastmem.h \
google/protobuf/stubs/hash.h \
google/protobuf/stubs/logging.h \
google/protobuf/stubs/macros.h \
@ -104,7 +103,6 @@ nobase_include_HEADERS = \
google/protobuf/generated_message_util.h \
google/protobuf/has_bits.h \
google/protobuf/implicit_weak_message.h \
google/protobuf/inlined_string_field.h \
google/protobuf/io/io_win32.h \
google/protobuf/map_entry.h \
google/protobuf/map_entry_lite.h \
@ -202,6 +200,7 @@ libprotobuf_lite_la_SOURCES = \
google/protobuf/stubs/time.h \
google/protobuf/any_lite.cc \
google/protobuf/arena.cc \
google/protobuf/arenastring.cc \
google/protobuf/extension_set.cc \
google/protobuf/generated_enum_util.cc \
google/protobuf/generated_message_util.cc \

5
src/google/protobuf/any.cc
View File

@ -51,9 +51,8 @@ void AnyMetadata::PackFrom(const Message& message,
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString(),
GetTypeUrl(message.GetDescriptor()->full_name(), type_url_prefix),
nullptr);
message.SerializeToString(value_->Mutable(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(),
nullptr));
message.SerializeToString(
value_->Mutable(ArenaStringPtr::EmptyDefault{}, nullptr));
}
bool AnyMetadata::UnpackTo(Message* message) const {

3
src/google/protobuf/any.h
View File

@ -60,7 +60,8 @@ class PROTOBUF_EXPORT AnyMetadata {
typedef ArenaStringPtr ValueType;
public:
// AnyMetadata does not take ownership of "type_url" and "value".
AnyMetadata(UrlType* type_url, ValueType* value);
constexpr AnyMetadata(UrlType* type_url, ValueType* value)
: type_url_(type_url), value_(value) {}
// Packs a message using the default type URL prefix: "type.googleapis.com".
// The resulted type URL will be "type.googleapis.com/<message_full_name>".

8
src/google/protobuf/any.pb.cc
View File

@ -112,12 +112,12 @@ Any::Any(const Any& from)
_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_);
type_url_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
if (!from._internal_type_url().empty()) {
type_url_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_type_url(),
type_url_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, from._internal_type_url(),
GetArena());
}
value_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
if (!from._internal_value().empty()) {
value_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_value(),
value_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, from._internal_value(),
GetArena());
}
// @@protoc_insertion_point(copy_constructor:google.protobuf.Any)
@ -162,8 +162,8 @@ void Any::Clear() {
// Prevent compiler warnings about cached_has_bits being unused
(void) cached_has_bits;
type_url_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena());
value_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena());
type_url_.ClearToEmpty();
value_.ClearToEmpty();
_internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>();
}

27
src/google/protobuf/any.pb.h
View File

@ -25,7 +25,6 @@
#include <google/protobuf/arenastring.h>
#include <google/protobuf/generated_message_table_driven.h>
#include <google/protobuf/generated_message_util.h>
#include <google/protobuf/inlined_string_field.h>
#include <google/protobuf/metadata_lite.h>
#include <google/protobuf/generated_message_reflection.h>
#include <google/protobuf/message.h>
@ -272,7 +271,7 @@ class PROTOBUF_EXPORT Any PROTOBUF_FINAL :
// string type_url = 1;
inline void Any::clear_type_url() {
type_url_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena());
type_url_.ClearToEmpty();
}
inline const std::string& Any::type_url() const {
// @@protoc_insertion_point(field_get:google.protobuf.Any.type_url)
@ -291,31 +290,30 @@ inline const std::string& Any::_internal_type_url() const {
}
inline void Any::_internal_set_type_url(const std::string& value) {
type_url_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value, GetArena());
type_url_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, value, GetArena());
}
inline void Any::set_type_url(std::string&& value) {
type_url_.Set(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArena());
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::move(value), GetArena());
// @@protoc_insertion_point(field_set_rvalue:google.protobuf.Any.type_url)
}
inline void Any::set_type_url(const char* value) {
GOOGLE_DCHECK(value != nullptr);
type_url_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value),
GetArena());
type_url_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(value), GetArena());
// @@protoc_insertion_point(field_set_char:google.protobuf.Any.type_url)
}
inline void Any::set_type_url(const char* value,
size_t size) {
type_url_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(
type_url_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(
reinterpret_cast<const char*>(value), size), GetArena());
// @@protoc_insertion_point(field_set_pointer:google.protobuf.Any.type_url)
}
inline std::string* Any::_internal_mutable_type_url() {
return type_url_.Mutable(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena());
return type_url_.Mutable(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, GetArena());
}
inline std::string* Any::release_type_url() {
// @@protoc_insertion_point(field_release:google.protobuf.Any.type_url)
@ -334,7 +332,7 @@ inline void Any::set_allocated_type_url(std::string* type_url) {
// bytes value = 2;
inline void Any::clear_value() {
value_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena());
value_.ClearToEmpty();
}
inline const std::string& Any::value() const {
// @@protoc_insertion_point(field_get:google.protobuf.Any.value)
@ -353,31 +351,30 @@ inline const std::string& Any::_internal_value() const {
}
inline void Any::_internal_set_value(const std::string& value) {
value_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value, GetArena());
value_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, value, GetArena());
}
inline void Any::set_value(std::string&& value) {
value_.Set(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArena());
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::move(value), GetArena());
// @@protoc_insertion_point(field_set_rvalue:google.protobuf.Any.value)
}
inline void Any::set_value(const char* value) {
GOOGLE_DCHECK(value != nullptr);
value_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value),
GetArena());
value_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(value), GetArena());
// @@protoc_insertion_point(field_set_char:google.protobuf.Any.value)
}
inline void Any::set_value(const void* value,
size_t size) {
value_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(
value_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(
reinterpret_cast<const char*>(value), size), GetArena());
// @@protoc_insertion_point(field_set_pointer:google.protobuf.Any.value)
}
inline std::string* Any::_internal_mutable_value() {
return value_.Mutable(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena());
return value_.Mutable(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, GetArena());
}
inline std::string* Any::release_value() {
// @@protoc_insertion_point(field_release:google.protobuf.Any.value)

7
src/google/protobuf/any_lite.cc
View File

@ -53,16 +53,13 @@ const char kAnyFullTypeName[] = "google.protobuf.Any";
const char kTypeGoogleApisComPrefix[] = "type.googleapis.com/";
const char kTypeGoogleProdComPrefix[] = "type.googleprod.com/";
AnyMetadata::AnyMetadata(UrlType* type_url, ValueType* value)
: type_url_(type_url), value_(value) {}
void AnyMetadata::InternalPackFrom(const MessageLite& message,
StringPiece type_url_prefix,
StringPiece type_name) {
type_url_->Set(&::google::protobuf::internal::GetEmptyString(),
GetTypeUrl(type_name, type_url_prefix), nullptr);
message.SerializeToString(value_->Mutable(
&::google::protobuf::internal::GetEmptyStringAlreadyInited(), nullptr));
message.SerializeToString(
value_->Mutable(ArenaStringPtr::EmptyDefault{}, nullptr));
}
bool AnyMetadata::InternalUnpackTo(StringPiece type_name,

8
src/google/protobuf/any_test.cc
View File

@ -33,10 +33,18 @@
#include <gtest/gtest.h>
// Must be included last.
#include <google/protobuf/port_def.inc>
namespace google {
namespace protobuf {
namespace {
TEST(AnyMetadataTest, ConstInit) {
PROTOBUF_CONSTINIT static internal::AnyMetadata metadata(nullptr, nullptr);
(void)metadata;
}
TEST(AnyTest, TestPackAndUnpack) {
protobuf_unittest::TestAny submessage;
submessage.set_int32_value(12345);

28
src/google/protobuf/api.pb.cc
View File

@ -204,12 +204,12 @@ Api::Api(const Api& from)
_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_);
name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
if (!from._internal_name().empty()) {
name_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_name(),
name_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, from._internal_name(),
GetArena());
}
version_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
if (!from._internal_version().empty()) {
version_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_version(),
version_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, from._internal_version(),
GetArena());
}
if (from._internal_has_source_context()) {
@ -268,8 +268,8 @@ void Api::Clear() {
methods_.Clear();
options_.Clear();
mixins_.Clear();
name_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena());
version_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena());
name_.ClearToEmpty();
version_.ClearToEmpty();
if (GetArena() == nullptr && source_context_ != nullptr) {
delete source_context_;
}
@ -614,17 +614,17 @@ Method::Method(const Method& from)
_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_);
name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
if (!from._internal_name().empty()) {
name_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_name(),
name_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, from._internal_name(),
GetArena());
}
request_type_url_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
if (!from._internal_request_type_url().empty()) {
request_type_url_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_request_type_url(),
request_type_url_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, from._internal_request_type_url(),
GetArena());
}
response_type_url_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
if (!from._internal_response_type_url().empty()) {
response_type_url_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_response_type_url(),
response_type_url_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, from._internal_response_type_url(),
GetArena());
}
::memcpy(&request_streaming_, &from.request_streaming_,
@ -679,9 +679,9 @@ void Method::Clear() {
(void) cached_has_bits;
options_.Clear();
name_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena());
request_type_url_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena());
response_type_url_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena());
name_.ClearToEmpty();
request_type_url_.ClearToEmpty();
response_type_url_.ClearToEmpty();
::memset(&request_streaming_, 0, static_cast<size_t>(
reinterpret_cast<char*>(&syntax_) -
reinterpret_cast<char*>(&request_streaming_)) + sizeof(syntax_));
@ -1008,12 +1008,12 @@ Mixin::Mixin(const Mixin& from)
_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_);
name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
if (!from._internal_name().empty()) {
name_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_name(),
name_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, from._internal_name(),
GetArena());
}
root_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
if (!from._internal_root().empty()) {
root_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_root(),
root_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, from._internal_root(),
GetArena());
}
// @@protoc_insertion_point(copy_constructor:google.protobuf.Mixin)
@ -1058,8 +1058,8 @@ void Mixin::Clear() {
// Prevent compiler warnings about cached_has_bits being unused
(void) cached_has_bits;
name_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena());
root_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena());
name_.ClearToEmpty();
root_.ClearToEmpty();
_internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>();
}

92
src/google/protobuf/api.pb.h
View File

@ -25,7 +25,6 @@
#include <google/protobuf/arenastring.h>
#include <google/protobuf/generated_message_table_driven.h>
#include <google/protobuf/generated_message_util.h>
#include <google/protobuf/inlined_string_field.h>
#include <google/protobuf/metadata_lite.h>
#include <google/protobuf/generated_message_reflection.h>
#include <google/protobuf/message.h>
@ -732,7 +731,7 @@ class PROTOBUF_EXPORT Mixin PROTOBUF_FINAL :
// string name = 1;
inline void Api::clear_name() {
name_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena());
name_.ClearToEmpty();
}
inline const std::string& Api::name() const {
// @@protoc_insertion_point(field_get:google.protobuf.Api.name)
@ -751,31 +750,30 @@ inline const std::string& Api::_internal_name() const {
}
inline void Api::_internal_set_name(const std::string& value) {
name_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value, GetArena());
name_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, value, GetArena());
}
inline void Api::set_name(std::string&& value) {
name_.Set(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArena());
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::move(value), GetArena());
// @@protoc_insertion_point(field_set_rvalue:google.protobuf.Api.name)
}
inline void Api::set_name(const char* value) {
GOOGLE_DCHECK(value != nullptr);
name_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value),
GetArena());
name_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(value), GetArena());
// @@protoc_insertion_point(field_set_char:google.protobuf.Api.name)
}
inline void Api::set_name(const char* value,
size_t size) {
name_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(
name_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(
reinterpret_cast<const char*>(value), size), GetArena());
// @@protoc_insertion_point(field_set_pointer:google.protobuf.Api.name)
}
inline std::string* Api::_internal_mutable_name() {
return name_.Mutable(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena());
return name_.Mutable(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, GetArena());
}
inline std::string* Api::release_name() {
// @@protoc_insertion_point(field_release:google.protobuf.Api.name)
@ -869,7 +867,7 @@ Api::options() const {
// string version = 4;
inline void Api::clear_version() {
version_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena());
version_.ClearToEmpty();
}
inline const std::string& Api::version() const {
// @@protoc_insertion_point(field_get:google.protobuf.Api.version)
@ -888,31 +886,30 @@ inline const std::string& Api::_internal_version() const {
}
inline void Api::_internal_set_version(const std::string& value) {
version_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value, GetArena());
version_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, value, GetArena());
}
inline void Api::set_version(std::string&& value) {
version_.Set(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArena());
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::move(value), GetArena());
// @@protoc_insertion_point(field_set_rvalue:google.protobuf.Api.version)
}
inline void Api::set_version(const char* value) {
GOOGLE_DCHECK(value != nullptr);
version_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value),
GetArena());
version_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(value), GetArena());
// @@protoc_insertion_point(field_set_char:google.protobuf.Api.version)
}
inline void Api::set_version(const char* value,
size_t size) {
version_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(
version_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(
reinterpret_cast<const char*>(value), size), GetArena());
// @@protoc_insertion_point(field_set_pointer:google.protobuf.Api.version)
}
inline std::string* Api::_internal_mutable_version() {
return version_.Mutable(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena());
return version_.Mutable(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, GetArena());
}
inline std::string* Api::release_version() {
// @@protoc_insertion_point(field_release:google.protobuf.Api.version)
@ -1071,7 +1068,7 @@ inline void Api::set_syntax(PROTOBUF_NAMESPACE_ID::Syntax value) {
// string name = 1;
inline void Method::clear_name() {
name_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena());
name_.ClearToEmpty();
}
inline const std::string& Method::name() const {
// @@protoc_insertion_point(field_get:google.protobuf.Method.name)
@ -1090,31 +1087,30 @@ inline const std::string& Method::_internal_name() const {
}
inline void Method::_internal_set_name(const std::string& value) {
name_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value, GetArena());
name_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, value, GetArena());
}
inline void Method::set_name(std::string&& value) {
name_.Set(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArena());
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::move(value), GetArena());
// @@protoc_insertion_point(field_set_rvalue:google.protobuf.Method.name)
}
inline void Method::set_name(const char* value) {
GOOGLE_DCHECK(value != nullptr);
name_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value),
GetArena());
name_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(value), GetArena());
// @@protoc_insertion_point(field_set_char:google.protobuf.Method.name)
}
inline void Method::set_name(const char* value,
size_t size) {
name_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(
name_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(
reinterpret_cast<const char*>(value), size), GetArena());
// @@protoc_insertion_point(field_set_pointer:google.protobuf.Method.name)
}
inline std::string* Method::_internal_mutable_name() {
return name_.Mutable(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena());
return name_.Mutable(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, GetArena());
}
inline std::string* Method::release_name() {
// @@protoc_insertion_point(field_release:google.protobuf.Method.name)
@ -1133,7 +1129,7 @@ inline void Method::set_allocated_name(std::string* name) {
// string request_type_url = 2;
inline void Method::clear_request_type_url() {
request_type_url_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena());
request_type_url_.ClearToEmpty();
}
inline const std::string& Method::request_type_url() const {
// @@protoc_insertion_point(field_get:google.protobuf.Method.request_type_url)
@ -1152,31 +1148,30 @@ inline const std::string& Method::_internal_request_type_url() const {
}
inline void Method::_internal_set_request_type_url(const std::string& value) {
request_type_url_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value, GetArena());
request_type_url_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, value, GetArena());
}
inline void Method::set_request_type_url(std::string&& value) {
request_type_url_.Set(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArena());
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::move(value), GetArena());
// @@protoc_insertion_point(field_set_rvalue:google.protobuf.Method.request_type_url)
}
inline void Method::set_request_type_url(const char* value) {
GOOGLE_DCHECK(value != nullptr);
request_type_url_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value),
GetArena());
request_type_url_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(value), GetArena());
// @@protoc_insertion_point(field_set_char:google.protobuf.Method.request_type_url)
}
inline void Method::set_request_type_url(const char* value,
size_t size) {
request_type_url_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(
request_type_url_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(
reinterpret_cast<const char*>(value), size), GetArena());
// @@protoc_insertion_point(field_set_pointer:google.protobuf.Method.request_type_url)
}
inline std::string* Method::_internal_mutable_request_type_url() {
return request_type_url_.Mutable(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena());
return request_type_url_.Mutable(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, GetArena());
}
inline std::string* Method::release_request_type_url() {
// @@protoc_insertion_point(field_release:google.protobuf.Method.request_type_url)
@ -1215,7 +1210,7 @@ inline void Method::set_request_streaming(bool value) {
// string response_type_url = 4;
inline void Method::clear_response_type_url() {
response_type_url_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena());
response_type_url_.ClearToEmpty();
}
inline const std::string& Method::response_type_url() const {
// @@protoc_insertion_point(field_get:google.protobuf.Method.response_type_url)
@ -1234,31 +1229,30 @@ inline const std::string& Method::_internal_response_type_url() const {
}
inline void Method::_internal_set_response_type_url(const std::string& value) {
response_type_url_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value, GetArena());
response_type_url_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, value, GetArena());
}
inline void Method::set_response_type_url(std::string&& value) {
response_type_url_.Set(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArena());
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::move(value), GetArena());
// @@protoc_insertion_point(field_set_rvalue:google.protobuf.Method.response_type_url)
}
inline void Method::set_response_type_url(const char* value) {
GOOGLE_DCHECK(value != nullptr);
response_type_url_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value),
GetArena());
response_type_url_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(value), GetArena());
// @@protoc_insertion_point(field_set_char:google.protobuf.Method.response_type_url)
}
inline void Method::set_response_type_url(const char* value,
size_t size) {
response_type_url_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(
response_type_url_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(
reinterpret_cast<const char*>(value), size), GetArena());
// @@protoc_insertion_point(field_set_pointer:google.protobuf.Method.response_type_url)
}
inline std::string* Method::_internal_mutable_response_type_url() {
return response_type_url_.Mutable(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena());
return response_type_url_.Mutable(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, GetArena());
}
inline std::string* Method::release_response_type_url() {
// @@protoc_insertion_point(field_release:google.protobuf.Method.response_type_url)
@ -1357,7 +1351,7 @@ inline void Method::set_syntax(PROTOBUF_NAMESPACE_ID::Syntax value) {
// string name = 1;
inline void Mixin::clear_name() {
name_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena());
name_.ClearToEmpty();
}
inline const std::string& Mixin::name() const {
// @@protoc_insertion_point(field_get:google.protobuf.Mixin.name)
@ -1376,31 +1370,30 @@ inline const std::string& Mixin::_internal_name() const {
}
inline void Mixin::_internal_set_name(const std::string& value) {
name_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value, GetArena());
name_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, value, GetArena());
}
inline void Mixin::set_name(std::string&& value) {
name_.Set(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArena());
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::move(value), GetArena());
// @@protoc_insertion_point(field_set_rvalue:google.protobuf.Mixin.name)
}
inline void Mixin::set_name(const char* value) {
GOOGLE_DCHECK(value != nullptr);
name_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value),
GetArena());
name_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(value), GetArena());
// @@protoc_insertion_point(field_set_char:google.protobuf.Mixin.name)
}
inline void Mixin::set_name(const char* value,
size_t size) {
name_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(
name_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(
reinterpret_cast<const char*>(value), size), GetArena());
// @@protoc_insertion_point(field_set_pointer:google.protobuf.Mixin.name)
}
inline std::string* Mixin::_internal_mutable_name() {
return name_.Mutable(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena());
return name_.Mutable(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, GetArena());
}
inline std::string* Mixin::release_name() {
// @@protoc_insertion_point(field_release:google.protobuf.Mixin.name)
@ -1419,7 +1412,7 @@ inline void Mixin::set_allocated_name(std::string* name) {
// string root = 2;
inline void Mixin::clear_root() {
root_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena());
root_.ClearToEmpty();
}
inline const std::string& Mixin::root() const {
// @@protoc_insertion_point(field_get:google.protobuf.Mixin.root)
@ -1438,31 +1431,30 @@ inline const std::string& Mixin::_internal_root() const {
}
inline void Mixin::_internal_set_root(const std::string& value) {
root_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value, GetArena());
root_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, value, GetArena());
}
inline void Mixin::set_root(std::string&& value) {
root_.Set(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArena());
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::move(value), GetArena());
// @@protoc_insertion_point(field_set_rvalue:google.protobuf.Mixin.root)
}
inline void Mixin::set_root(const char* value) {
GOOGLE_DCHECK(value != nullptr);
root_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value),
GetArena());
root_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(value), GetArena());
// @@protoc_insertion_point(field_set_char:google.protobuf.Mixin.root)
}
inline void Mixin::set_root(const char* value,
size_t size) {
root_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(
root_.Set(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, ::std::string(
reinterpret_cast<const char*>(value), size), GetArena());
// @@protoc_insertion_point(field_set_pointer:google.protobuf.Mixin.root)
}
inline std::string* Mixin::_internal_mutable_root() {
return root_.Mutable(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena());
return root_.Mutable(::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr::EmptyDefault{}, GetArena());
}
inline std::string* Mixin::release_root() {
// @@protoc_insertion_point(field_release:google.protobuf.Mixin.root)

304
src/google/protobuf/arena.cc
View File

@ -47,10 +47,14 @@ static const size_t kMaxCleanupListElements = 64; // 1kB on 64-bit.
namespace google {
namespace protobuf {
PROTOBUF_EXPORT /*static*/ void* (*const ArenaOptions::kDefaultBlockAlloc)(
size_t) = &::operator new;
namespace internal {
std::atomic<LifecycleId> ArenaImpl::lifecycle_id_generator_;
ArenaImpl::CacheAlignedLifecycleIdGenerator ArenaImpl::lifecycle_id_generator_;
#if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL)
ArenaImpl::ThreadCache& ArenaImpl::thread_cache() {
static internal::ThreadLocalStorage<ThreadCache>* thread_cache_ =
@ -59,78 +63,205 @@ ArenaImpl::ThreadCache& ArenaImpl::thread_cache() {
}
#elif defined(PROTOBUF_USE_DLLS)
ArenaImpl::ThreadCache& ArenaImpl::thread_cache() {
static PROTOBUF_THREAD_LOCAL ThreadCache thread_cache_ = {-1, NULL};
static PROTOBUF_THREAD_LOCAL ThreadCache thread_cache_ = {
0, static_cast<LifecycleIdAtomic>(-1), nullptr};
return thread_cache_;
}
#else
PROTOBUF_THREAD_LOCAL ArenaImpl::ThreadCache ArenaImpl::thread_cache_ = {-1,
NULL};
PROTOBUF_THREAD_LOCAL ArenaImpl::ThreadCache ArenaImpl::thread_cache_ = {
0, static_cast<LifecycleIdAtomic>(-1), nullptr};
#endif
void ArenaImpl::Init() {
lifecycle_id_ =
lifecycle_id_generator_.fetch_add(1, std::memory_order_relaxed);
void ArenaFree(void* object, size_t size) {
#if defined(__GXX_DELETE_WITH_SIZE__) || defined(__cpp_sized_deallocation)
::operator delete(object, size);
#else
(void)size;
::operator delete(object);
#endif
}
ArenaImpl::ArenaImpl(const ArenaOptions& options) {
ArenaMetricsCollector* collector = nullptr;
bool record_allocs = false;
if (options.make_metrics_collector != nullptr) {
collector = (*options.make_metrics_collector)();
record_allocs = (collector && collector->RecordAllocs());
}
// Get memory where we can store non-default options if needed.
// Use supplied initial_block if it is large enough.
size_t min_block_size = kOptionsSize + kBlockHeaderSize + kSerialArenaSize;
char* mem = options.initial_block;
size_t mem_size = options.initial_block_size;
GOOGLE_DCHECK_EQ(reinterpret_cast<uintptr_t>(mem) & 7, 0);
if (mem == nullptr || mem_size < min_block_size) {
// Supplied initial block is not big enough.
mem_size = std::max(min_block_size, options.start_block_size);
mem = reinterpret_cast<char*>((*options.block_alloc)(mem_size));
}
// Create the special block.
const bool special = true;
const bool user_owned = (mem == options.initial_block);
auto block =
new (mem) SerialArena::Block(mem_size, nullptr, special, user_owned);
// Options occupy the beginning of the initial block.
options_ = new (block->Pointer(block->pos())) Options;
#ifdef ADDRESS_SANITIZER
ASAN_UNPOISON_MEMORY_REGION(options_, kOptionsSize);
#endif // ADDRESS_SANITIZER
options_->start_block_size = options.start_block_size;
options_->max_block_size = options.max_block_size;
options_->block_alloc = options.block_alloc;
options_->block_dealloc = options.block_dealloc;
options_->metrics_collector = collector;
block->set_pos(block->pos() + kOptionsSize);
Init(record_allocs);
SetInitialBlock(block);
}
void ArenaImpl::Init(bool record_allocs) {
ThreadCache& tc = thread_cache();
auto id = tc.next_lifecycle_id;
constexpr uint64 kInc = ThreadCache::kPerThreadIds * 2;
if (PROTOBUF_PREDICT_FALSE((id & (kInc - 1)) == 0)) {
if (sizeof(lifecycle_id_generator_.id) == 4) {
// 2^32 is dangerous low to guarantee uniqueness. If we start dolling out
// unique id's in ranges of kInc it's unacceptably low. In this case
// we increment by 1. The additional range of kPerThreadIds that are used
// per thread effectively pushes the overflow time from weeks to years
// of continuous running.
id = lifecycle_id_generator_.id.fetch_add(1, std::memory_order_relaxed) *
kInc;
} else {
id =
lifecycle_id_generator_.id.fetch_add(kInc, std::memory_order_relaxed);
}
}
tc.next_lifecycle_id = id + 2;
// We store "record_allocs" in the low bit of lifecycle_id_.
lifecycle_id_ = id | (record_allocs ? 1 : 0);
hint_.store(nullptr, std::memory_order_relaxed);
threads_.store(nullptr, std::memory_order_relaxed);
space_allocated_.store(0, std::memory_order_relaxed);
}
if (initial_block_) {
// Thread which calls Init() owns the first block. This allows the
// single-threaded case to allocate on the first block without having to
// perform atomic operations.
new (initial_block_) Block(options_.initial_block_size, NULL);
SerialArena* serial =
SerialArena::New(initial_block_, &thread_cache(), this);
serial->set_next(NULL);
threads_.store(serial, std::memory_order_relaxed);
space_allocated_.store(options_.initial_block_size,
std::memory_order_relaxed);
CacheSerialArena(serial);
} else {
space_allocated_.store(0, std::memory_order_relaxed);
}
void ArenaImpl::SetInitialBlock(SerialArena::Block* block) {
// Calling thread owns the first block. This allows the single-threaded case
// to allocate on the first block without having to perform atomic operations.
SerialArena* serial = SerialArena::New(block, &thread_cache(), this);
serial->set_next(NULL);
threads_.store(serial, std::memory_order_relaxed);
space_allocated_.store(block->size(), std::memory_order_relaxed);
CacheSerialArena(serial);
}
ArenaImpl::~ArenaImpl() {
// Have to do this in a first pass, because some of the destructors might
// refer to memory in other blocks.
CleanupList();
FreeBlocks();
ArenaMetricsCollector* collector = nullptr;
auto deallocator = &ArenaFree;
if (options_) {
collector = options_->metrics_collector;
deallocator = options_->block_dealloc;
}
PerBlock([deallocator](SerialArena::Block* b) {
#ifdef ADDRESS_SANITIZER
// This memory was provided by the underlying allocator as unpoisoned, so
// return it in an unpoisoned state.
ASAN_UNPOISON_MEMORY_REGION(b->Pointer(0), b->size());
#endif // ADDRESS_SANITIZER
if (!b->user_owned()) {
(*deallocator)(b, b->size());
}
});
if (collector) {
collector->OnDestroy(SpaceAllocated());
}
}
uint64 ArenaImpl::Reset() {
if (options_ && options_->metrics_collector) {
options_->metrics_collector->OnReset(SpaceAllocated());
}
// Have to do this in a first pass, because some of the destructors might
// refer to memory in other blocks.
CleanupList();
uint64 space_allocated = FreeBlocks();
Init();
// Discard all blocks except the special block (if present).
uint64 space_allocated = 0;
SerialArena::Block* special_block = nullptr;
auto deallocator = (options_ ? options_->block_dealloc : &ArenaFree);
PerBlock(
[&space_allocated, &special_block, deallocator](SerialArena::Block* b) {
space_allocated += b->size();
#ifdef ADDRESS_SANITIZER
// This memory was provided by the underlying allocator as unpoisoned,
// so return it in an unpoisoned state.
ASAN_UNPOISON_MEMORY_REGION(b->Pointer(0), b->size());
#endif // ADDRESS_SANITIZER
if (!b->special()) {
(*deallocator)(b, b->size());
} else {
// Prepare special block for reuse.
// Note: if options_ is present, it occupies the beginning of the
// block and therefore pos is advanced past it.
GOOGLE_DCHECK(special_block == nullptr);
special_block = b;
}
});
Init(record_allocs());
if (special_block != nullptr) {
// next() should still be nullptr since we are using a stack discipline, but
// clear it anyway to reduce fragility.
GOOGLE_DCHECK_EQ(special_block->next(), nullptr);
special_block->clear_next();
special_block->set_pos(kBlockHeaderSize + (options_ ? kOptionsSize : 0));
SetInitialBlock(special_block);
}
return space_allocated;
}
ArenaImpl::Block* ArenaImpl::NewBlock(Block* last_block, size_t min_bytes) {
std::pair<void*, size_t> ArenaImpl::NewBuffer(size_t last_size,
size_t min_bytes) {
size_t size;
if (last_block) {
if (last_size != -1) {
// Double the current block size, up to a limit.
size = std::min(2 * last_block->size(), options_.max_block_size);
auto max_size = options_ ? options_->max_block_size : kDefaultMaxBlockSize;
size = std::min(2 * last_size, max_size);
} else {
size = options_.start_block_size;
size = options_ ? options_->start_block_size : kDefaultStartBlockSize;
}
// Verify that min_bytes + kBlockHeaderSize won't overflow.
GOOGLE_CHECK_LE(min_bytes, std::numeric_limits<size_t>::max() - kBlockHeaderSize);
size = std::max(size, kBlockHeaderSize + min_bytes);
void* mem = options_.block_alloc(size);
Block* b = new (mem) Block(size, last_block);
void* mem = options_ ? (*options_->block_alloc)(size) : ::operator new(size);
space_allocated_.fetch_add(size, std::memory_order_relaxed);
return {mem, size};
}
SerialArena::Block* SerialArena::NewBlock(SerialArena::Block* last_block,
size_t min_bytes, ArenaImpl* arena) {
void* mem;
size_t size;
std::tie(mem, size) =
arena->NewBuffer(last_block ? last_block->size() : -1, min_bytes);
Block* b = new (mem) Block(size, last_block, false, false);
return b;
}
ArenaImpl::Block::Block(size_t size, Block* next)
: next_(next), pos_(kBlockHeaderSize), size_(size) {}
PROTOBUF_NOINLINE
void ArenaImpl::SerialArena::AddCleanupFallback(void* elem,
void (*cleanup)(void*)) {
void SerialArena::AddCleanupFallback(void* elem, void (*cleanup)(void*)) {
size_t size = cleanup_ ? cleanup_->size * 2 : kMinCleanupListElements;
size = std::min(size, kMaxCleanupListElements);
size_t bytes = internal::AlignUpTo8(CleanupChunk::SizeOf(size));
@ -182,11 +313,11 @@ void ArenaImpl::AddCleanupFallback(void* elem, void (*cleanup)(void*)) {
}
PROTOBUF_NOINLINE
void* ArenaImpl::SerialArena::AllocateAlignedFallback(size_t n) {
void* SerialArena::AllocateAlignedFallback(size_t n) {
// Sync back to current's pos.
head_->set_pos(head_->size() - (limit_ - ptr_));
head_ = arena_->NewBlock(head_, n);
head_ = NewBlock(head_, n, arena_);
ptr_ = head_->Pointer(head_->pos());
limit_ = head_->Pointer(head_->size());
@ -207,10 +338,14 @@ uint64 ArenaImpl::SpaceUsed() const {
for (; serial; serial = serial->next()) {
space_used += serial->SpaceUsed();
}
// Remove the overhead of Options structure, if any.
if (options_) {
space_used -= kOptionsSize;
}
return space_used;
}
uint64 ArenaImpl::SerialArena::SpaceUsed() const {
uint64 SerialArena::SpaceUsed() const {
// Get current block's size from ptr_ (since we can't trust head_->pos().
uint64 space_used = ptr_ - head_->Pointer(kBlockHeaderSize);
// Get subsequent block size from b->pos().
@ -218,57 +353,10 @@ uint64 ArenaImpl::SerialArena::SpaceUsed() const {
space_used += (b->pos() - kBlockHeaderSize);
}
// Remove the overhead of the SerialArena itself.
space_used -= kSerialArenaSize;
space_used -= ArenaImpl::kSerialArenaSize;
return space_used;
}
uint64 ArenaImpl::FreeBlocks() {
uint64 space_allocated = 0;
// By omitting an Acquire barrier we ensure that any user code that doesn't
// properly synchronize Reset() or the destructor will throw a TSAN warning.
SerialArena* serial = threads_.load(std::memory_order_relaxed);
while (serial) {
// This is inside a block we are freeing, so we need to read it now.
SerialArena* next = serial->next();
space_allocated += ArenaImpl::SerialArena::Free(serial, initial_block_,
options_.block_dealloc);
// serial is dead now.
serial = next;
}
return space_allocated;
}
uint64 ArenaImpl::SerialArena::Free(ArenaImpl::SerialArena* serial,
Block* initial_block,
void (*block_dealloc)(void*, size_t)) {
uint64 space_allocated = 0;
// We have to be careful in this function, since we will be freeing the Block
// that contains this SerialArena. Be careful about accessing |serial|.
for (Block* b = serial->head_; b;) {
// This is inside the block we are freeing, so we need to read it now.
Block* next_block = b->next();
space_allocated += (b->size());
#ifdef ADDRESS_SANITIZER
// This memory was provided by the underlying allocator as unpoisoned, so
// return it in an unpoisoned state.
ASAN_UNPOISON_MEMORY_REGION(b->Pointer(0), b->size());
#endif // ADDRESS_SANITIZER
if (b != initial_block) {
block_dealloc(b, b->size());
}
b = next_block;
}
return space_allocated;
}
void ArenaImpl::CleanupList() {
// By omitting an Acquire barrier we ensure that any user code that doesn't
// properly synchronize Reset() or the destructor will throw a TSAN warning.
@ -279,13 +367,13 @@ void ArenaImpl::CleanupList() {
}
}
void ArenaImpl::SerialArena::CleanupList() {
void SerialArena::CleanupList() {
if (cleanup_ != NULL) {
CleanupListFallback();
}
}
void ArenaImpl::SerialArena::CleanupListFallback() {
void SerialArena::CleanupListFallback() {
// The first chunk might be only partially full, so calculate its size
// from cleanup_ptr_. Subsequent chunks are always full, so use list->size.
size_t n = cleanup_ptr_ - &cleanup_->nodes[0];
@ -305,13 +393,11 @@ void ArenaImpl::SerialArena::CleanupListFallback() {
}
}
ArenaImpl::SerialArena* ArenaImpl::SerialArena::New(Block* b, void* owner,
ArenaImpl* arena) {
GOOGLE_DCHECK_EQ(b->pos(), kBlockHeaderSize); // Should be a fresh block
GOOGLE_DCHECK_LE(kBlockHeaderSize + kSerialArenaSize, b->size());
SerialArena* serial =
reinterpret_cast<SerialArena*>(b->Pointer(kBlockHeaderSize));
b->set_pos(kBlockHeaderSize + kSerialArenaSize);
SerialArena* SerialArena::New(Block* b, void* owner, ArenaImpl* arena) {
auto pos = b->pos();
GOOGLE_DCHECK_LE(pos + ArenaImpl::kSerialArenaSize, b->size());
SerialArena* serial = reinterpret_cast<SerialArena*>(b->Pointer(pos));
b->set_pos(pos + ArenaImpl::kSerialArenaSize);
serial->arena_ = arena;
serial->owner_ = owner;
serial->head_ = b;
@ -324,7 +410,7 @@ ArenaImpl::SerialArena* ArenaImpl::SerialArena::New(Block* b, void* owner,
}
PROTOBUF_NOINLINE
ArenaImpl::SerialArena* ArenaImpl::GetSerialArenaFallback(void* me) {
SerialArena* ArenaImpl::GetSerialArenaFallback(void* me) {
// Look for this SerialArena in our linked list.
SerialArena* serial = threads_.load(std::memory_order_acquire);
for (; serial; serial = serial->next()) {
@ -336,7 +422,7 @@ ArenaImpl::SerialArena* ArenaImpl::GetSerialArenaFallback(void* me) {
if (!serial) {
// This thread doesn't have any SerialArena, which also means it doesn't
// have any blocks yet. So we'll allocate its first block now.
Block* b = NewBlock(NULL, kSerialArenaSize);
SerialArena::Block* b = SerialArena::NewBlock(NULL, kSerialArenaSize, this);
serial = SerialArena::New(b, me, this);
SerialArena* head = threads_.load(std::memory_order_relaxed);
@ -350,6 +436,8 @@ ArenaImpl::SerialArena* ArenaImpl::GetSerialArenaFallback(void* me) {
return serial;
}
ArenaMetricsCollector::~ArenaMetricsCollector() {}
} // namespace internal
PROTOBUF_FUNC_ALIGN(32)
@ -357,25 +445,5 @@ void* Arena::AllocateAlignedNoHook(size_t n) {
return impl_.AllocateAligned(n);
}
void Arena::CallDestructorHooks() {
uint64 space_allocated = impl_.SpaceAllocated();
// Call the reset hook
if (on_arena_reset_ != NULL) {
on_arena_reset_(this, hooks_cookie_, space_allocated);
}
// Call the destruction hook
if (on_arena_destruction_ != NULL) {
on_arena_destruction_(this, hooks_cookie_, space_allocated);
}
}
void Arena::OnArenaAllocation(const std::type_info* allocated_type,
size_t n) const {
if (on_arena_allocation_ != NULL) {
on_arena_allocation_(allocated_type, n, hooks_cookie_);
}
}
} // namespace protobuf
} // namespace google

148
src/google/protobuf/arena.h
View File

@ -102,15 +102,6 @@ template <typename T>
void arena_delete_object(void* object) {
delete reinterpret_cast<T*>(object);
}
inline void arena_free(void* object, size_t size) {
#if defined(__GXX_DELETE_WITH_SIZE__) || defined(__cpp_sized_deallocation)
::operator delete(object, size);
#else
(void)size;
::operator delete(object);
#endif
}
} // namespace internal
// ArenaOptions provides optional additional parameters to arena construction
@ -151,44 +142,30 @@ struct ArenaOptions {
max_block_size(kDefaultMaxBlockSize),
initial_block(NULL),
initial_block_size(0),
block_alloc(&::operator new),
block_dealloc(&internal::arena_free),
on_arena_init(NULL),
on_arena_reset(NULL),
on_arena_destruction(NULL),
on_arena_allocation(NULL) {}
block_alloc(kDefaultBlockAlloc),
block_dealloc(&internal::ArenaFree),
make_metrics_collector(nullptr) {}
PROTOBUF_EXPORT static void* (*const kDefaultBlockAlloc)(size_t);
private:
// Hooks for adding external functionality such as user-specific metrics
// collection, specific debugging abilities, etc.
// Init hook (if set) will always be called at Arena init time. Init hook may
// return a pointer to a cookie to be stored in the arena. Reset and
// destruction hooks will then be called with the same cookie pointer. This
// allows us to save an external object per arena instance and use it on the
// other hooks (Note: If init hook returns NULL, the other hooks will NOT be
// called on this arena instance).
// on_arena_reset and on_arena_destruction also receive the space used in the
// arena just before the reset.
void* (*on_arena_init)(Arena* arena);
void (*on_arena_reset)(Arena* arena, void* cookie, uint64 space_used);
void (*on_arena_destruction)(Arena* arena, void* cookie, uint64 space_used);
// type_info is promised to be static - its lifetime extends to
// match program's lifetime (It is given by typeid operator).
// Note: typeid(void) will be passed as allocated_type every time we
// intentionally want to avoid monitoring an allocation. (i.e. internal
// allocations for managing the arena)
void (*on_arena_allocation)(const std::type_info* allocated_type,
uint64 alloc_size, void* cookie);
// If make_metrics_collector is not nullptr, it will be called at Arena init
// time. It may return a pointer to a collector instance that will be notified
// of interesting events related to the arena.
internal::ArenaMetricsCollector* (*make_metrics_collector)();
// Constants define default starting block size and max block size for
// arena allocator behavior -- see descriptions above.
static const size_t kDefaultStartBlockSize = 256;
static const size_t kDefaultMaxBlockSize = 8192;
static const size_t kDefaultStartBlockSize =
internal::ArenaImpl::kDefaultStartBlockSize;
static const size_t kDefaultMaxBlockSize =
internal::ArenaImpl::kDefaultMaxBlockSize;
friend void arena_metrics::EnableArenaMetrics(ArenaOptions*);
friend class Arena;
friend class ArenaOptionsTestFriend;
friend class internal::ArenaImpl;
};
// Support for non-RTTI environments. (The metrics hooks API uses type
@ -246,11 +223,20 @@ struct ArenaOptions {
// should not rely on this protocol.
class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final {
public:
// Arena constructor taking custom options. See ArenaOptions below for
// Default constructor with sensible default options, tuned for average
// use-cases.
inline Arena() : impl_() {}
// Construct an arena with default options, except for the supplied
// initial block. It is more efficient to use this constructor
// instead of passing ArenaOptions if the only configuration needed
// by the caller is supplying an initial block.
inline Arena(char* initial_block, size_t initial_block_size)
: impl_(initial_block, initial_block_size) {}
// Arena constructor taking custom options. See ArenaOptions above for
// descriptions of the options available.
explicit Arena(const ArenaOptions& options) : impl_(options) {
Init(options);
}
explicit Arena(const ArenaOptions& options) : impl_(options) {}
// Block overhead. Use this as a guide for how much to over-allocate the
// initial block if you want an allocation of size N to fit inside it.
@ -261,27 +247,10 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final {
static const size_t kBlockOverhead = internal::ArenaImpl::kBlockHeaderSize +
internal::ArenaImpl::kSerialArenaSize;
// Default constructor with sensible default options, tuned for average
// use-cases.
Arena() : impl_(ArenaOptions()) { Init(ArenaOptions()); }
inline ~Arena() {}
~Arena() {
if (hooks_cookie_) {
CallDestructorHooks();
}
}
void Init(const ArenaOptions& options) {
on_arena_allocation_ = options.on_arena_allocation;
on_arena_reset_ = options.on_arena_reset;
on_arena_destruction_ = options.on_arena_destruction;
// Call the initialization hook
if (options.on_arena_init != NULL) {
hooks_cookie_ = options.on_arena_init(this);
} else {
hooks_cookie_ = NULL;
}
}
// TODO(protobuf-team): Fix callers to use constructor and delete this method.
void Init(const ArenaOptions&) {}
// API to create proto2 message objects on the arena. If the arena passed in
// is NULL, then a heap allocated object is returned. Type T must be a message
@ -362,13 +331,7 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final {
// Any objects allocated on this arena are unusable after this call. It also
// returns the total space used by the arena which is the sums of the sizes
// of the allocated blocks. This method is not thread-safe.
PROTOBUF_NOINLINE uint64 Reset() {
// Call the reset hook
if (on_arena_reset_ != NULL) {
on_arena_reset_(this, hooks_cookie_, impl_.SpaceAllocated());
}
return impl_.Reset();
}
uint64 Reset() { return impl_.Reset(); }
// Adds |object| to a list of heap-allocated objects to be freed with |delete|
// when the arena is destroyed or reset.
@ -515,28 +478,31 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final {
}
}
void CallDestructorHooks();
void OnArenaAllocation(const std::type_info* allocated_type, size_t n) const;
inline void AllocHook(const std::type_info* allocated_type, size_t n) const {
if (PROTOBUF_PREDICT_FALSE(hooks_cookie_ != NULL)) {
OnArenaAllocation(allocated_type, n);
}
impl_.RecordAlloc(allocated_type, n);
}
// Allocate and also optionally call on_arena_allocation callback with the
// allocated type info when the hooks are in place in ArenaOptions and
// the cookie is not null.
// Allocate and also optionally call collector with the allocated type info
// when allocation recording is enabled.
template <typename T>
PROTOBUF_ALWAYS_INLINE void* AllocateInternal(bool skip_explicit_ownership) {
static_assert(alignof(T) <= 8, "T is overaligned, see b/151247138");
const size_t n = internal::AlignUpTo8(sizeof(T));
AllocHook(RTTI_TYPE_ID(T), n);
// Monitor allocation if needed.
impl_.RecordAlloc(RTTI_TYPE_ID(T), n);
if (skip_explicit_ownership) {
return AllocateAlignedNoHook(n);
return AllocateAlignedTo<alignof(T)>(sizeof(T));
} else {
return impl_.AllocateAlignedAndAddCleanup(
n, &internal::arena_destruct_object<T>);
if (alignof(T) <= 8) {
return impl_.AllocateAlignedAndAddCleanup(
n, &internal::arena_destruct_object<T>);
} else {
auto ptr =
reinterpret_cast<uintptr_t>(impl_.AllocateAlignedAndAddCleanup(
sizeof(T) + alignof(T) - 8,
&internal::arena_destruct_object<T>));
return reinterpret_cast<void*>((ptr + alignof(T) - 8) &
(~alignof(T) + 1));
}
}
}
@ -591,10 +557,12 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final {
PROTOBUF_ALWAYS_INLINE T* CreateInternalRawArray(size_t num_elements) {
GOOGLE_CHECK_LE(num_elements, std::numeric_limits<size_t>::max() / sizeof(T))
<< "Requested size is too large to fit into size_t.";
// We count on compiler to realize that if sizeof(T) is a multiple of
// 8 AlignUpTo can be elided.
const size_t n = internal::AlignUpTo8(sizeof(T) * num_elements);
// Monitor allocation if needed.
AllocHook(RTTI_TYPE_ID(T), n);
return static_cast<T*>(AllocateAlignedNoHook(n));
impl_.RecordAlloc(RTTI_TYPE_ID(T), n);
return static_cast<T*>(AllocateAlignedTo<alignof(T)>(n));
}
template <typename T, typename... Args>
@ -687,7 +655,6 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final {
// For friends of arena.
void* AllocateAligned(size_t n) {
AllocHook(NULL, n);
return AllocateAlignedNoHook(internal::AlignUpTo8(n));
}
template<size_t Align>
@ -698,7 +665,7 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final {
// TODO(b/151247138): if the pointer would have been aligned already,
// this is wasting space. We should pass the alignment down.
uintptr_t ptr = reinterpret_cast<uintptr_t>(AllocateAligned(n + Align - 8));
ptr = (ptr + Align - 1) & -Align;
ptr = (ptr + Align - 1) & (~Align + 1);
return reinterpret_cast<void*>(ptr);
}
@ -706,15 +673,6 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final {
internal::ArenaImpl impl_;
void (*on_arena_allocation_)(const std::type_info* allocated_type,
uint64 alloc_size, void* cookie);
void (*on_arena_reset_)(Arena* arena, void* cookie, uint64 space_used);
void (*on_arena_destruction_)(Arena* arena, void* cookie, uint64 space_used);
// The arena may save a cookie it receives from the external on_init hook
// and then use it when calling the on_reset and on_destruction hooks.
void* hooks_cookie_;
template <typename Type>
friend class internal::GenericTypeHandler;
friend struct internal::ArenaStringPtr; // For AllocateAligned.

509
src/google/protobuf/arena_impl.h
View File

@ -48,6 +48,9 @@
namespace google {
namespace protobuf {
struct ArenaOptions;
namespace internal {
inline size_t AlignUpTo8(size_t n) {
@ -55,7 +58,195 @@ inline size_t AlignUpTo8(size_t n) {
return (n + 7) & static_cast<size_t>(-8);
}
using LifecycleId = int64_t;
using LifecycleIdAtomic = uint64_t;
void PROTOBUF_EXPORT ArenaFree(void* object, size_t size);
// MetricsCollector collects stats for a particular arena.
class PROTOBUF_EXPORT ArenaMetricsCollector {
public:
virtual ~ArenaMetricsCollector();
// Invoked when the arena is about to be destroyed. This method will
// typically finalize any metric collection and delete the collector.
// space_allocated is the space used by the arena.
virtual void OnDestroy(uint64 space_allocated) = 0;
// OnReset() is called when the associated arena is reset.
// space_allocated is the space used by the arena just before the reset.
virtual void OnReset(uint64 space_allocated) = 0;
// Does OnAlloc() need to be called? If false, metric collection overhead
// will be reduced since we will not do extra work per allocation.
virtual bool RecordAllocs() = 0;
// OnAlloc is called when an allocation happens.
// type_info is promised to be static - its lifetime extends to
// match program's lifetime (It is given by typeid operator).
// Note: typeid(void) will be passed as allocated_type every time we
// intentionally want to avoid monitoring an allocation. (i.e. internal
// allocations for managing the arena)
virtual void OnAlloc(const std::type_info* allocated_type,
uint64 alloc_size) = 0;
};
class ArenaImpl;
// A thread-unsafe Arena that can only be used within its owning thread.
class PROTOBUF_EXPORT SerialArena {
public:
// Blocks are variable length malloc-ed objects. The following structure
// describes the common header for all blocks.
class PROTOBUF_EXPORT Block {
public:
Block(size_t size, Block* next, bool special, bool user_owned)
: next_and_bits_(reinterpret_cast<uintptr_t>(next) | (special ? 1 : 0) |
(user_owned ? 2 : 0)),
pos_(kBlockHeaderSize),
size_(size) {
GOOGLE_DCHECK_EQ(reinterpret_cast<uintptr_t>(next) & 3, 0u);
}
char* Pointer(size_t n) {
GOOGLE_DCHECK(n <= size_);
return reinterpret_cast<char*>(this) + n;
}
// One of the blocks may be special. This is either a user-supplied
// initial block, or a block we created at startup to hold Options info.
// A special block is not deleted by Reset.
bool special() const { return (next_and_bits_ & 1) != 0; }
// Whether or not this current block is owned by the user.
// Only special blocks can be user_owned.
bool user_owned() const { return (next_and_bits_ & 2) != 0; }
Block* next() const {
const uintptr_t bottom_bits = 3;
return reinterpret_cast<Block*>(next_and_bits_ & ~bottom_bits);
}
void clear_next() {
next_and_bits_ &= 3; // Set next to nullptr, preserve bottom bits.
}
size_t pos() const { return pos_; }
size_t size() const { return size_; }
void set_pos(size_t pos) { pos_ = pos; }
private:
// Holds pointer to next block for this thread + special/user_owned bits.
uintptr_t next_and_bits_;
size_t pos_;
size_t size_;
// data follows
};
// The allocate/free methods here are a little strange, since SerialArena is
// allocated inside a Block which it also manages. This is to avoid doing
// an extra allocation for the SerialArena itself.
// Creates a new SerialArena inside Block* and returns it.
static SerialArena* New(Block* b, void* owner, ArenaImpl* arena);
void CleanupList();
uint64 SpaceUsed() const;
bool HasSpace(size_t n) { return n <= static_cast<size_t>(limit_ - ptr_); }
void* AllocateAligned(size_t n) {
GOOGLE_DCHECK_EQ(internal::AlignUpTo8(n), n); // Must be already aligned.
GOOGLE_DCHECK_GE(limit_, ptr_);
if (PROTOBUF_PREDICT_FALSE(!HasSpace(n))) {
return AllocateAlignedFallback(n);
}
void* ret = ptr_;
ptr_ += n;
#ifdef ADDRESS_SANITIZER
ASAN_UNPOISON_MEMORY_REGION(ret, n);
#endif // ADDRESS_SANITIZER
return ret;
}
// Allocate space if the current region provides enough space.
bool MaybeAllocateAligned(size_t n, void** out) {
GOOGLE_DCHECK_EQ(internal::AlignUpTo8(n), n); // Must be already aligned.
GOOGLE_DCHECK_GE(limit_, ptr_);
if (PROTOBUF_PREDICT_FALSE(!HasSpace(n))) return false;
void* ret = ptr_;
ptr_ += n;
#ifdef ADDRESS_SANITIZER
ASAN_UNPOISON_MEMORY_REGION(ret, n);
#endif // ADDRESS_SANITIZER
*out = ret;
return true;
}
void AddCleanup(void* elem, void (*cleanup)(void*)) {
if (PROTOBUF_PREDICT_FALSE(cleanup_ptr_ == cleanup_limit_)) {
AddCleanupFallback(elem, cleanup);
return;
}
cleanup_ptr_->elem = elem;
cleanup_ptr_->cleanup = cleanup;
cleanup_ptr_++;
}
void* AllocateAlignedAndAddCleanup(size_t n, void (*cleanup)(void*)) {
void* ret = AllocateAligned(n);
AddCleanup(ret, cleanup);
return ret;
}
Block* head() const { return head_; }
void* owner() const { return owner_; }
SerialArena* next() const { return next_; }
void set_next(SerialArena* next) { next_ = next; }
static Block* NewBlock(Block* last_block, size_t min_bytes, ArenaImpl* arena);
private:
// Node contains the ptr of the object to be cleaned up and the associated
// cleanup function ptr.
struct CleanupNode {
void* elem; // Pointer to the object to be cleaned up.
void (*cleanup)(void*); // Function pointer to the destructor or deleter.
};
// Cleanup uses a chunked linked list, to reduce pointer chasing.
struct CleanupChunk {
static size_t SizeOf(size_t i) {
return sizeof(CleanupChunk) + (sizeof(CleanupNode) * (i - 1));
}
size_t size; // Total elements in the list.
CleanupChunk* next; // Next node in the list.
CleanupNode nodes[1]; // True length is |size|.
};
ArenaImpl* arena_; // Containing arena.
void* owner_; // &ThreadCache of this thread;
Block* head_; // Head of linked list of blocks.
CleanupChunk* cleanup_; // Head of cleanup list.
SerialArena* next_; // Next SerialArena in this linked list.
// Next pointer to allocate from. Always 8-byte aligned. Points inside
// head_ (and head_->pos will always be non-canonical). We keep these
// here to reduce indirection.
char* ptr_;
char* limit_;
// Next CleanupList members to append to. These point inside cleanup_.
CleanupNode* cleanup_ptr_;
CleanupNode* cleanup_limit_;
void* AllocateAlignedFallback(size_t n);
void AddCleanupFallback(void* elem, void (*cleanup)(void*));
void CleanupListFallback();
public:
static constexpr size_t kBlockHeaderSize =
(sizeof(Block) + 7) & static_cast<size_t>(-8);
};
// This class provides the core Arena memory allocation library. Different
// implementations only need to implement the public interface below.
@ -65,37 +256,23 @@ using LifecycleId = int64_t;
// use #ifdef the select the best implementation based on hardware / OS.
class PROTOBUF_EXPORT ArenaImpl {
public:
struct Options {
size_t start_block_size;
size_t max_block_size;
char* initial_block;
size_t initial_block_size;
void* (*block_alloc)(size_t);
void (*block_dealloc)(void*, size_t);
static const size_t kDefaultStartBlockSize = 256;
static const size_t kDefaultMaxBlockSize = 8192;
template <typename O>
explicit Options(const O& options)
: start_block_size(options.start_block_size),
max_block_size(options.max_block_size),
initial_block(options.initial_block),
initial_block_size(options.initial_block_size),
block_alloc(options.block_alloc),
block_dealloc(options.block_dealloc) {}
};
ArenaImpl() { Init(false); }
template <typename O>
explicit ArenaImpl(const O& options) : options_(options) {
if (options_.initial_block != NULL && options_.initial_block_size > 0) {
GOOGLE_CHECK_GE(options_.initial_block_size, sizeof(Block))
<< ": Initial block size too small for header.";
initial_block_ = reinterpret_cast<Block*>(options_.initial_block);
} else {
initial_block_ = NULL;
ArenaImpl(char* mem, size_t size) {
GOOGLE_DCHECK_EQ(reinterpret_cast<uintptr_t>(mem) & 7, 0u);
Init(false);
// Ignore initial block if it is too small.
if (mem != nullptr && size >= kBlockHeaderSize + kSerialArenaSize) {
SetInitialBlock(new (mem) SerialArena::Block(size, nullptr, true, true));
}
Init();
}
explicit ArenaImpl(const ArenaOptions& options);
// Destructor deletes all owned heap allocated objects, and destructs objects
// that have non-trivial destructors, except for proto2 message objects whose
// destructors can be skipped. Also, frees all blocks except the initial block
@ -134,178 +311,64 @@ class PROTOBUF_EXPORT ArenaImpl {
// Add object pointer and cleanup function pointer to the list.
void AddCleanup(void* elem, void (*cleanup)(void*));
inline void RecordAlloc(const std::type_info* allocated_type,
size_t n) const {
if (PROTOBUF_PREDICT_FALSE(record_allocs())) {
options_->metrics_collector->OnAlloc(allocated_type, n);
}
}
std::pair<void*, size_t> NewBuffer(size_t last_size, size_t min_bytes);
private:
friend class ArenaBenchmark;
// Pointer to a linked list of SerialArena.
std::atomic<SerialArena*> threads_;
std::atomic<SerialArena*> hint_; // Fast thread-local block access
std::atomic<size_t> space_allocated_; // Total size of all allocated blocks.
// Unique for each arena. Changes on Reset().
// Least-significant-bit is 1 iff allocations should be recorded.
uint64 lifecycle_id_;
struct Options {
size_t start_block_size;
size_t max_block_size;
void* (*block_alloc)(size_t);
void (*block_dealloc)(void*, size_t);
ArenaMetricsCollector* metrics_collector;
};
Options* options_ = nullptr;
void* AllocateAlignedFallback(size_t n);
void* AllocateAlignedAndAddCleanupFallback(size_t n, void (*cleanup)(void*));
void AddCleanupFallback(void* elem, void (*cleanup)(void*));
// Node contains the ptr of the object to be cleaned up and the associated
// cleanup function ptr.
struct CleanupNode {
void* elem; // Pointer to the object to be cleaned up.
void (*cleanup)(void*); // Function pointer to the destructor or deleter.
};
void Init(bool record_allocs);
void SetInitialBlock(
SerialArena::Block* block); // Can be called right after Init()
// Cleanup uses a chunked linked list, to reduce pointer chasing.
struct CleanupChunk {
static size_t SizeOf(size_t i) {
return sizeof(CleanupChunk) + (sizeof(CleanupNode) * (i - 1));
}
size_t size; // Total elements in the list.
CleanupChunk* next; // Next node in the list.
CleanupNode nodes[1]; // True length is |size|.
};
// Return true iff allocations should be recorded in a metrics collector.
inline bool record_allocs() const { return lifecycle_id_ & 1; }
class Block;
// A thread-unsafe Arena that can only be used within its owning thread.
class PROTOBUF_EXPORT SerialArena {
public:
// The allocate/free methods here are a little strange, since SerialArena is
// allocated inside a Block which it also manages. This is to avoid doing
// an extra allocation for the SerialArena itself.
// Creates a new SerialArena inside Block* and returns it.
static SerialArena* New(Block* b, void* owner, ArenaImpl* arena);
// Destroys this SerialArena, freeing all blocks with the given dealloc
// function, except any block equal to |initial_block|.
static uint64 Free(SerialArena* serial, Block* initial_block,
void (*block_dealloc)(void*, size_t));
void CleanupList();
uint64 SpaceUsed() const;
bool HasSpace(size_t n) { return n <= static_cast<size_t>(limit_ - ptr_); }
void* AllocateAligned(size_t n) {
GOOGLE_DCHECK_EQ(internal::AlignUpTo8(n), n); // Must be already aligned.
GOOGLE_DCHECK_GE(limit_, ptr_);
if (PROTOBUF_PREDICT_FALSE(!HasSpace(n))) {
return AllocateAlignedFallback(n);
// Invoke fn(b) for every Block* b.
template <typename Functor>
void PerBlock(Functor fn) {
// By omitting an Acquire barrier we ensure that any user code that doesn't
// properly synchronize Reset() or the destructor will throw a TSAN warning.
SerialArena* serial = threads_.load(std::memory_order_relaxed);
while (serial) {
// fn() may delete blocks and arenas, so fetch next pointers before fn();
SerialArena* cur = serial;
serial = serial->next();
for (auto* block = cur->head(); block != nullptr;) {
auto* b = block;
block = b->next();
fn(b);
}
void* ret = ptr_;
ptr_ += n;
#ifdef ADDRESS_SANITIZER
ASAN_UNPOISON_MEMORY_REGION(ret, n);
#endif // ADDRESS_SANITIZER
return ret;
}
}
// Allocate space if the current region provides enough space.
bool MaybeAllocateAligned(size_t n, void** out) {
GOOGLE_DCHECK_EQ(internal::AlignUpTo8(n), n); // Must be already aligned.
GOOGLE_DCHECK_GE(limit_, ptr_);
if (PROTOBUF_PREDICT_FALSE(!HasSpace(n))) return false;
void* ret = ptr_;
ptr_ += n;
#ifdef ADDRESS_SANITIZER
ASAN_UNPOISON_MEMORY_REGION(ret, n);
#endif // ADDRESS_SANITIZER
*out = ret;
return true;
}
void AddCleanup(void* elem, void (*cleanup)(void*)) {
if (PROTOBUF_PREDICT_FALSE(cleanup_ptr_ == cleanup_limit_)) {
AddCleanupFallback(elem, cleanup);
return;
}
cleanup_ptr_->elem = elem;
cleanup_ptr_->cleanup = cleanup;
cleanup_ptr_++;
}
void* AllocateAlignedAndAddCleanup(size_t n, void (*cleanup)(void*)) {
void* ret = AllocateAligned(n);
AddCleanup(ret, cleanup);
return ret;
}
void* owner() const { return owner_; }
SerialArena* next() const { return next_; }
void set_next(SerialArena* next) { next_ = next; }
private:
void* AllocateAlignedFallback(size_t n);
void AddCleanupFallback(void* elem, void (*cleanup)(void*));
void CleanupListFallback();
ArenaImpl* arena_; // Containing arena.
void* owner_; // &ThreadCache of this thread;
Block* head_; // Head of linked list of blocks.
CleanupChunk* cleanup_; // Head of cleanup list.
SerialArena* next_; // Next SerialArena in this linked list.
// Next pointer to allocate from. Always 8-byte aligned. Points inside
// head_ (and head_->pos will always be non-canonical). We keep these
// here to reduce indirection.
char* ptr_;
char* limit_;
// Next CleanupList members to append to. These point inside cleanup_.
CleanupNode* cleanup_ptr_;
CleanupNode* cleanup_limit_;
};
// Blocks are variable length malloc-ed objects. The following structure
// describes the common header for all blocks.
class PROTOBUF_EXPORT Block {
public:
Block(size_t size, Block* next);
char* Pointer(size_t n) {
GOOGLE_DCHECK(n <= size_);
return reinterpret_cast<char*>(this) + n;
}
Block* next() const { return next_; }
size_t pos() const { return pos_; }
size_t size() const { return size_; }
void set_pos(size_t pos) { pos_ = pos; }
private:
Block* next_; // Next block for this thread.
size_t pos_;
size_t size_;
// data follows
};
struct ThreadCache {
#if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL)
// If we are using the ThreadLocalStorage class to store the ThreadCache,
// then the ThreadCache's default constructor has to be responsible for
// initializing it.
ThreadCache() : last_lifecycle_id_seen(-1), last_serial_arena(NULL) {}
#endif
// The ThreadCache is considered valid as long as this matches the
// lifecycle_id of the arena being used.
LifecycleId last_lifecycle_id_seen;
SerialArena* last_serial_arena;
};
static std::atomic<LifecycleId> lifecycle_id_generator_;
#if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL)
// Android ndk does not support __thread keyword so we use a custom thread
// local storage class we implemented.
// iOS also does not support the __thread keyword.
static ThreadCache& thread_cache();
#elif defined(PROTOBUF_USE_DLLS)
// Thread local variables cannot be exposed through DLL interface but we can
// wrap them in static functions.
static ThreadCache& thread_cache();
#else
static PROTOBUF_THREAD_LOCAL ThreadCache thread_cache_;
static ThreadCache& thread_cache() { return thread_cache_; }
#endif
void Init();
// Free all blocks and return the total space used which is the sums of sizes
// of the all the allocated blocks.
uint64 FreeBlocks();
// Delete or Destruct all objects owned by the arena.
void CleanupList();
@ -319,16 +382,6 @@ class PROTOBUF_EXPORT ArenaImpl {
hint_.store(serial, std::memory_order_release);
}
std::atomic<SerialArena*>
threads_; // Pointer to a linked list of SerialArena.
std::atomic<SerialArena*> hint_; // Fast thread-local block access
std::atomic<size_t> space_allocated_; // Total size of all allocated blocks.
Block* initial_block_; // If non-NULL, points to the block that came from
// user data.
Block* NewBlock(Block* last_block, size_t min_bytes);
PROTOBUF_ALWAYS_INLINE bool GetSerialArenaFast(SerialArena** arena) {
if (GetSerialArenaFromThreadCache(arena)) return true;
@ -356,9 +409,58 @@ class PROTOBUF_EXPORT ArenaImpl {
return false;
}
SerialArena* GetSerialArenaFallback(void* me);
LifecycleId lifecycle_id_; // Unique for each arena. Changes on Reset().
Options options_;
#ifdef _MSC_VER
#pragma warning(disable : 4324)
#endif
struct alignas(64) ThreadCache {
#if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL)
// If we are using the ThreadLocalStorage class to store the ThreadCache,
// then the ThreadCache's default constructor has to be responsible for
// initializing it.
ThreadCache()
: next_lifecycle_id(0),
last_lifecycle_id_seen(-1),
last_serial_arena(NULL) {}
#endif
// Number of per-thread lifecycle IDs to reserve. Must be power of two.
// To reduce contention on a global atomic, each thread reserves a batch of
// IDs. The following number is calculated based on a stress test with
// ~6500 threads all frequently allocating a new arena.
static constexpr size_t kPerThreadIds = 256;
// Next lifecycle ID available to this thread. We need to reserve a new
// batch, if `next_lifecycle_id & (kPerThreadIds - 1) == 0`.
uint64 next_lifecycle_id;
// The ThreadCache is considered valid as long as this matches the
// lifecycle_id of the arena being used.
uint64 last_lifecycle_id_seen;
SerialArena* last_serial_arena;
};
// Lifecycle_id can be highly contended variable in a situation of lots of
// arena creation. Make sure that other global variables are not sharing the
// cacheline.
#ifdef _MSC_VER
#pragma warning(disable : 4324)
#endif
struct alignas(64) CacheAlignedLifecycleIdGenerator {
std::atomic<LifecycleIdAtomic> id;
};
static CacheAlignedLifecycleIdGenerator lifecycle_id_generator_;
#if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL)
// Android ndk does not support __thread keyword so we use a custom thread
// local storage class we implemented.
// iOS also does not support the __thread keyword.
static ThreadCache& thread_cache();
#elif defined(PROTOBUF_USE_DLLS)
// Thread local variables cannot be exposed through DLL interface but we can
// wrap them in static functions.
static ThreadCache& thread_cache();
#else
static PROTOBUF_THREAD_LOCAL ThreadCache thread_cache_;
static ThreadCache& thread_cache() { return thread_cache_; }
#endif
GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ArenaImpl);
// All protos have pointers back to the arena hence Arena must have
@ -369,10 +471,11 @@ class PROTOBUF_EXPORT ArenaImpl {
public:
// kBlockHeaderSize is sizeof(Block), aligned up to the nearest multiple of 8
// to protect the invariant that pos is always at a multiple of 8.
static const size_t kBlockHeaderSize =
(sizeof(Block) + 7) & static_cast<size_t>(-8);
static const size_t kSerialArenaSize =
static constexpr size_t kBlockHeaderSize = SerialArena::kBlockHeaderSize;
static constexpr size_t kSerialArenaSize =
(sizeof(SerialArena) + 7) & static_cast<size_t>(-8);
static constexpr size_t kOptionsSize =
(sizeof(Options) + 7) & static_cast<size_t>(-8);
static_assert(kBlockHeaderSize % 8 == 0,
"kBlockHeaderSize must be a multiple of 8.");
static_assert(kSerialArenaSize % 8 == 0,

202
src/google/protobuf/arena_unittest.cc
View File

@ -31,6 +31,7 @@
#include <google/protobuf/arena.h>
#include <algorithm>
#include <cstddef>
#include <cstring>
#include <memory>
#include <string>
@ -272,28 +273,36 @@ TEST(ArenaTest, CreateWithMoveArguments) {
}
TEST(ArenaTest, InitialBlockTooSmall) {
// Construct a small (64 byte) initial block of memory to be used by the
// arena allocator; then, allocate an object which will not fit in the
// initial block.
std::vector<char> arena_block(96);
ArenaOptions options;
options.initial_block = &arena_block[0];
options.initial_block_size = arena_block.size();
Arena arena(options);
// Construct a small blocks of memory to be used by the arena allocator; then,
// allocate an object which will not fit in the initial block.
for (int size = 0; size <= Arena::kBlockOverhead + 32; size++) {
std::vector<char> arena_block(size);
ArenaOptions options;
options.initial_block = arena_block.data();
options.initial_block_size = arena_block.size();
char* p = Arena::CreateArray<char>(&arena, 96);
uintptr_t allocation = reinterpret_cast<uintptr_t>(p);
// Try sometimes with non-default block sizes so that we exercise paths
// with and without ArenaImpl::Options.
if ((size % 2) != 0) {
options.start_block_size += 8;
}
// Ensure that the arena allocator did not return memory pointing into the
// initial block of memory.
uintptr_t arena_start = reinterpret_cast<uintptr_t>(&arena_block[0]);
uintptr_t arena_end = arena_start + arena_block.size();
EXPECT_FALSE(allocation >= arena_start && allocation < arena_end);
Arena arena(options);
// Write to the memory we allocated; this should (but is not guaranteed to)
// trigger a check for heap corruption if the object was allocated from the
// initially-provided block.
memset(p, '\0', 96);
char* p = Arena::CreateArray<char>(&arena, 96);
uintptr_t allocation = reinterpret_cast<uintptr_t>(p);
// Ensure that the arena allocator did not return memory pointing into the
// initial block of memory.
uintptr_t arena_start = reinterpret_cast<uintptr_t>(arena_block.data());
uintptr_t arena_end = arena_start + arena_block.size();
EXPECT_FALSE(allocation >= arena_start && allocation < arena_end);
// Write to the memory we allocated; this should (but is not guaranteed to)
// trigger a check for heap corruption if the object was allocated from the
// initially-provided block.
memset(p, '\0', 96);
}
}
TEST(ArenaTest, Parsing) {
@ -988,10 +997,7 @@ TEST(ArenaTest, ExtensionsOnArena) {
TEST(ArenaTest, RepeatedFieldOnArena) {
// Preallocate an initial arena block to avoid mallocs during hooked region.
std::vector<char> arena_block(1024 * 1024);
ArenaOptions options;
options.initial_block = &arena_block[0];
options.initial_block_size = arena_block.size();
Arena arena(options);
Arena arena(arena_block.data(), arena_block.size());
{
internal::NoHeapChecker no_heap;
@ -1189,10 +1195,7 @@ TEST(ArenaTest, RepeatedFieldWithNonPODType) {
uint64 Align8(uint64 n) { return (n + 7) & -8; }
TEST(ArenaTest, SpaceAllocated_and_Used) {
ArenaOptions options;
options.start_block_size = 256;
options.max_block_size = 8192;
Arena arena_1(options);
Arena arena_1;
EXPECT_EQ(0, arena_1.SpaceAllocated());
EXPECT_EQ(0, arena_1.SpaceUsed());
EXPECT_EQ(0, arena_1.Reset());
@ -1204,6 +1207,9 @@ TEST(ArenaTest, SpaceAllocated_and_Used) {
// Test with initial block.
std::vector<char> arena_block(1024);
ArenaOptions options;
options.start_block_size = 256;
options.max_block_size = 8192;
options.initial_block = &arena_block[0];
options.initial_block_size = arena_block.size();
Arena arena_2(options);
@ -1214,19 +1220,25 @@ TEST(ArenaTest, SpaceAllocated_and_Used) {
EXPECT_EQ(1024, arena_2.SpaceAllocated());
EXPECT_EQ(Align8(55), arena_2.SpaceUsed());
EXPECT_EQ(1024, arena_2.Reset());
}
// Reset options to test doubling policy explicitly.
options.initial_block = NULL;
options.initial_block_size = 0;
Arena arena_3(options);
EXPECT_EQ(0, arena_3.SpaceUsed());
Arena::CreateArray<char>(&arena_3, 160);
EXPECT_EQ(256, arena_3.SpaceAllocated());
EXPECT_EQ(Align8(160), arena_3.SpaceUsed());
Arena::CreateArray<char>(&arena_3, 70);
EXPECT_EQ(256 + 512, arena_3.SpaceAllocated());
EXPECT_EQ(Align8(160) + Align8(70), arena_3.SpaceUsed());
EXPECT_EQ(256 + 512, arena_3.Reset());
TEST(ArenaTest, BlockSizeDoubling) {
Arena arena;
EXPECT_EQ(0, arena.SpaceUsed());
EXPECT_EQ(0, arena.SpaceAllocated());
// Allocate something to get initial block size.
Arena::CreateArray<char>(&arena, 1);
auto first_block_size = arena.SpaceAllocated();
// Keep allocating until space used increases.
while (arena.SpaceAllocated() == first_block_size) {
Arena::CreateArray<char>(&arena, 1);
}
ASSERT_GT(arena.SpaceAllocated(), first_block_size);
auto second_block_size = (arena.SpaceAllocated() - first_block_size);
EXPECT_EQ(second_block_size, 2*first_block_size);
}
TEST(ArenaTest, Alignment) {
@ -1304,81 +1316,93 @@ TEST(ArenaTest, AddCleanup) {
}
}
// A helper utility class to only contain static hook functions, some
// counters to be used to verify the counters have been called and a cookie
// value to be verified.
class ArenaHooksTestUtil {
namespace {
uint32 hooks_num_init = 0;
uint32 hooks_num_allocations = 0;
uint32 hooks_num_reset = 0;
uint32 hooks_num_destruct = 0;
void ClearHookCounts() {
hooks_num_init = 0;
hooks_num_allocations = 0;
hooks_num_reset = 0;
hooks_num_destruct = 0;
}
} // namespace
// A helper utility class that handles arena callbacks.
class ArenaOptionsTestFriend : public internal::ArenaMetricsCollector {
public:
static void* on_init(Arena* arena) {
++num_init;
int* cookie = new int(kCookieValue);
return static_cast<void*>(cookie);
static internal::ArenaMetricsCollector* NewWithAllocs() {
return new ArenaOptionsTestFriend(true);
}
static void on_allocation(const std::type_info* /*unused*/, uint64 alloc_size,
void* cookie) {
++num_allocations;
int cookie_value = *static_cast<int*>(cookie);
EXPECT_EQ(kCookieValue, cookie_value);
static internal::ArenaMetricsCollector* NewWithoutAllocs() {
return new ArenaOptionsTestFriend(false);
}
static void on_reset(Arena* arena, void* cookie, uint64 space_used) {
++num_reset;
int cookie_value = *static_cast<int*>(cookie);
EXPECT_EQ(kCookieValue, cookie_value);
static void Enable(ArenaOptions* options) {
ClearHookCounts();
options->make_metrics_collector = &ArenaOptionsTestFriend::NewWithAllocs;
}
static void on_destruction(Arena* arena, void* cookie, uint64 space_used) {
++num_destruct;
int cookie_value = *static_cast<int*>(cookie);
EXPECT_EQ(kCookieValue, cookie_value);
delete static_cast<int*>(cookie);
static void EnableWithoutAllocs(ArenaOptions* options) {
ClearHookCounts();
options->make_metrics_collector = &ArenaOptionsTestFriend::NewWithoutAllocs;
}
static const int kCookieValue = 999;
static uint32 num_init;
static uint32 num_allocations;
static uint32 num_reset;
static uint32 num_destruct;
};
uint32 ArenaHooksTestUtil::num_init = 0;
uint32 ArenaHooksTestUtil::num_allocations = 0;
uint32 ArenaHooksTestUtil::num_reset = 0;
uint32 ArenaHooksTestUtil::num_destruct = 0;
const int ArenaHooksTestUtil::kCookieValue;
class ArenaOptionsTestFriend {
public:
static void Set(ArenaOptions* options) {
options->on_arena_init = ArenaHooksTestUtil::on_init;
options->on_arena_allocation = ArenaHooksTestUtil::on_allocation;
options->on_arena_reset = ArenaHooksTestUtil::on_reset;
options->on_arena_destruction = ArenaHooksTestUtil::on_destruction;
explicit ArenaOptionsTestFriend(bool record_allocs)
: record_allocs_(record_allocs) {
++hooks_num_init;
}
void OnDestroy(uint64 space_allocated) override {
++hooks_num_destruct;
delete this;
}
void OnReset(uint64 space_allocated) override { ++hooks_num_reset; }
bool RecordAllocs() override { return record_allocs_; }
void OnAlloc(const std::type_info* allocated_type,
uint64 alloc_size) override {
++hooks_num_allocations;
}
private:
bool record_allocs_;
};
// Test the hooks are correctly called and that the cookie is passed.
// Test the hooks are correctly called.
TEST(ArenaTest, ArenaHooksSanity) {
ArenaOptions options;
ArenaOptionsTestFriend::Set(&options);
ArenaOptionsTestFriend::Enable(&options);
// Scope for defining the arena
{
Arena arena(options);
EXPECT_EQ(1, ArenaHooksTestUtil::num_init);
EXPECT_EQ(0, ArenaHooksTestUtil::num_allocations);
EXPECT_EQ(1, hooks_num_init);
EXPECT_EQ(0, hooks_num_allocations);
Arena::Create<uint64>(&arena);
if (std::is_trivially_destructible<uint64>::value) {
EXPECT_EQ(1, ArenaHooksTestUtil::num_allocations);
EXPECT_EQ(1, hooks_num_allocations);
} else {
EXPECT_EQ(2, ArenaHooksTestUtil::num_allocations);
EXPECT_EQ(2, hooks_num_allocations);
}
arena.Reset();
arena.Reset();
EXPECT_EQ(2, ArenaHooksTestUtil::num_reset);
EXPECT_EQ(2, hooks_num_reset);
}
EXPECT_EQ(3, ArenaHooksTestUtil::num_reset);
EXPECT_EQ(1, ArenaHooksTestUtil::num_destruct);
EXPECT_EQ(2, hooks_num_reset);
EXPECT_EQ(1, hooks_num_destruct);
}
// Test that allocation hooks are not called when we don't need them.
TEST(ArenaTest, ArenaHooksWhenAllocationsNotNeeded) {
ArenaOptions options;
ArenaOptionsTestFriend::EnableWithoutAllocs(&options);
Arena arena(options);
EXPECT_EQ(0, hooks_num_allocations);
Arena::Create<uint64>(&arena);
EXPECT_EQ(0, hooks_num_allocations);
}

254
src/google/protobuf/arenastring.cc Normal file
View File

@ -0,0 +1,254 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <google/protobuf/arenastring.h>
#include <google/protobuf/stubs/logging.h>
#include <google/protobuf/stubs/common.h>
#include <google/protobuf/parse_context.h>
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/message_lite.h>
#include <google/protobuf/stubs/mutex.h>
#include <google/protobuf/stubs/strutil.h>
#include <google/protobuf/stubs/stl_util.h>
// clang-format off
#include <google/protobuf/port_def.inc>
// clang-format on
namespace google {
namespace protobuf {
namespace internal {
const std::string& LazyString::Init() const {
static WrappedMutex mu{GOOGLE_PROTOBUF_LINKER_INITIALIZED};
mu.Lock();
const std::string* res = inited_.load(std::memory_order_acquire);
if (res == nullptr) {
auto init_value = init_value_;
res = ::new (static_cast<void*>(string_buf_))
std::string(init_value.ptr, init_value.size);
inited_.store(res, std::memory_order_release);
}
mu.Unlock();
return *res;
}
void ArenaStringPtr::Set(const std::string* default_value,
ConstStringParam value, ::google::protobuf::Arena* arena) {
if (IsDefault(default_value)) {
tagged_ptr_.Set(Arena::Create<std::string>(arena, value));
} else {
UnsafeMutablePointer()->assign(value.data(), value.length());
}
}
void ArenaStringPtr::Set(const std::string* default_value, std::string&& value,
::google::protobuf::Arena* arena) {
if (IsDefault(default_value)) {
if (arena == nullptr) {
tagged_ptr_.Set(new std::string(std::move(value)));
} else {
tagged_ptr_.Set(Arena::Create<std::string>(arena, std::move(value)));
}
} else if (IsDonatedString()) {
std::string* current = tagged_ptr_.Get();
auto* s = new (current) std::string(std::move(value));
arena->OwnDestructor(s);
tagged_ptr_.Set(s);
} else /* !IsDonatedString() */ {
*UnsafeMutablePointer() = std::move(value);
}
}
void ArenaStringPtr::Set(EmptyDefault, ConstStringParam value,
::google::protobuf::Arena* arena) {
Set(&GetEmptyStringAlreadyInited(), value, arena);
}
void ArenaStringPtr::Set(EmptyDefault, std::string&& value,
::google::protobuf::Arena* arena) {
Set(&GetEmptyStringAlreadyInited(), std::move(value), arena);
}
void ArenaStringPtr::Set(NonEmptyDefault, ConstStringParam value,
::google::protobuf::Arena* arena) {
Set(nullptr, value, arena);
}
void ArenaStringPtr::Set(NonEmptyDefault, std::string&& value,
::google::protobuf::Arena* arena) {
Set(nullptr, std::move(value), arena);
}
std::string* ArenaStringPtr::Mutable(EmptyDefault, ::google::protobuf::Arena* arena) {
if (!IsDonatedString() && !IsDefault(&GetEmptyStringAlreadyInited())) {
return UnsafeMutablePointer();
} else {
return MutableSlow(arena);
}
}
std::string* ArenaStringPtr::Mutable(const LazyString& default_value,
::google::protobuf::Arena* arena) {
if (!IsDonatedString() && !IsDefault(nullptr)) {
return UnsafeMutablePointer();
} else {
return MutableSlow(arena, default_value);
}
}
std::string* ArenaStringPtr::MutableNoCopy(const std::string* default_value,
::google::protobuf::Arena* arena) {
if (!IsDonatedString() && !IsDefault(default_value)) {
return UnsafeMutablePointer();
} else {
GOOGLE_DCHECK(IsDefault(default_value));
// Allocate empty. The contents are not relevant.
std::string* new_string = Arena::Create<std::string>(arena);
tagged_ptr_.Set(new_string);
return new_string;
}
}
template <typename... Lazy>
std::string* ArenaStringPtr::MutableSlow(::google::protobuf::Arena* arena,
const Lazy&... lazy_default) {
const std::string* const default_value =
sizeof...(Lazy) == 0 ? &GetEmptyStringAlreadyInited() : nullptr;
GOOGLE_DCHECK(IsDefault(default_value));
std::string* new_string =
Arena::Create<std::string>(arena, lazy_default.get()...);
tagged_ptr_.Set(new_string);
return new_string;
}
std::string* ArenaStringPtr::Release(const std::string* default_value,
::google::protobuf::Arena* arena) {
if (IsDefault(default_value)) {
return nullptr;
} else {
return ReleaseNonDefault(default_value, arena);
}
}
std::string* ArenaStringPtr::ReleaseNonDefault(const std::string* default_value,
::google::protobuf::Arena* arena) {
GOOGLE_DCHECK(!IsDefault(default_value));
if (!IsDonatedString()) {
std::string* released;
if (arena != nullptr) {
released = new std::string;
released->swap(*UnsafeMutablePointer());
} else {
released = UnsafeMutablePointer();
}
tagged_ptr_.Set(const_cast<std::string*>(default_value));
return released;
} else /* IsDonatedString() */ {
GOOGLE_DCHECK(arena != nullptr);
std::string* released = new std::string(Get());
tagged_ptr_.Set(const_cast<std::string*>(default_value));
return released;
}
}
void ArenaStringPtr::SetAllocated(const std::string* default_value,
std::string* value, ::google::protobuf::Arena* arena) {
// Release what we have first.
if (arena == nullptr && !IsDefault(default_value)) {
delete UnsafeMutablePointer();
}
if (value == nullptr) {
tagged_ptr_.Set(const_cast<std::string*>(default_value));
} else {
#ifdef NDEBUG
tagged_ptr_.Set(value);
if (arena != nullptr) {
arena->Own(value);
}
#else
// On debug builds, copy the string so the address differs. delete will
// fail if value was a stack-allocated temporary/etc., which would have
// failed when arena ran its cleanup list.
std::string* new_value = Arena::Create<std::string>(arena, *value);
delete value;
tagged_ptr_.Set(new_value);
#endif
}
}
void ArenaStringPtr::Destroy(const std::string* default_value,
::google::protobuf::Arena* arena) {
if (arena == nullptr) {
GOOGLE_DCHECK(!IsDonatedString());
if (!IsDefault(default_value)) {
delete UnsafeMutablePointer();
}
}
}
void ArenaStringPtr::Destroy(EmptyDefault, ::google::protobuf::Arena* arena) {
Destroy(&GetEmptyStringAlreadyInited(), arena);
}
void ArenaStringPtr::Destroy(NonEmptyDefault, ::google::protobuf::Arena* arena) {
Destroy(nullptr, arena);
}
void ArenaStringPtr::ClearToEmpty() {
if (IsDefault(&GetEmptyStringAlreadyInited())) {
// Already set to default -- do nothing.
} else {
// Unconditionally mask away the tag.
//
// UpdateDonatedString uses assign when capacity is larger than the new
// value, which is trivially true in the donated string case.
// const_cast<std::string*>(PtrValue<std::string>())->clear();
tagged_ptr_.Get()->clear();
}
}
void ArenaStringPtr::ClearToDefault(const LazyString& default_value,
::google::protobuf::Arena* arena) {
(void)arena;
if (IsDefault(nullptr)) {
// Already set to default -- do nothing.
} else if (!IsDonatedString()) {
UnsafeMutablePointer()->assign(default_value.get());
}
}
} // namespace internal
} // namespace protobuf
} // namespace google

487
src/google/protobuf/arenastring.h
View File

@ -37,7 +37,6 @@
#include <google/protobuf/stubs/logging.h>
#include <google/protobuf/stubs/common.h>
#include <google/protobuf/stubs/fastmem.h>
#include <google/protobuf/arena.h>
#include <google/protobuf/port.h>
@ -48,261 +47,329 @@
#endif
// This is the implementation of arena string fields written for the open-source
// release. The ArenaStringPtr struct below is an internal implementation class
// and *should not be used* by user code. It is used to collect string
// operations together into one place and abstract away the underlying
// string-field pointer representation, so that (for example) an alternate
// implementation that knew more about ::std::string's internals could integrate
// more closely with the arena allocator.
namespace google {
namespace protobuf {
namespace internal {
// Lazy string instance to support string fields with non-empty default.
// These are initialized on the first call to .get().
class PROTOBUF_EXPORT LazyString {
public:
// We explicitly make LazyString an aggregate so that MSVC can do constant
// initialization on it without marking it `constexpr`.
// We do not want to use `constexpr` because it makes it harder to have extern
// storage for it and causes library bloat.
struct InitValue {
const char* ptr;
size_t size;
};
// We keep a union of the initialization value and the std::string to save on
// space. We don't need the string array after Init() is done.
union {
mutable InitValue init_value_;
alignas(std::string) mutable char string_buf_[sizeof(std::string)];
};
mutable std::atomic<const std::string*> inited_;
const std::string& get() const {
// This check generates less code than a call-once invocation.
auto* res = inited_.load(std::memory_order_acquire);
if (PROTOBUF_PREDICT_FALSE(res == nullptr)) return Init();
return *res;
}
private:
// Initialize the string in `string_buf_`, update `inited_` and return it.
// We return it here to avoid having to read it again in the inlined code.
const std::string& Init() const;
};
template <typename T>
class TaggedPtr {
public:
void Set(T* p) { ptr_ = reinterpret_cast<uintptr_t>(p); }
T* Get() const { return reinterpret_cast<T*>(ptr_); }
TaggedPtr() = default;
explicit constexpr TaggedPtr(const std::string* ptr)
: ptr_(const_cast<std::string*>(ptr)) {}
bool IsNull() { return ptr_ == 0; }
void SetTagged(T* p) {
Set(p);
ptr_ = reinterpret_cast<void*>(as_int() | 1);
}
void Set(T* p) { ptr_ = p; }
T* Get() const { return reinterpret_cast<T*>(as_int() & -2); }
bool IsTagged() const { return as_int() & 1; }
// Returned value is only safe to dereference if IsTagged() == false.
// It is safe to compare.
T* UnsafeGet() const { return static_cast<T*>(ptr_); }
bool IsNull() { return ptr_ == nullptr; }
private:
uintptr_t ptr_;
uintptr_t as_int() const { return reinterpret_cast<uintptr_t>(ptr_); }
void* ptr_;
};
struct PROTOBUF_EXPORT ArenaStringPtr {
inline void Set(const ::std::string* default_value,
const ::std::string& value, Arena* arena) {
if (ptr_ == default_value) {
CreateInstance(arena, &value);
} else {
*ptr_ = value;
}
}
static_assert(std::is_trivial<TaggedPtr<std::string>>::value,
"TaggedPtr must be trivial");
inline void SetLite(const ::std::string* default_value,
const ::std::string& value, Arena* arena) {
Set(default_value, value, arena);
}
// This class encapsulates a pointer to a std::string with or without a donated
// buffer, tagged by bottom bit. It is a high-level wrapper that almost directly
// corresponds to the interface required by string fields in generated
// code. It replaces the old std::string* pointer in such cases.
//
// The object has different but similar code paths for when the default value is
// the empty string and when it is a non-empty string.
// The empty string is handled different throughout the library and there is a
// single global instance of it we can share.
//
// For fields with an empty string default value, there are three distinct
// states:
//
// - Pointer set to 'String' tag (LSB is 0), equal to
// &GetEmptyStringAlreadyInited(): field is set to its default value. Points
// to a true std::string*, but we do not own that std::string* (it's a
// globally shared instance).
//
// - Pointer set to 'String' tag (LSB is 0), but not equal to the global empty
// string: field points to a true std::string* instance that we own. This
// instance is either on the heap or on the arena (i.e. registered on
// free()/destructor-call list) as appropriate.
//
// - Pointer set to 'DonatedString' tag (LSB is 1): points to a std::string
// instance with a buffer on the arena (arena != NULL, always, in this case).
//
// For fields with a non-empty string default value, there are three distinct
// states:
//
// - Pointer set to 'String' tag (LSB is 0), equal to `nullptr`:
// Field is in "default" mode and does not point to any actual instance.
// Methods that might need to create an instance of the object will pass a
// `const LazyString&` for it.
//
// - Pointer set to 'String' tag (LSB is 0), but not equal to `nullptr`:
// field points to a true std::string* instance that we own. This instance is
// either on the heap or on the arena (i.e. registered on
// free()/destructor-call list) as appropriate.
//
// - Pointer set to 'DonatedString' tag (LSB is 1): points to a std::string
// instance with a buffer on the arena (arena != NULL, always, in this case).
//
// Generated code and reflection code both ensure that ptr_ is never null for
// fields with an empty default.
// Because ArenaStringPtr is used in oneof unions, its constructor is a NOP and
// so the field is always manually initialized via method calls.
//
// Side-note: why pass information about the default on every API call? Because
// we don't want to hold it in a member variable, or else this would go into
// every proto message instance. This would be a huge waste of space, since the
// default instance pointer is typically a global (static class field). We want
// the generated code to be as efficient as possible, and if we take
// the default value information as a parameter that's in practice taken from a
// static class field, and compare ptr_ to the default value, we end up with a
// single "cmp %reg, GLOBAL" in the resulting machine code. (Note that this also
// requires the String tag to be 0 so we can avoid the mask before comparing.)
struct PROTOBUF_EXPORT ArenaStringPtr {
ArenaStringPtr() = default;
explicit constexpr ArenaStringPtr(const std::string* default_value)
: tagged_ptr_(default_value) {}
// Some methods below are overloaded on a `default_value` and on tags.
// The tagged overloads help reduce code size in the callers in generated
// code, while the `default_value` overloads are useful from reflection.
// By-value empty struct arguments are elided in the ABI.
struct EmptyDefault {};
struct NonEmptyDefault {};
void Set(const std::string* default_value, ConstStringParam value,
::google::protobuf::Arena* arena);
void Set(const std::string* default_value, std::string&& value,
::google::protobuf::Arena* arena);
void Set(EmptyDefault, ConstStringParam value, ::google::protobuf::Arena* arena);
void Set(EmptyDefault, std::string&& value, ::google::protobuf::Arena* arena);
void Set(NonEmptyDefault, ConstStringParam value, ::google::protobuf::Arena* arena);
void Set(NonEmptyDefault, std::string&& value, ::google::protobuf::Arena* arena);
// Basic accessors.
inline const ::std::string& Get() const { return *ptr_; }
inline ::std::string* Mutable(const ::std::string* default_value,
Arena* arena) {
if (ptr_ == default_value) {
CreateInstance(arena, default_value);
}
return ptr_;
const std::string& Get() const PROTOBUF_ALWAYS_INLINE {
// Unconditionally mask away the tag.
return *tagged_ptr_.Get();
}
const std::string* GetPointer() const PROTOBUF_ALWAYS_INLINE {
// Unconditionally mask away the tag.
return tagged_ptr_.Get();
}
// Release returns a ::std::string* instance that is heap-allocated and is not
// Own()'d by any arena. If the field was not set, it returns NULL. The caller
// retains ownership. Clears this field back to NULL state. Used to implement
// release_<field>() methods on generated classes.
inline ::std::string* Release(const ::std::string* default_value,
Arena* arena) {
if (ptr_ == default_value) {
return NULL;
}
return ReleaseNonDefault(default_value, arena);
}
// For fields with an empty default value.
std::string* Mutable(EmptyDefault, ::google::protobuf::Arena* arena);
// For fields with a non-empty default value.
std::string* Mutable(const LazyString& default_value, ::google::protobuf::Arena* arena);
// Similar to Release, but ptr_ cannot be the default_value.
inline ::std::string* ReleaseNonDefault(const ::std::string* default_value,
Arena* arena) {
GOOGLE_DCHECK(!IsDefault(default_value));
::std::string* released = NULL;
if (arena != NULL) {
// ptr_ is owned by the arena.
released = new ::std::string;
released->swap(*ptr_);
} else {
released = ptr_;
}
ptr_ = const_cast< ::std::string*>(default_value);
return released;
}
// Release returns a std::string* instance that is heap-allocated and is not
// Own()'d by any arena. If the field is not set, this returns NULL. The
// caller retains ownership. Clears this field back to NULL state. Used to
// implement release_<field>() methods on generated classes.
std::string* Release(const std::string* default_value,
::google::protobuf::Arena* arena);
std::string* ReleaseNonDefault(const std::string* default_value,
::google::protobuf::Arena* arena);
// UnsafeArenaRelease returns a ::std::string*, but it may be arena-owned
// (i.e. have its destructor already registered) if arena != NULL. If the
// field was not set, this returns NULL. This method clears this field back to
// NULL state. Used to implement unsafe_arena_release_<field>() methods on
// generated classes.
inline ::std::string* UnsafeArenaRelease(const ::std::string* default_value,
Arena* /* arena */) {
if (ptr_ == default_value) {
return NULL;
}
::std::string* released = ptr_;
ptr_ = const_cast< ::std::string*>(default_value);
return released;
}
// Takes a string that is heap-allocated, and takes ownership. The string's
// destructor is registered with the arena. Used to implement
// Takes a std::string that is heap-allocated, and takes ownership. The
// std::string's destructor is registered with the arena. Used to implement
// set_allocated_<field> in generated classes.
inline void SetAllocated(const ::std::string* default_value,
::std::string* value, Arena* arena) {
if (arena == NULL && ptr_ != default_value) {
Destroy(default_value, arena);
}
if (value != NULL) {
ptr_ = value;
if (arena != NULL) {
arena->Own(value);
}
} else {
ptr_ = const_cast< ::std::string*>(default_value);
}
}
// Takes a string that has lifetime equal to the arena's lifetime. The arena
// must be non-null. It is safe only to pass this method a value returned by
// UnsafeArenaRelease() on another field of a message in the same arena. Used
// to implement unsafe_arena_set_allocated_<field> in generated classes.
inline void UnsafeArenaSetAllocated(const ::std::string* default_value,
::std::string* value,
Arena* /* arena */) {
if (value != NULL) {
ptr_ = value;
} else {
ptr_ = const_cast< ::std::string*>(default_value);
}
}
void SetAllocated(const std::string* default_value, std::string* value,
::google::protobuf::Arena* arena);
// Swaps internal pointers. Arena-safety semantics: this is guarded by the
// logic in Swap()/UnsafeArenaSwap() at the message level, so this method is
// 'unsafe' if called directly.
PROTOBUF_ALWAYS_INLINE void Swap(ArenaStringPtr* other) {
std::swap(ptr_, other->ptr_);
}
PROTOBUF_ALWAYS_INLINE void Swap(ArenaStringPtr* other,
const ::std::string* default_value,
Arena* arena) {
#ifndef NDEBUG
// For debug builds, we swap the contents of the string, rather than the
// string instances themselves. This invalidates previously taken const
// references that are (per our documentation) invalidated by calling Swap()
// on the message.
//
// If both strings are the default_value, swapping is uninteresting.
// Otherwise, we use ArenaStringPtr::Mutable() to access the string, to
// ensure that we do not try to mutate default_value itself.
if (IsDefault(default_value) && other->IsDefault(default_value)) {
return;
}
::std::string* this_ptr = Mutable(default_value, arena);
::std::string* other_ptr = other->Mutable(default_value, arena);
this_ptr->swap(*other_ptr);
#else
std::swap(ptr_, other->ptr_);
(void)default_value;
(void)arena;
#endif
}
inline void Swap(ArenaStringPtr* other, const std::string* default_value,
Arena* arena) PROTOBUF_ALWAYS_INLINE;
// Frees storage (if not on an arena).
inline void Destroy(const ::std::string* default_value, Arena* arena) {
if (arena == NULL && ptr_ != default_value) {
delete ptr_;
}
}
void Destroy(const std::string* default_value, ::google::protobuf::Arena* arena);
void Destroy(EmptyDefault, ::google::protobuf::Arena* arena);
void Destroy(NonEmptyDefault, ::google::protobuf::Arena* arena);
// Clears content, but keeps allocated string if arena != NULL, to avoid the
// overhead of heap operations. After this returns, the content (as seen by
// the user) will always be the empty string. Assumes that |default_value|
// is an empty string.
inline void ClearToEmpty(const ::std::string* default_value,
Arena* /* arena */) {
if (ptr_ == default_value) {
// Already set to default (which is empty) -- do nothing.
} else {
ptr_->clear();
}
}
// Clears content, but keeps allocated std::string, to avoid the overhead of
// heap operations. After this returns, the content (as seen by the user) will
// always be the empty std::string. Assumes that |default_value| is an empty
// std::string.
void ClearToEmpty();
// Clears content, assuming that the current value is not the empty string
// default.
inline void ClearNonDefaultToEmpty() { ptr_->clear(); }
// Clears content, assuming that the current value is not the empty
// string default.
void ClearNonDefaultToEmpty();
// Clears content, but keeps allocated string if arena != NULL, to avoid the
// overhead of heap operations. After this returns, the content (as seen by
// the user) will always be equal to |default_value|.
inline void ClearToDefault(const ::std::string* default_value,
Arena* /* arena */) {
if (ptr_ == default_value) {
// Already set to default -- do nothing.
} else {
// Have another allocated string -- rather than throwing this away and
// resetting ptr_ to the canonical default string instance, we just reuse
// this instance.
*ptr_ = *default_value;
}
}
// Clears content, but keeps allocated std::string if arena != NULL, to avoid
// the overhead of heap operations. After this returns, the content (as seen
// by the user) will always be equal to |default_value|.
void ClearToDefault(const LazyString& default_value, ::google::protobuf::Arena* arena);
// Called from generated code / reflection runtime only. Resets value to point
// to a default string pointer, with the semantics that this ArenaStringPtr
// does not own the pointed-to memory. Disregards initial value of ptr_ (so
// this is the *ONLY* safe method to call after construction or when
// reinitializing after becoming the active field in a oneof union).
inline void UnsafeSetDefault(const ::std::string* default_value) {
// Casting away 'const' is safe here: accessors ensure that ptr_ is only
// returned as a const if it is equal to default_value.
ptr_ = const_cast< ::std::string*>(default_value);
}
// to a default string pointer, with the semantics that this
// ArenaStringPtr does not own the pointed-to memory. Disregards initial value
// of ptr_ (so this is the *ONLY* safe method to call after construction or
// when reinitializing after becoming the active field in a oneof union).
inline void UnsafeSetDefault(const std::string* default_value);
// Returns a mutable pointer, but doesn't initialize the string to the
// default value.
std::string* MutableNoArenaNoDefault(const std::string* default_value);
// Get a mutable pointer with unspecified contents.
// Similar to `MutableNoArenaNoDefault`, but also handles the arena case.
// If the value was donated, the contents are discarded.
std::string* MutableNoCopy(const std::string* default_value,
::google::protobuf::Arena* arena);
// Destroy the string. Assumes `arena == nullptr`.
inline void DestroyNoArena(const ::std::string* default_value) {
if (ptr_ != default_value) {
delete ptr_;
}
}
void DestroyNoArena(const std::string* default_value);
// Internal accessor used only at parse time to provide direct access to the
// raw pointer from the shared parse routine (in the non-arenas case). The
// parse routine does the string allocation in order to save code size in the
// generated parsing code.
inline ::std::string** UnsafeRawStringPointer() { return &ptr_; }
inline bool IsDefault(const ::std::string* default_value) const {
return ptr_ == default_value;
}
// Internal accessors!!!!
void UnsafeSetTaggedPointer(TaggedPtr< ::std::string> value) {
ptr_ = value.Get();
// Internal setter used only at parse time to directly set a donated string
// value.
void UnsafeSetTaggedPointer(TaggedPtr<std::string> value) {
tagged_ptr_ = value;
}
// Generated code only! An optimization, in certain cases the generated
// code is certain we can obtain a string with no default checks and
// code is certain we can obtain a std::string with no default checks and
// tag tests.
::std::string* UnsafeMutablePointer() { return ptr_; }
std::string* UnsafeMutablePointer() PROTOBUF_RETURNS_NONNULL;
inline bool IsDefault(const std::string* default_value) const {
// Relies on the fact that kPtrTagString == 0, so if IsString(), ptr_ is the
// actual std::string pointer (and if !IsString(), ptr_ will never be equal
// to any aligned |default_value| pointer). The key is that we want to avoid
// masking in the fastpath const-pointer Get() case for non-arena code.
return tagged_ptr_.UnsafeGet() == default_value;
}
private:
::std::string* ptr_;
TaggedPtr<std::string> tagged_ptr_;
bool IsDonatedString() const { return false; }
// Slow paths.
// MutableSlow requires that !IsString() || IsDefault
// Variadic to support 0 args for EmptyDefault and 1 arg for LazyString.
template <typename... Lazy>
std::string* MutableSlow(::google::protobuf::Arena* arena, const Lazy&... lazy_default);
PROTOBUF_NOINLINE
void CreateInstance(Arena* arena, const ::std::string* initial_value) {
GOOGLE_DCHECK(initial_value != NULL);
// uses "new ::std::string" when arena is nullptr
ptr_ = Arena::Create< ::std::string>(arena, *initial_value);
}
PROTOBUF_NOINLINE
void CreateInstanceNoArena(const ::std::string* initial_value) {
GOOGLE_DCHECK(initial_value != NULL);
ptr_ = new ::std::string(*initial_value);
}
};
inline void ArenaStringPtr::UnsafeSetDefault(const std::string* value) {
tagged_ptr_.Set(const_cast<std::string*>(value));
}
inline void ArenaStringPtr::Swap(ArenaStringPtr* other,
const std::string* default_value,
Arena* arena) {
#ifndef NDEBUG
// For debug builds, we swap the contents of the string, rather than the
// std::string instances themselves. This invalidates previously taken const
// references that are (per our documentation) invalidated by calling Swap()
// on the message.
//
// If both strings are the default_value, swapping is uninteresting.
// Otherwise, we use ArenaStringPtr::Mutable() to access the std::string, to
// ensure that we do not try to mutate default_value itself.
if (IsDefault(default_value) && other->IsDefault(default_value)) {
return;
}
if (default_value == nullptr) {
// If we have non-empty default, then `default_value` is null and we can't
// call Mutable the same way. Just do the regular swap.
std::swap(tagged_ptr_, other->tagged_ptr_);
} else {
std::string* this_ptr = Mutable(EmptyDefault{}, arena);
std::string* other_ptr = other->Mutable(EmptyDefault{}, arena);
this_ptr->swap(*other_ptr);
}
#else
std::swap(tagged_ptr_, other->tagged_ptr_);
#endif
}
inline void ArenaStringPtr::ClearNonDefaultToEmpty() {
// Unconditionally mask away the tag.
tagged_ptr_.Get()->clear();
}
inline std::string* ArenaStringPtr::MutableNoArenaNoDefault(
const std::string* default_value) {
// VERY IMPORTANT for performance and code size: this will reduce to a member
// variable load, a pointer check (against |default_value|, in practice a
// static global) and a branch to the slowpath (which calls operator new and
// the ctor). DO NOT add any tagged-pointer operations here.
if (IsDefault(default_value)) {
std::string* new_string = new std::string();
tagged_ptr_.Set(new_string);
return new_string;
} else {
return UnsafeMutablePointer();
}
}
inline void ArenaStringPtr::DestroyNoArena(const std::string* default_value) {
if (!IsDefault(default_value)) {
delete UnsafeMutablePointer();
}
}
inline std::string* ArenaStringPtr::UnsafeMutablePointer() {
GOOGLE_DCHECK(!tagged_ptr_.IsTagged());
GOOGLE_DCHECK(tagged_ptr_.UnsafeGet() != nullptr);
return tagged_ptr_.UnsafeGet();
}
} // namespace internal
} // namespace protobuf
} // namespace google
#include <google/protobuf/port_undef.inc>
#endif // GOOGLE_PROTOBUF_ARENASTRING_H__

119
src/google/protobuf/arenastring_unittest.cc
View File

@ -28,8 +28,6 @@
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Based on mvels@'s frankenstring.
#include <google/protobuf/arenastring.h>
#include <algorithm>
@ -42,10 +40,14 @@
#include <google/protobuf/stubs/common.h>
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/io/zero_copy_stream_impl.h>
#include <google/protobuf/generated_message_util.h>
#include <gtest/gtest.h>
#include <google/protobuf/stubs/strutil.h>
// Must be included last.
#include <google/protobuf/port_def.inc>
namespace google {
namespace protobuf {
@ -53,82 +55,119 @@ using internal::ArenaStringPtr;
static std::string WrapString(const char* value) { return value; }
using EmptyDefault = ArenaStringPtr::EmptyDefault;
const internal::LazyString nonempty_default{{{"default", 7}}, {nullptr}};
// Test ArenaStringPtr with arena == NULL.
TEST(ArenaStringPtrTest, ArenaStringPtrOnHeap) {
ArenaStringPtr field;
std::string default_value = "default";
field.UnsafeSetDefault(&default_value);
EXPECT_EQ(std::string("default"), field.Get());
field.Set(&default_value, WrapString("Test short"), NULL);
const std::string* empty_default = &internal::GetEmptyString();
field.UnsafeSetDefault(empty_default);
EXPECT_EQ(std::string(""), field.Get());
field.Set(empty_default, WrapString("Test short"), NULL);
EXPECT_EQ(std::string("Test short"), field.Get());
field.Set(&default_value, WrapString("Test long long long long value"), NULL);
field.Set(empty_default, WrapString("Test long long long long value"), NULL);
EXPECT_EQ(std::string("Test long long long long value"), field.Get());
field.Set(&default_value, std::string(""), NULL);
field.Destroy(&default_value, NULL);
field.Set(empty_default, std::string(""), NULL);
field.Destroy(empty_default, NULL);
ArenaStringPtr field2;
field2.UnsafeSetDefault(&default_value);
std::string* mut = field2.Mutable(&default_value, NULL);
EXPECT_EQ(mut, field2.Mutable(&default_value, NULL));
field2.UnsafeSetDefault(empty_default);
std::string* mut = field2.Mutable(EmptyDefault{}, NULL);
EXPECT_EQ(mut, field2.Mutable(EmptyDefault{}, NULL));
EXPECT_EQ(mut, &field2.Get());
EXPECT_NE(&default_value, mut);
EXPECT_EQ(std::string("default"), *mut);
EXPECT_NE(empty_default, mut);
EXPECT_EQ(std::string(""), *mut);
*mut = "Test long long long long value"; // ensure string allocates storage
EXPECT_EQ(std::string("Test long long long long value"), field2.Get());
field2.Destroy(&default_value, NULL);
field2.Destroy(empty_default, NULL);
ArenaStringPtr field3;
field3.UnsafeSetDefault(nullptr);
mut = field3.Mutable(nonempty_default, NULL);
EXPECT_EQ(mut, field3.Mutable(nonempty_default, NULL));
EXPECT_EQ(mut, &field3.Get());
EXPECT_NE(nullptr, mut);
EXPECT_EQ(std::string("default"), *mut);
*mut = "Test long long long long value"; // ensure string allocates storage
EXPECT_EQ(std::string("Test long long long long value"), field3.Get());
field3.Destroy(nullptr, NULL);
}
TEST(ArenaStringPtrTest, ArenaStringPtrOnArena) {
Arena arena;
ArenaStringPtr field;
std::string default_value = "default";
field.UnsafeSetDefault(&default_value);
EXPECT_EQ(std::string("default"), field.Get());
field.Set(&default_value, WrapString("Test short"), &arena);
const std::string* empty_default = &internal::GetEmptyString();
field.UnsafeSetDefault(empty_default);
EXPECT_EQ(std::string(""), field.Get());
field.Set(empty_default, WrapString("Test short"), &arena);
EXPECT_EQ(std::string("Test short"), field.Get());
field.Set(&default_value, WrapString("Test long long long long value"),
field.Set(empty_default, WrapString("Test long long long long value"),
&arena);
EXPECT_EQ(std::string("Test long long long long value"), field.Get());
field.Set(&default_value, std::string(""), &arena);
field.Destroy(&default_value, &arena);
field.Set(empty_default, std::string(""), &arena);
field.Destroy(empty_default, &arena);
ArenaStringPtr field2;
field2.UnsafeSetDefault(&default_value);
std::string* mut = field2.Mutable(&default_value, &arena);
EXPECT_EQ(mut, field2.Mutable(&default_value, &arena));
field2.UnsafeSetDefault(empty_default);
std::string* mut = field2.Mutable(EmptyDefault{}, &arena);
EXPECT_EQ(mut, field2.Mutable(EmptyDefault{}, &arena));
EXPECT_EQ(mut, &field2.Get());
EXPECT_NE(&default_value, mut);
EXPECT_EQ(std::string("default"), *mut);
EXPECT_NE(empty_default, mut);
EXPECT_EQ(std::string(""), *mut);
*mut = "Test long long long long value"; // ensure string allocates storage
EXPECT_EQ(std::string("Test long long long long value"), field2.Get());
field2.Destroy(&default_value, &arena);
field2.Destroy(empty_default, &arena);
ArenaStringPtr field3;
field3.UnsafeSetDefault(nullptr);
mut = field3.Mutable(nonempty_default, &arena);
EXPECT_EQ(mut, field3.Mutable(nonempty_default, &arena));
EXPECT_EQ(mut, &field3.Get());
EXPECT_NE(nullptr, mut);
EXPECT_EQ(std::string("default"), *mut);
*mut = "Test long long long long value"; // ensure string allocates storage
EXPECT_EQ(std::string("Test long long long long value"), field3.Get());
field3.Destroy(nullptr, &arena);
}
TEST(ArenaStringPtrTest, ArenaStringPtrOnArenaNoSSO) {
Arena arena;
ArenaStringPtr field;
std::string default_value = "default";
field.UnsafeSetDefault(&default_value);
EXPECT_EQ(std::string("default"), field.Get());
const std::string* empty_default = &internal::GetEmptyString();
field.UnsafeSetDefault(empty_default);
EXPECT_EQ(std::string(""), field.Get());
// Avoid triggering the SSO optimization by setting the string to something
// larger than the internal buffer.
field.Set(&default_value, WrapString("Test long long long long value"),
field.Set(empty_default, WrapString("Test long long long long value"),
&arena);
EXPECT_EQ(std::string("Test long long long long value"), field.Get());
field.Set(&default_value, std::string(""), &arena);
field.Destroy(&default_value, &arena);
field.Set(empty_default, std::string(""), &arena);
field.Destroy(empty_default, &arena);
ArenaStringPtr field2;
field2.UnsafeSetDefault(&default_value);
std::string* mut = field2.Mutable(&default_value, &arena);
EXPECT_EQ(mut, field2.Mutable(&default_value, &arena));
field2.UnsafeSetDefault(empty_default);
std::string* mut = field2.Mutable(EmptyDefault{}, &arena);
EXPECT_EQ(mut, field2.Mutable(EmptyDefault{}, &arena));
EXPECT_EQ(mut, &field2.Get());
EXPECT_NE(&default_value, mut);
EXPECT_EQ(std::string("default"), *mut);
EXPECT_NE(empty_default, mut);
EXPECT_EQ(std::string(""), *mut);
*mut = "Test long long long long value"; // ensure string allocates storage
EXPECT_EQ(std::string("Test long long long long value"), field2.Get());
field2.Destroy(&default_value, &arena);
field2.Destroy(empty_default, &arena);
ArenaStringPtr field3;
field3.UnsafeSetDefault(nullptr);
mut = field3.Mutable(nonempty_default, &arena);
EXPECT_EQ(mut, field3.Mutable(nonempty_default, &arena));
EXPECT_EQ(mut, &field3.Get());
EXPECT_NE(nullptr, mut);
EXPECT_EQ(std::string("default"), *mut);
*mut = "Test long long long long value"; // ensure string allocates storage
EXPECT_EQ(std::string("Test long long long long value"), field3.Get());
field3.Destroy(nullptr, &arena);
}

26
src/google/protobuf/compiler/command_line_interface.cc
View File

@ -1233,6 +1233,7 @@ bool CommandLineInterface::AllowProto3Optional(
return false;
}
bool CommandLineInterface::VerifyInputFilesInDescriptors(
DescriptorDatabase* database) {
for (const auto& input_file : input_files_) {
@ -1251,6 +1252,7 @@ bool CommandLineInterface::VerifyInputFilesInDescriptors(
<< std::endl;
return false;
}
}
return true;
}
@ -1301,6 +1303,7 @@ bool CommandLineInterface::ParseInputFiles(
break;
}
// Enforce --direct_dependencies
if (direct_dependencies_explicitly_set_) {
bool indirect_imports = false;
@ -1347,6 +1350,7 @@ void CommandLineInterface::Clear() {
disallow_services_ = false;
direct_dependencies_explicitly_set_ = false;
allow_proto3_optional_ = false;
deterministic_output_ = false;
}
bool CommandLineInterface::MakeProtoProtoPathRelative(
@ -1582,6 +1586,11 @@ CommandLineInterface::ParseArgumentStatus CommandLineInterface::ParseArguments(
<< std::endl;
return PARSE_ARGUMENT_FAIL;
}
if (mode_ != MODE_ENCODE && deterministic_output_) {
std::cerr << "Can only use --deterministic_output with --encode."
<< std::endl;
return PARSE_ARGUMENT_FAIL;
}
if (!dependency_out_name_.empty() && input_files_.size() > 1) {
std::cerr
<< "Can only process one input file when using --dependency_out=FILE."
@ -1650,7 +1659,8 @@ bool CommandLineInterface::ParseArgument(const char* arg, std::string* name,
*name == "--include_imports" || *name == "--include_source_info" ||
*name == "--version" || *name == "--decode_raw" ||
*name == "--print_free_field_numbers" ||
*name == "--experimental_allow_proto3_optional") {
*name == "--experimental_allow_proto3_optional" ||
*name == "--deterministic_output") {
// HACK: These are the only flags that don't take a value.
// They probably should not be hard-coded like this but for now it's
// not worth doing better.
@ -1857,6 +1867,7 @@ CommandLineInterface::InterpretArgument(const std::string& name,
} else if (name == "--disallow_services") {
disallow_services_ = true;
} else if (name == "--experimental_allow_proto3_optional") {
allow_proto3_optional_ = true;
@ -1890,6 +1901,9 @@ CommandLineInterface::InterpretArgument(const std::string& name,
codec_type_ = value;
} else if (name == "--deterministic_output") {
deterministic_output_ = true;
} else if (name == "--error_format") {
if (value == "gcc") {
error_format_ = ERROR_FORMAT_GCC;
@ -2032,6 +2046,12 @@ void CommandLineInterface::PrintHelpText() {
"must\n"
" be defined in PROTO_FILES or their "
"imports.\n"
" --deterministic_output When using --encode, ensure map fields "
"are\n"
" deterministically ordered. Note that"
"this order is not\n"
" canonical, and changes across builds"
"or releases of protoc.\n"
" --decode=MESSAGE_TYPE Read a binary message of the given "
"type from\n"
" standard input and write it in text "
@ -2444,7 +2464,9 @@ bool CommandLineInterface::EncodeOrDecode(const DescriptorPool* pool) {
if (mode_ == MODE_ENCODE) {
// Output is binary.
if (!message->SerializePartialToZeroCopyStream(&out)) {
io::CodedOutputStream coded_out(&out);
coded_out.SetSerializationDeterministic(deterministic_output_);
if (!message->SerializePartialToCodedStream(&coded_out)) {
std::cerr << "output: I/O error." << std::endl;
return false;
}

3
src/google/protobuf/compiler/command_line_interface.h
View File

@ -451,6 +451,9 @@ class PROTOC_EXPORT CommandLineInterface {
// Was the --experimental_allow_proto3_optional flag used?
bool allow_proto3_optional_ = false;
// When using --encode, this will be passed to SetSerializationDeterministic.
bool deterministic_output_ = false;
GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(CommandLineInterface);
};

41
src/google/protobuf/compiler/command_line_interface_unittest.cc
View File

@ -40,12 +40,14 @@
#include <unistd.h>
#endif
#include <memory>
#include <string>
#include <vector>
#include <google/protobuf/stubs/stringprintf.h>
#include <google/protobuf/testing/file.h>
#include <google/protobuf/testing/file.h>
#include <google/protobuf/testing/file.h>
#include <google/protobuf/any.pb.h>
#include <google/protobuf/compiler/mock_code_generator.h>
#include <google/protobuf/compiler/subprocess.h>
#include <google/protobuf/compiler/code_generator.h>
@ -59,10 +61,10 @@
#include <google/protobuf/descriptor.h>
#include <google/protobuf/testing/googletest.h>
#include <gtest/gtest.h>
#include <google/protobuf/stubs/strutil.h>
#include <google/protobuf/stubs/substitute.h>
#include <google/protobuf/io/io_win32.h>
#include <google/protobuf/stubs/strutil.h>
namespace google {
namespace protobuf {
@ -122,7 +124,7 @@ class CommandLineInterfaceTest : public testing::Test {
void SwitchToTempDirectory() {
File::ChangeWorkingDirectory(temp_directory_);
}
#else // !PROTOBUF_OPENSOURCE
#else // !PROTOBUF_OPENSOURCE
// TODO(teboring): Figure out how to change and get working directory in
// google3.
#endif // !PROTOBUF_OPENSOURCE
@ -680,6 +682,9 @@ TEST_F(CommandLineInterfaceTest, MultipleInputs_UnusedImport_DescriptorSetIn) {
FileDescriptorProto::descriptor()->file();
descriptor_file->CopyTo(file_descriptor_set.add_file());
FileDescriptorProto& any_proto = *file_descriptor_set.add_file();
google::protobuf::Any::descriptor()->file()->CopyTo(&any_proto);
const FileDescriptor* custom_file =
protobuf_unittest::AggregateMessage::descriptor()->file();
FileDescriptorProto* file_descriptor_proto = file_descriptor_set.add_file();
@ -1387,6 +1392,7 @@ TEST_F(CommandLineInterfaceTest, AllowServicesHasService) {
ExpectGenerated("test_generator", "", "foo.proto", "Foo");
}
TEST_F(CommandLineInterfaceTest, DirectDependencies_Missing_EmptyList) {
CreateTempFile("foo.proto",
"syntax = \"proto2\";\n"
@ -2567,7 +2573,10 @@ class EncodeDecodeTest : public testing::TestWithParam<EncodeDecodeTestMode> {
bool Run(const std::string& command, bool specify_proto_files = true) {
std::vector<std::string> args;
args.push_back("protoc");
SplitStringUsing(command, " ", &args);
for (StringPiece split_piece :
Split(command, " ", true)) {
args.push_back(std::string(split_piece));
}
if (specify_proto_files) {
switch (GetParam()) {
case PROTO_PATH:
@ -2726,6 +2735,32 @@ TEST_P(EncodeDecodeTest, ProtoParseError) {
"net/proto2/internal/no_such_file.proto: No such file or directory\n");
}
TEST_P(EncodeDecodeTest, EncodeDeterministicOutput) {
RedirectStdinFromFile(TestUtil::GetTestDataPath(
"net/proto2/internal/"
"testdata/text_format_unittest_data_oneof_implemented.txt"));
std::string args;
if (GetParam() != DESCRIPTOR_SET_IN) {
args.append(
TestUtil::MaybeTranslatePath("net/proto2/internal/unittest.proto"));
}
EXPECT_TRUE(Run(
args + " --encode=protobuf_unittest.TestAllTypes --deterministic_output"));
ExpectStdoutMatchesBinaryFile(TestUtil::GetTestDataPath(
"net/proto2/internal/testdata/golden_message_oneof_implemented"));
ExpectStderrMatchesText("");
}
TEST_P(EncodeDecodeTest, DecodeDeterministicOutput) {
RedirectStdinFromFile(TestUtil::GetTestDataPath(
"net/proto2/internal/testdata/golden_message_oneof_implemented"));
EXPECT_FALSE(
Run(TestUtil::MaybeTranslatePath("net/proto2/internal/unittest.proto") +
" --decode=protobuf_unittest.TestAllTypes --deterministic_output"));
ExpectStderrMatchesText(
"Can only use --deterministic_output with --encode.\n");
}
INSTANTIATE_TEST_SUITE_P(FileDescriptorSetSource, EncodeDecodeTest,
testing::Values(PROTO_PATH, DESCRIPTOR_SET_IN));
} // anonymous namespace

9
src/google/protobuf/compiler/cpp/cpp_field.h
View File

@ -164,10 +164,6 @@ class FieldGenerator {
return false;
}
// Generate code that allocates the fields's default instance.
virtual void GenerateDefaultInstanceAllocator(
io::Printer* /*printer*/) const {}
// Generate lines to serialize this field directly to the array "target",
// which are placed within the message's SerializeWithCachedSizesToArray()
// method. This must also advance "target" past the written bytes.
@ -178,11 +174,6 @@ class FieldGenerator {
// are placed in the message's ByteSize() method.
virtual void GenerateByteSize(io::Printer* printer) const = 0;
// Any tags about field layout decisions (such as inlining) to embed in the
// offset.
virtual uint32 CalculateFieldTag() const { return 0; }
virtual bool IsInlined() const { return false; }
void SetHasBitIndex(int32 has_bit_index);
protected:

3
src/google/protobuf/compiler/cpp/cpp_file.cc
View File

@ -941,7 +941,6 @@ void FileGenerator::GenerateInitForSCC(const SCC* scc,
if (scc_analyzer_.GetSCC(message_generators_[i]->descriptor_) != scc) {
continue;
}
message_generators_[i]->GenerateFieldDefaultInstances(printer);
format(
"{\n"
" void* ptr = &$1$;\n"
@ -1198,7 +1197,6 @@ void FileGenerator::GenerateForwardDeclarations(io::Printer* printer) {
decls[Namespace(d, options_)].AddEnum(d);
}
{
NamespaceOpener ns(format);
for (const auto& pair : decls) {
@ -1281,7 +1279,6 @@ void FileGenerator::GenerateLibraryIncludes(io::Printer* printer) {
IncludeFile("net/proto2/public/arenastring.h", printer);
IncludeFile("net/proto2/public/generated_message_table_driven.h", printer);
IncludeFile("net/proto2/public/generated_message_util.h", printer);
IncludeFile("net/proto2/public/inlined_string_field.h", printer);
IncludeFile("net/proto2/public/metadata_lite.h", printer);
if (HasDescriptorMethods(file_, options_)) {

22
src/google/protobuf/compiler/cpp/cpp_helpers.cc
View File

@ -787,25 +787,6 @@ std::string SafeFunctionName(const Descriptor* descriptor,
return function_name;
}
bool IsStringInlined(const FieldDescriptor* descriptor,
const Options& options) {
if (options.opensource_runtime) return false;
// TODO(ckennelly): Handle inlining for any.proto.
if (IsAnyMessage(descriptor->containing_type(), options)) return false;
if (descriptor->containing_type()->options().map_entry()) return false;
// We rely on has bits to distinguish field presence for release_$name$. When
// there is no hasbit, we cannot use the address of the string instance when
// the field has been inlined.
if (!HasHasbit(descriptor)) return false;
if (options.access_info_map) {
if (descriptor->is_required()) return true;
}
return false;
}
static bool HasLazyFields(const Descriptor* descriptor,
const Options& options) {
for (int field_idx = 0; field_idx < descriptor->field_count(); field_idx++) {
@ -1470,8 +1451,7 @@ class ParseLoopGenerator {
GetOptimizeFor(field->file(), options_) != FileOptions::LITE_RUNTIME &&
// For now only use arena string for strings with empty defaults.
field->default_value_string().empty() &&
!IsStringInlined(field, options_) && !field->real_containing_oneof() &&
ctype == FieldOptions::STRING) {
!field->real_containing_oneof() && ctype == FieldOptions::STRING) {
GenerateArenaString(field);
} else {
std::string name;

10
src/google/protobuf/compiler/cpp/cpp_helpers.h
View File

@ -318,8 +318,6 @@ inline bool IsWeak(const FieldDescriptor* field, const Options& options) {
return false;
}
bool IsStringInlined(const FieldDescriptor* descriptor, const Options& options);
// For a string field, returns the effective ctype. If the actual ctype is
// not supported, returns the default of STRING.
FieldOptions::CType EffectiveStringCType(const FieldDescriptor* field,
@ -410,14 +408,6 @@ inline bool IsProto2MessageSet(const Descriptor* descriptor,
descriptor->full_name() == "google.protobuf.bridge.MessageSet";
}
inline bool IsProto2MessageSetFile(const FileDescriptor* file,
const Options& options) {
return !options.opensource_runtime &&
options.enforce_mode != EnforceOptimizeMode::kLiteRuntime &&
!options.lite_implicit_weak_fields &&
file->name() == "net/proto2/bridge/proto/message_set.proto";
}
inline bool IsMapEntryMessage(const Descriptor* descriptor) {
return descriptor->options().map_entry();
}

24
src/google/protobuf/compiler/cpp/cpp_message.cc
View File

@ -1699,9 +1699,6 @@ uint32 CalcFieldNum(const FieldGenerator& generator,
int type = field->type();
if (type == FieldDescriptor::TYPE_STRING ||
type == FieldDescriptor::TYPE_BYTES) {
if (generator.IsInlined()) {
type = internal::FieldMetadata::kInlinedType;
}
// string field
if (IsCord(field, options)) {
type = internal::FieldMetadata::kCordType;
@ -1902,13 +1899,6 @@ int MessageGenerator::GenerateFieldMetadata(io::Printer* printer) {
return num_field_metadata;
}
void MessageGenerator::GenerateFieldDefaultInstances(io::Printer* printer) {
// Construct the default instances for all fields that need one.
for (auto field : FieldRange(descriptor_)) {
field_generators_.get(field).GenerateDefaultInstanceAllocator(printer);
}
}
void MessageGenerator::GenerateClassMethods(io::Printer* printer) {
Formatter format(printer, variables_);
if (IsMapEntryMessage(descriptor_)) {
@ -2111,14 +2101,9 @@ size_t MessageGenerator::GenerateParseOffsets(io::Printer* printer) {
}
processing_type = static_cast<unsigned>(field->type());
const FieldGenerator& generator = field_generators_.get(field);
if (field->type() == FieldDescriptor::TYPE_STRING) {
switch (EffectiveStringCType(field, options_)) {
case FieldOptions::STRING:
if (generator.IsInlined()) {
processing_type = internal::TYPE_STRING_INLINED;
break;
}
break;
case FieldOptions::CORD:
processing_type = internal::TYPE_STRING_CORD;
@ -2130,10 +2115,6 @@ size_t MessageGenerator::GenerateParseOffsets(io::Printer* printer) {
} else if (field->type() == FieldDescriptor::TYPE_BYTES) {
switch (EffectiveStringCType(field, options_)) {
case FieldOptions::STRING:
if (generator.IsInlined()) {
processing_type = internal::TYPE_BYTES_INLINED;
break;
}
break;
case FieldOptions::CORD:
processing_type = internal::TYPE_BYTES_CORD;
@ -2317,11 +2298,6 @@ std::pair<size_t, size_t> MessageGenerator::GenerateOffsets(
format("PROTOBUF_FIELD_OFFSET($classtype$, $1$_)", FieldName(field));
}
uint32 tag = field_generators_.get(field).CalculateFieldTag();
if (tag != 0) {
format(" | $1$", tag);
}
if (!IsFieldUsed(field, options_)) {
format(" | 0x80000000u, // unused\n");
} else {

3
src/google/protobuf/compiler/cpp/cpp_message.h
View File

@ -82,9 +82,6 @@ class MessageGenerator {
// Source file stuff.
// Generates code that creates default instances for fields.
void GenerateFieldDefaultInstances(io::Printer* printer);
// Generate all non-inline methods for this class.
void GenerateClassMethods(io::Printer* printer);

207
src/google/protobuf/compiler/cpp/cpp_string_field.cc
View File

@ -55,12 +55,31 @@ void SetStringVariables(const FieldDescriptor* descriptor,
StrCat(descriptor->default_value_string().length());
std::string default_variable_string = MakeDefaultName(descriptor);
(*variables)["default_variable_name"] = default_variable_string;
(*variables)["default_variable"] =
if (!descriptor->default_value_string().empty()) {
(*variables)["lazy_variable"] =
QualifiedClassName(descriptor->containing_type(), options) +
"::" + default_variable_string;
}
(*variables)["default_string"] =
descriptor->default_value_string().empty()
? "::" + (*variables)["proto_ns"] +
"::internal::GetEmptyStringAlreadyInited()"
: (*variables)["lazy_variable"] + ".get()";
(*variables)["init_value"] =
descriptor->default_value_string().empty()
? "&::" + (*variables)["proto_ns"] +
"::internal::GetEmptyStringAlreadyInited()"
: "&" + QualifiedClassName(descriptor->containing_type(), options) +
"::" + default_variable_string + ".get()";
: "nullptr";
(*variables)["default_value_tag"] =
"::" + (*variables)["proto_ns"] + "::internal::ArenaStringPtr::" +
(descriptor->default_value_string().empty() ? "Empty" : "NonEmpty") +
"Default{}";
(*variables)["default_variable_or_tag"] =
(*variables)[descriptor->default_value_string().empty()
? "default_value_tag"
: "lazy_variable"];
(*variables)["pointer_type"] =
descriptor->type() == FieldDescriptor::TYPE_BYTES ? "void" : "char";
(*variables)["null_check"] = (*variables)["DCHK"] + "(value != nullptr);\n";
@ -75,9 +94,6 @@ void SetStringVariables(const FieldDescriptor* descriptor,
} else {
(*variables)["string_piece"] = "::StringPiece";
}
(*variables)["lite"] =
HasDescriptorMethods(descriptor->file(), options) ? "" : "Lite";
}
} // namespace
@ -86,9 +102,7 @@ void SetStringVariables(const FieldDescriptor* descriptor,
StringFieldGenerator::StringFieldGenerator(const FieldDescriptor* descriptor,
const Options& options)
: FieldGenerator(descriptor, options),
lite_(!HasDescriptorMethods(descriptor->file(), options)),
inlined_(IsStringInlined(descriptor, options)) {
: FieldGenerator(descriptor, options) {
SetStringVariables(descriptor, &variables_, options);
}
@ -96,23 +110,15 @@ StringFieldGenerator::~StringFieldGenerator() {}
void StringFieldGenerator::GeneratePrivateMembers(io::Printer* printer) const {
Formatter format(printer, variables_);
if (inlined_) {
format("::$proto_ns$::internal::InlinedStringField $name$_;\n");
} else {
format("::$proto_ns$::internal::ArenaStringPtr $name$_;\n");
}
format("::$proto_ns$::internal::ArenaStringPtr $name$_;\n");
}
void StringFieldGenerator::GenerateStaticMembers(io::Printer* printer) const {
Formatter format(printer, variables_);
if (!descriptor_->default_value_string().empty()) {
// We make the default instance public, so it can be initialized by
// non-friend code.
format(
"public:\n"
"static ::$proto_ns$::internal::ExplicitlyConstructed<std::string>"
" $default_variable_name$;\n"
"private:\n");
"static const ::$proto_ns$::internal::LazyString"
" $default_variable_name$;\n");
}
}
@ -187,7 +193,13 @@ void StringFieldGenerator::GenerateInlineAccessorDefinitions(
format(
"inline const std::string& $classname$::$name$() const {\n"
"$annotate_accessor$"
" // @@protoc_insertion_point(field_get:$full_name$)\n"
" // @@protoc_insertion_point(field_get:$full_name$)\n");
if (!descriptor_->default_value_string().empty()) {
format(
" if ($name$_.IsDefault(nullptr)) return "
"$default_variable_name$.get();\n");
}
format(
" return _internal_$name$();\n"
"}\n"
"inline void $classname$::set_$name$(const std::string& value) {\n"
@ -206,21 +218,20 @@ void StringFieldGenerator::GenerateInlineAccessorDefinitions(
"inline void $classname$::_internal_set_$name$(const std::string& "
"value) {\n"
" $set_hasbit$\n"
" $name$_.Set$lite$($default_variable$, value, GetArena());\n"
" $name$_.Set($default_value_tag$, value, GetArena());\n"
"}\n"
"inline void $classname$::set_$name$(std::string&& value) {\n"
"$annotate_accessor$"
" $set_hasbit$\n"
" $name$_.Set$lite$(\n"
" $default_variable$, ::std::move(value), GetArena());\n"
" $name$_.Set(\n"
" $default_value_tag$, ::std::move(value), GetArena());\n"
" // @@protoc_insertion_point(field_set_rvalue:$full_name$)\n"
"}\n"
"inline void $classname$::set_$name$(const char* value) {\n"
"$annotate_accessor$"
" $null_check$"
" $set_hasbit$\n"
" $name$_.Set$lite$($default_variable$, $string_piece$(value),\n"
" GetArena());\n"
" $name$_.Set($default_value_tag$, $string_piece$(value), GetArena());\n"
" // @@protoc_insertion_point(field_set_char:$full_name$)\n"
"}\n");
if (!options_.opensource_runtime) {
@ -228,7 +239,7 @@ void StringFieldGenerator::GenerateInlineAccessorDefinitions(
"inline void $classname$::set_$name$(::StringPiece value) {\n"
"$annotate_accessor$"
" $set_hasbit$\n"
" $name$_.Set$lite$($default_variable$, value,GetArena());\n"
" $name$_.Set($default_value_tag$, value,GetArena());\n"
" // @@protoc_insertion_point(field_set_string_piece:$full_name$)\n"
"}\n");
}
@ -238,13 +249,13 @@ void StringFieldGenerator::GenerateInlineAccessorDefinitions(
" size_t size) {\n"
"$annotate_accessor$"
" $set_hasbit$\n"
" $name$_.Set$lite$($default_variable$, $string_piece$(\n"
" $name$_.Set($default_value_tag$, $string_piece$(\n"
" reinterpret_cast<const char*>(value), size), GetArena());\n"
" // @@protoc_insertion_point(field_set_pointer:$full_name$)\n"
"}\n"
"inline std::string* $classname$::_internal_mutable_$name$() {\n"
" $set_hasbit$\n"
" return $name$_.Mutable($default_variable$, GetArena());\n"
" return $name$_.Mutable($default_variable_or_tag$, GetArena());\n"
"}\n"
"inline std::string* $classname$::$release_name$() {\n"
"$annotate_accessor$"
@ -256,10 +267,9 @@ void StringFieldGenerator::GenerateInlineAccessorDefinitions(
" return nullptr;\n"
" }\n"
" $clear_hasbit$\n"
" return $name$_.ReleaseNonDefault("
"$default_variable$, GetArena());\n");
" return $name$_.ReleaseNonDefault($init_value$, GetArena());\n");
} else {
format(" return $name$_.Release($default_variable$, GetArena());\n");
format(" return $name$_.Release($init_value$, GetArena());\n");
}
format(
@ -271,7 +281,7 @@ void StringFieldGenerator::GenerateInlineAccessorDefinitions(
" } else {\n"
" $clear_hasbit$\n"
" }\n"
" $name$_.SetAllocated($default_variable$, $name$,\n"
" $name$_.SetAllocated($init_value$, $name$,\n"
" GetArena());\n"
" // @@protoc_insertion_point(field_set_allocated:$full_name$)\n"
"}\n");
@ -281,23 +291,19 @@ void StringFieldGenerator::GenerateNonInlineAccessorDefinitions(
io::Printer* printer) const {
Formatter format(printer, variables_);
if (!descriptor_->default_value_string().empty()) {
// Initialized in GenerateDefaultInstanceAllocator.
format(
"::$proto_ns$::internal::ExplicitlyConstructed<std::string> "
"$classname$::$default_variable_name$;\n");
"const ::$proto_ns$::internal::LazyString "
"$classname$::$default_variable_name$"
"{{{$default$, $default_length$}}, {nullptr}};\n");
}
}
void StringFieldGenerator::GenerateClearingCode(io::Printer* printer) const {
Formatter format(printer, variables_);
// Two-dimension specialization here: supporting arenas or not, and default
// value is the empty string or not. Complexity here ensures the minimal
// number of branches / amount of extraneous code at runtime (given that the
// below methods are inlined one-liners)!
if (descriptor_->default_value_string().empty()) {
format("$name$_.ClearToEmpty($default_variable$, GetArena());\n");
format("$name$_.ClearToEmpty();\n");
} else {
format("$name$_.ClearToDefault($default_variable$, GetArena());\n");
format("$name$_.ClearToDefault($lazy_variable$, GetArena());\n");
}
}
@ -311,31 +317,19 @@ void StringFieldGenerator::GenerateMessageClearingCode(
// If we have a hasbit, then the Clear() method of the protocol buffer
// will have checked that this field is set. If so, we can avoid redundant
// checks against default_variable.
// checks against the default variable.
const bool must_be_present = HasHasbit(descriptor_);
if (inlined_ && must_be_present) {
// Calling mutable_$name$() gives us a string reference and sets the has bit
// for $name$ (in proto2). We may get here when the string field is inlined
// but the string's contents have not been changed by the user, so we cannot
// make an assertion about the contents of the string and could never make
// an assertion about the string instance.
//
// For non-inlined strings, we distinguish from non-default by comparing
// instances, rather than contents.
format("$DCHK$(!$name$_.IsDefault($default_variable$));\n");
}
if (descriptor_->default_value_string().empty()) {
if (must_be_present) {
format("$name$_.ClearNonDefaultToEmpty();\n");
} else {
format("$name$_.ClearToEmpty($default_variable$, GetArena());\n");
format("$name$_.ClearToEmpty();\n");
}
} else {
// Clear to a non-empty default is more involved, as we try to use the
// Arena if one is present and may need to reallocate the string.
format("$name$_.ClearToDefault($default_variable$, GetArena());\n");
format("$name$_.ClearToDefault($lazy_variable$, GetArena());\n ");
}
}
@ -347,23 +341,12 @@ void StringFieldGenerator::GenerateMergingCode(io::Printer* printer) const {
void StringFieldGenerator::GenerateSwappingCode(io::Printer* printer) const {
Formatter format(printer, variables_);
if (inlined_) {
format("$name$_.Swap(&other->$name$_);\n");
} else {
format("$name$_.Swap(&other->$name$_, $default_variable$, GetArena());\n");
}
format("$name$_.Swap(&other->$name$_, $init_value$, GetArena());\n");
}
void StringFieldGenerator::GenerateConstructorCode(io::Printer* printer) const {
Formatter format(printer, variables_);
// TODO(ckennelly): Construct non-empty strings as part of the initializer
// list.
if (inlined_ && descriptor_->default_value_string().empty()) {
// Automatic initialization will construct the string.
return;
}
format("$name$_.UnsafeSetDefault($default_variable$);\n");
format("$name$_.UnsafeSetDefault($init_value$);\n");
}
void StringFieldGenerator::GenerateCopyConstructorCode(
@ -381,7 +364,7 @@ void StringFieldGenerator::GenerateCopyConstructorCode(
// TODO(gpike): improve this
format(
"$name$_.Set$lite$($default_variable$, from._internal_$name$(),\n"
"$name$_.Set($default_value_tag$, from._internal_$name$(), \n"
" GetArena());\n");
format.Outdent();
@ -390,40 +373,7 @@ void StringFieldGenerator::GenerateCopyConstructorCode(
void StringFieldGenerator::GenerateDestructorCode(io::Printer* printer) const {
Formatter format(printer, variables_);
if (inlined_) {
// The destructor is automatically invoked.
return;
}
format("$name$_.DestroyNoArena($default_variable$);\n");
}
bool StringFieldGenerator::GenerateArenaDestructorCode(
io::Printer* printer) const {
Formatter format(printer, variables_);
if (!inlined_) {
return false;
}
format("_this->$name$_.DestroyNoArena($default_variable$);\n");
return true;
}
void StringFieldGenerator::GenerateDefaultInstanceAllocator(
io::Printer* printer) const {
Formatter format(printer, variables_);
if (!descriptor_->default_value_string().empty()) {
format(
"$ns$::$classname$::$default_variable_name$.DefaultConstruct();\n"
"*$ns$::$classname$::$default_variable_name$.get_mutable() = "
"std::string($default$, $default_length$);\n"
"::$proto_ns$::internal::OnShutdownDestroyString(\n"
" $ns$::$classname$::$default_variable_name$.get_mutable());\n");
}
}
bool StringFieldGenerator::MergeFromCodedStreamNeedsArena() const {
return !lite_ && !inlined_ && !options_.opensource_runtime;
format("$name$_.DestroyNoArena($init_value$);\n");
}
void StringFieldGenerator::GenerateSerializeWithCachedSizesToArray(
@ -449,17 +399,11 @@ void StringFieldGenerator::GenerateByteSize(io::Printer* printer) const {
" this->_internal_$name$());\n");
}
uint32 StringFieldGenerator::CalculateFieldTag() const {
return inlined_ ? 1 : 0;
}
// ===================================================================
StringOneofFieldGenerator::StringOneofFieldGenerator(
const FieldDescriptor* descriptor, const Options& options)
: StringFieldGenerator(descriptor, options) {
inlined_ = false;
SetCommonOneofFieldVariables(descriptor, &variables_);
variables_["field_name"] = UnderscoresToCamelCase(descriptor->name(), true);
variables_["oneof_index"] =
@ -491,16 +435,16 @@ void StringOneofFieldGenerator::GenerateInlineAccessorDefinitions(
" if (_internal_has_$name$()) {\n"
" return $field_member$.Get();\n"
" }\n"
" return *$default_variable$;\n"
" return $default_string$;\n"
"}\n"
"inline void $classname$::_internal_set_$name$(const std::string& "
"value) {\n"
" if (!_internal_has_$name$()) {\n"
" clear_$oneof_name$();\n"
" set_has_$name$();\n"
" $field_member$.UnsafeSetDefault($default_variable$);\n"
" $field_member$.UnsafeSetDefault($init_value$);\n"
" }\n"
" $field_member$.Set$lite$($default_variable$, value, GetArena());\n"
" $field_member$.Set($default_value_tag$, value, GetArena());\n"
"}\n"
"inline void $classname$::set_$name$(std::string&& value) {\n"
"$annotate_accessor$"
@ -508,10 +452,10 @@ void StringOneofFieldGenerator::GenerateInlineAccessorDefinitions(
" if (!_internal_has_$name$()) {\n"
" clear_$oneof_name$();\n"
" set_has_$name$();\n"
" $field_member$.UnsafeSetDefault($default_variable$);\n"
" $field_member$.UnsafeSetDefault($init_value$);\n"
" }\n"
" $field_member$.Set$lite$(\n"
" $default_variable$, ::std::move(value), GetArena());\n"
" $field_member$.Set(\n"
" $default_value_tag$, ::std::move(value), GetArena());\n"
" // @@protoc_insertion_point(field_set_rvalue:$full_name$)\n"
"}\n"
"inline void $classname$::set_$name$(const char* value) {\n"
@ -520,9 +464,9 @@ void StringOneofFieldGenerator::GenerateInlineAccessorDefinitions(
" if (!_internal_has_$name$()) {\n"
" clear_$oneof_name$();\n"
" set_has_$name$();\n"
" $field_member$.UnsafeSetDefault($default_variable$);\n"
" $field_member$.UnsafeSetDefault($init_value$);\n"
" }\n"
" $field_member$.Set$lite$($default_variable$,\n"
" $field_member$.Set($default_value_tag$,\n"
" $string_piece$(value), GetArena());\n"
" // @@protoc_insertion_point(field_set_char:$full_name$)\n"
"}\n");
@ -533,10 +477,9 @@ void StringOneofFieldGenerator::GenerateInlineAccessorDefinitions(
" if (!_internal_has_$name$()) {\n"
" clear_$oneof_name$();\n"
" set_has_$name$();\n"
" $field_member$.UnsafeSetDefault($default_variable$);\n"
" $field_member$.UnsafeSetDefault($init_value$);\n"
" }\n"
" $field_member$.Set$lite$($default_variable$, value,\n"
" GetArena());\n"
" $field_member$.Set($default_value_tag$, value, GetArena());\n"
" // @@protoc_insertion_point(field_set_string_piece:$full_name$)\n"
"}\n");
}
@ -548,10 +491,10 @@ void StringOneofFieldGenerator::GenerateInlineAccessorDefinitions(
" if (!_internal_has_$name$()) {\n"
" clear_$oneof_name$();\n"
" set_has_$name$();\n"
" $field_member$.UnsafeSetDefault($default_variable$);\n"
" $field_member$.UnsafeSetDefault($init_value$);\n"
" }\n"
" $field_member$.Set$lite$(\n"
" $default_variable$, $string_piece$(\n"
" $field_member$.Set(\n"
" $default_value_tag$, $string_piece$(\n"
" reinterpret_cast<const char*>(value), size),\n"
" GetArena());\n"
" // @@protoc_insertion_point(field_set_pointer:$full_name$)\n"
@ -560,16 +503,17 @@ void StringOneofFieldGenerator::GenerateInlineAccessorDefinitions(
" if (!_internal_has_$name$()) {\n"
" clear_$oneof_name$();\n"
" set_has_$name$();\n"
" $field_member$.UnsafeSetDefault($default_variable$);\n"
" $field_member$.UnsafeSetDefault($init_value$);\n"
" }\n"
" return $field_member$.Mutable($default_variable$, GetArena());\n"
" return $field_member$.Mutable(\n"
" $default_variable_or_tag$, GetArena());\n"
"}\n"
"inline std::string* $classname$::$release_name$() {\n"
"$annotate_accessor$"
" // @@protoc_insertion_point(field_release:$full_name$)\n"
" if (_internal_has_$name$()) {\n"
" clear_has_$oneof_name$();\n"
" return $field_member$.Release($default_variable$, GetArena());\n"
" return $field_member$.ReleaseNonDefault($init_value$, GetArena());\n"
" } else {\n"
" return nullptr;\n"
" }\n"
@ -594,7 +538,7 @@ void StringOneofFieldGenerator::GenerateInlineAccessorDefinitions(
void StringOneofFieldGenerator::GenerateClearingCode(
io::Printer* printer) const {
Formatter format(printer, variables_);
format("$field_member$.Destroy($default_variable$, GetArena());\n");
format("$field_member$.Destroy($default_value_tag$, GetArena());\n");
}
void StringOneofFieldGenerator::GenerateMessageClearingCode(
@ -609,10 +553,7 @@ void StringOneofFieldGenerator::GenerateSwappingCode(
void StringOneofFieldGenerator::GenerateConstructorCode(
io::Printer* printer) const {
Formatter format(printer, variables_);
format(
"$ns$::_$classname$_default_instance_.$name$_.UnsafeSetDefault(\n"
" $default_variable$);\n");
// Nothing required here.
}
// ===================================================================

10
src/google/protobuf/compiler/cpp/cpp_string_field.h
View File

@ -63,18 +63,8 @@ class StringFieldGenerator : public FieldGenerator {
void GenerateConstructorCode(io::Printer* printer) const;
void GenerateCopyConstructorCode(io::Printer* printer) const;
void GenerateDestructorCode(io::Printer* printer) const;
bool GenerateArenaDestructorCode(io::Printer* printer) const;
void GenerateDefaultInstanceAllocator(io::Printer* printer) const;
void GenerateSerializeWithCachedSizesToArray(io::Printer* printer) const;
void GenerateByteSize(io::Printer* printer) const;
uint32 CalculateFieldTag() const;
bool IsInlined() const { return inlined_; }
bool MergeFromCodedStreamNeedsArena() const;
protected:
const bool lite_;
bool inlined_;
private:
GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(StringFieldGenerator);

5
src/google/protobuf/compiler/java/java_enum.cc
View File

@ -77,9 +77,10 @@ void EnumGenerator::Generate(io::Printer* printer) {
WriteEnumDocComment(printer, descriptor_);
MaybePrintGeneratedAnnotation(context_, printer, descriptor_, immutable_api_);
printer->Print(
"public enum $classname$\n"
"$deprecation$public enum $classname$\n"
" implements com.google.protobuf.ProtocolMessageEnum {\n",
"classname", descriptor_->name());
"classname", descriptor_->name(), "deprecation",
descriptor_->options().deprecated() ? "@java.lang.Deprecated " : "");
printer->Annotate("classname", descriptor_);
printer->Indent();

4
src/google/protobuf/compiler/java/java_enum_field.cc
View File

@ -81,7 +81,7 @@ void SetEnumVariables(const FieldDescriptor* descriptor, int messageBitIndex,
// with v2.5.0/v2.6.1, and remove the @SuppressWarnings annotations.
(*variables)["for_number"] = "valueOf";
if (SupportFieldPresence(descriptor)) {
if (HasHasbit(descriptor)) {
// For singular messages and builders, one bit is used for the hasField bit.
(*variables)["get_has_field_bit_message"] = GenerateGetBit(messageBitIndex);
(*variables)["get_has_field_bit_builder"] = GenerateGetBit(builderBitIndex);
@ -145,7 +145,7 @@ ImmutableEnumFieldGenerator::ImmutableEnumFieldGenerator(
ImmutableEnumFieldGenerator::~ImmutableEnumFieldGenerator() {}
int ImmutableEnumFieldGenerator::GetNumBitsForMessage() const {
return SupportFieldPresence(descriptor_) ? 1 : 0;
return HasHasbit(descriptor_) ? 1 : 0;
}
int ImmutableEnumFieldGenerator::GetNumBitsForBuilder() const {

4
src/google/protobuf/compiler/java/java_enum_field_lite.cc
View File

@ -83,7 +83,7 @@ void SetEnumVariables(const FieldDescriptor* descriptor, int messageBitIndex,
descriptor->options().deprecated() ? "@java.lang.Deprecated " : "";
(*variables)["required"] = descriptor->is_required() ? "true" : "false";
if (SupportFieldPresence(descriptor)) {
if (HasHasbit(descriptor)) {
// For singular messages and builders, one bit is used for the hasField bit.
(*variables)["get_has_field_bit_message"] = GenerateGetBit(messageBitIndex);
@ -137,7 +137,7 @@ ImmutableEnumFieldLiteGenerator::ImmutableEnumFieldLiteGenerator(
ImmutableEnumFieldLiteGenerator::~ImmutableEnumFieldLiteGenerator() {}
int ImmutableEnumFieldLiteGenerator::GetNumBitsForMessage() const {
return SupportFieldPresence(descriptor_) ? 1 : 0;
return HasHasbit(descriptor_) ? 1 : 0;
}
void ImmutableEnumFieldLiteGenerator::GenerateInterfaceMembers(

5
src/google/protobuf/compiler/java/java_enum_lite.cc
View File

@ -78,9 +78,10 @@ void EnumLiteGenerator::Generate(io::Printer* printer) {
WriteEnumDocComment(printer, descriptor_);
MaybePrintGeneratedAnnotation(context_, printer, descriptor_, immutable_api_);
printer->Print(
"public enum $classname$\n"
"$deprecation$public enum $classname$\n"
" implements com.google.protobuf.Internal.EnumLite {\n",
"classname", descriptor_->name());
"classname", descriptor_->name(), "deprecation",
descriptor_->options().deprecated() ? "@java.lang.Deprecated " : "");
printer->Annotate("classname", descriptor_);
printer->Indent();

6
src/google/protobuf/compiler/java/java_file.cc
View File

@ -76,7 +76,6 @@ struct FieldDescriptorCompare {
typedef std::set<const FieldDescriptor*, FieldDescriptorCompare>
FieldDescriptorSet;
// Recursively searches the given message to collect extensions.
// Returns true if all the extensions can be recognized. The extensions will be
// appended in to the extensions parameter.
@ -86,9 +85,7 @@ bool CollectExtensions(const Message& message, FieldDescriptorSet* extensions) {
const Reflection* reflection = message.GetReflection();
// There are unknown fields that could be extensions, thus this call fails.
UnknownFieldSet unknown_fields;
unknown_fields.MergeFrom(reflection->GetUnknownFields(message));
if (unknown_fields.field_count() > 0) return false;
if (reflection->GetUnknownFields(message).field_count() > 0) return false;
std::vector<const FieldDescriptor*> fields;
reflection->ListFields(message, &fields);
@ -390,6 +387,7 @@ void FileGenerator::Generate(io::Printer* printer) {
printer->Print("}\n");
}
void FileGenerator::GenerateDescriptorInitializationCodeForImmutable(
io::Printer* printer) {
printer->Print(

1
src/google/protobuf/compiler/java/java_file.h
View File

@ -78,6 +78,7 @@ class FileGenerator {
void Generate(io::Printer* printer);
// If we aren't putting everything into one file, this will write all the
// files other than the outer file (i.e. one for each message, enum, and
// service type).

1
src/google/protobuf/compiler/java/java_message.cc
View File

@ -1355,7 +1355,6 @@ void ImmutableMessageGenerator::GenerateInitializers(io::Printer* printer) {
}
}
void ImmutableMessageGenerator::GenerateAnyMethods(io::Printer* printer) {
printer->Print(
"private static String getTypeUrl(\n"

70
src/google/protobuf/compiler/java/java_name_resolver.cc
View File

@ -160,7 +160,14 @@ std::string ClassNameResolver::GetFileImmutableClassName(
std::string ClassNameResolver::GetFileClassName(const FileDescriptor* file,
bool immutable) {
if (immutable) {
return GetFileClassName(file, immutable, false);
}
std::string ClassNameResolver::GetFileClassName(const FileDescriptor* file,
bool immutable, bool kotlin) {
if (kotlin) {
return GetFileImmutableClassName(file) + "Kt";
} else if (immutable) {
return GetFileImmutableClassName(file);
} else {
return "Mutable" + GetFileImmutableClassName(file);
@ -200,9 +207,14 @@ std::string ClassNameResolver::GetDescriptorClassName(
std::string ClassNameResolver::GetClassName(const FileDescriptor* descriptor,
bool immutable) {
return GetClassName(descriptor, immutable, false);
}
std::string ClassNameResolver::GetClassName(const FileDescriptor* descriptor,
bool immutable, bool kotlin) {
std::string result = FileJavaPackage(descriptor, immutable);
if (!result.empty()) result += '.';
result += GetFileClassName(descriptor, immutable);
result += GetFileClassName(descriptor, immutable, kotlin);
return result;
}
@ -211,50 +223,79 @@ std::string ClassNameResolver::GetClassName(const FileDescriptor* descriptor,
std::string ClassNameResolver::GetClassFullName(
const std::string& name_without_package, const FileDescriptor* file,
bool immutable, bool is_own_file) {
return GetClassFullName(name_without_package, file, immutable, is_own_file,
false);
}
std::string ClassNameResolver::GetClassFullName(
const std::string& name_without_package, const FileDescriptor* file,
bool immutable, bool is_own_file, bool kotlin) {
std::string result;
if (is_own_file) {
result = FileJavaPackage(file, immutable);
} else {
result = GetClassName(file, immutable);
result = GetClassName(file, immutable, kotlin);
}
if (!result.empty()) {
result += '.';
}
result += name_without_package;
if (kotlin) result += "Kt";
return result;
}
std::string ClassNameResolver::GetClassName(const Descriptor* descriptor,
bool immutable) {
return GetClassFullName(ClassNameWithoutPackage(descriptor, immutable),
descriptor->file(), immutable,
MultipleJavaFiles(descriptor->file(), immutable));
return GetClassName(descriptor, immutable, false);
}
std::string ClassNameResolver::GetClassName(const Descriptor* descriptor,
bool immutable, bool kotlin) {
return GetClassFullName(
ClassNameWithoutPackage(descriptor, immutable), descriptor->file(),
immutable, MultipleJavaFiles(descriptor->file(), immutable), kotlin);
}
std::string ClassNameResolver::GetClassName(const EnumDescriptor* descriptor,
bool immutable) {
return GetClassFullName(ClassNameWithoutPackage(descriptor, immutable),
descriptor->file(), immutable,
MultipleJavaFiles(descriptor->file(), immutable));
return GetClassName(descriptor, immutable, false);
}
std::string ClassNameResolver::GetClassName(const EnumDescriptor* descriptor,
bool immutable, bool kotlin) {
return GetClassFullName(
ClassNameWithoutPackage(descriptor, immutable), descriptor->file(),
immutable, MultipleJavaFiles(descriptor->file(), immutable), kotlin);
}
std::string ClassNameResolver::GetClassName(const ServiceDescriptor* descriptor,
bool immutable) {
return GetClassName(descriptor, immutable, false);
}
std::string ClassNameResolver::GetClassName(const ServiceDescriptor* descriptor,
bool immutable, bool kotlin) {
return GetClassFullName(ClassNameWithoutPackage(descriptor, immutable),
descriptor->file(), immutable,
IsOwnFile(descriptor, immutable));
IsOwnFile(descriptor, immutable), kotlin);
}
// Get the Java Class style full name of a message.
std::string ClassNameResolver::GetJavaClassFullName(
const std::string& name_without_package, const FileDescriptor* file,
bool immutable) {
return GetJavaClassFullName(name_without_package, file, immutable, false);
}
std::string ClassNameResolver::GetJavaClassFullName(
const std::string& name_without_package, const FileDescriptor* file,
bool immutable, bool kotlin) {
std::string result;
if (MultipleJavaFiles(file, immutable)) {
result = FileJavaPackage(file, immutable);
if (!result.empty()) result += '.';
} else {
result = GetClassName(file, immutable);
result = GetClassName(file, immutable, kotlin);
if (!result.empty()) result += '$';
}
result += StringReplace(name_without_package, ".", "$", true);
@ -263,7 +304,12 @@ std::string ClassNameResolver::GetJavaClassFullName(
std::string ClassNameResolver::GetExtensionIdentifierName(
const FieldDescriptor* descriptor, bool immutable) {
return GetClassName(descriptor->containing_type(), immutable) + "." +
return GetExtensionIdentifierName(descriptor, immutable, false);
}
std::string ClassNameResolver::GetExtensionIdentifierName(
const FieldDescriptor* descriptor, bool immutable, bool kotlin) {
return GetClassName(descriptor->containing_type(), immutable, kotlin) + "." +
descriptor->name();
}

18
src/google/protobuf/compiler/java/java_name_resolver.h
View File

@ -60,6 +60,8 @@ class ClassNameResolver {
// Gets the unqualified outer class name for the file.
std::string GetFileClassName(const FileDescriptor* file, bool immutable);
std::string GetFileClassName(const FileDescriptor* file, bool immutable,
bool kotlin);
// Gets the unqualified immutable outer class name of a file.
std::string GetFileImmutableClassName(const FileDescriptor* file);
// Gets the unqualified default immutable outer class name of a file
@ -80,9 +82,17 @@ class ClassNameResolver {
// Gets the fully-qualified class name corresponding to the given descriptor.
std::string GetClassName(const Descriptor* descriptor, bool immutable);
std::string GetClassName(const Descriptor* descriptor, bool immutable,
bool kotlin);
std::string GetClassName(const EnumDescriptor* descriptor, bool immutable);
std::string GetClassName(const EnumDescriptor* descriptor, bool immutable,
bool kotlin);
std::string GetClassName(const ServiceDescriptor* descriptor, bool immutable);
std::string GetClassName(const ServiceDescriptor* descriptor, bool immutable,
bool kotlin);
std::string GetClassName(const FileDescriptor* descriptor, bool immutable);
std::string GetClassName(const FileDescriptor* descriptor, bool immutable,
bool kotlin);
template <class DescriptorType>
std::string GetImmutableClassName(const DescriptorType* descriptor) {
@ -96,6 +106,8 @@ class ClassNameResolver {
// Gets the fully qualified name of an extension identifier.
std::string GetExtensionIdentifierName(const FieldDescriptor* descriptor,
bool immutable);
std::string GetExtensionIdentifierName(const FieldDescriptor* descriptor,
bool immutable, bool kotlin);
// Gets the fully qualified name for generated classes in Java convention.
// Nested classes will be separated using '$' instead of '.'
@ -109,9 +121,15 @@ class ClassNameResolver {
std::string GetClassFullName(const std::string& name_without_package,
const FileDescriptor* file, bool immutable,
bool is_own_file);
std::string GetClassFullName(const std::string& name_without_package,
const FileDescriptor* file, bool immutable,
bool is_own_file, bool kotlin);
// Get the Java Class style full name of a message.
std::string GetJavaClassFullName(const std::string& name_without_package,
const FileDescriptor* file, bool immutable);
std::string GetJavaClassFullName(const std::string& name_without_package,
const FileDescriptor* file, bool immutable,
bool kotlin);
// Caches the result to provide better performance.
std::map<const FileDescriptor*, std::string>
file_immutable_outer_class_names_;

4
src/google/protobuf/compiler/java/java_primitive_field.cc
View File

@ -133,7 +133,7 @@ void SetPrimitiveVariables(const FieldDescriptor* descriptor,
}
(*variables)["on_changed"] = "onChanged();";
if (SupportFieldPresence(descriptor)) {
if (HasHasbit(descriptor)) {
// For singular messages and builders, one bit is used for the hasField bit.
(*variables)["get_has_field_bit_message"] = GenerateGetBit(messageBitIndex);
(*variables)["get_has_field_bit_builder"] = GenerateGetBit(builderBitIndex);
@ -195,7 +195,7 @@ ImmutablePrimitiveFieldGenerator::ImmutablePrimitiveFieldGenerator(
ImmutablePrimitiveFieldGenerator::~ImmutablePrimitiveFieldGenerator() {}
int ImmutablePrimitiveFieldGenerator::GetNumBitsForMessage() const {
return SupportFieldPresence(descriptor_) ? 1 : 0;
return HasHasbit(descriptor_) ? 1 : 0;
}
int ImmutablePrimitiveFieldGenerator::GetNumBitsForBuilder() const {

4
src/google/protobuf/compiler/java/java_primitive_field_lite.cc
View File

@ -140,7 +140,7 @@ void SetPrimitiveVariables(const FieldDescriptor* descriptor,
(*variables)["fixed_size"] = StrCat(fixed_size);
}
if (SupportFieldPresence(descriptor)) {
if (HasHasbit(descriptor)) {
// For singular messages and builders, one bit is used for the hasField bit.
(*variables)["get_has_field_bit_message"] = GenerateGetBit(messageBitIndex);
@ -188,7 +188,7 @@ ImmutablePrimitiveFieldLiteGenerator::ImmutablePrimitiveFieldLiteGenerator(
ImmutablePrimitiveFieldLiteGenerator::~ImmutablePrimitiveFieldLiteGenerator() {}
int ImmutablePrimitiveFieldLiteGenerator::GetNumBitsForMessage() const {
return SupportFieldPresence(descriptor_) ? 1 : 0;
return HasHasbit(descriptor_) ? 1 : 0;
}
void ImmutablePrimitiveFieldLiteGenerator::GenerateInterfaceMembers(

4
src/google/protobuf/compiler/java/java_string_field.cc
View File

@ -90,7 +90,7 @@ void SetPrimitiveVariables(const FieldDescriptor* descriptor,
descriptor->options().deprecated() ? "@java.lang.Deprecated " : "";
(*variables)["on_changed"] = "onChanged();";
if (SupportFieldPresence(descriptor)) {
if (HasHasbit(descriptor)) {
// For singular messages and builders, one bit is used for the hasField bit.
(*variables)["get_has_field_bit_message"] = GenerateGetBit(messageBitIndex);
(*variables)["get_has_field_bit_builder"] = GenerateGetBit(builderBitIndex);
@ -147,7 +147,7 @@ ImmutableStringFieldGenerator::ImmutableStringFieldGenerator(
ImmutableStringFieldGenerator::~ImmutableStringFieldGenerator() {}
int ImmutableStringFieldGenerator::GetNumBitsForMessage() const {
return SupportFieldPresence(descriptor_) ? 1 : 0;
return HasHasbit(descriptor_) ? 1 : 0;
}
int ImmutableStringFieldGenerator::GetNumBitsForBuilder() const {

4
src/google/protobuf/compiler/java/java_string_field_lite.cc
View File

@ -85,7 +85,7 @@ void SetPrimitiveVariables(const FieldDescriptor* descriptor,
descriptor->options().deprecated() ? "@java.lang.Deprecated " : "";
(*variables)["required"] = descriptor->is_required() ? "true" : "false";
if (SupportFieldPresence(descriptor)) {
if (HasHasbit(descriptor)) {
// For singular messages and builders, one bit is used for the hasField bit.
(*variables)["get_has_field_bit_message"] = GenerateGetBit(messageBitIndex);
@ -127,7 +127,7 @@ ImmutableStringFieldLiteGenerator::ImmutableStringFieldLiteGenerator(
ImmutableStringFieldLiteGenerator::~ImmutableStringFieldLiteGenerator() {}
int ImmutableStringFieldLiteGenerator::GetNumBitsForMessage() const {
return SupportFieldPresence(descriptor_) ? 1 : 0;
return HasHasbit(descriptor_) ? 1 : 0;
}
// A note about how strings are handled. In the SPEED and CODE_SIZE runtimes,

1
src/google/protobuf/compiler/main.cc
View File

@ -65,6 +65,7 @@ int ProtobufMain(int argc, char* argv[]) {
"Generate Java source file.");
// Proto2 Python
python::Generator py_generator;
cli.RegisterGenerator("--python_out", "--python_opt", &py_generator,

8
src/google/protobuf/compiler/mock_code_generator.cc
View File

@ -119,7 +119,7 @@ void MockCodeGenerator::ExpectGenerated(
std::vector<std::string> insertion_list;
if (!insertions.empty()) {
SplitStringUsing(insertions, ",", &insertion_list);
insertion_list = Split(insertions, ",", true);
}
EXPECT_EQ(lines.size(), 3 + insertion_list.size() * 2);
@ -250,12 +250,10 @@ bool MockCodeGenerator::Generate(const FileDescriptor* file,
bool insert_endlines = HasPrefixString(parameter, "insert_endlines=");
if (insert_endlines || HasPrefixString(parameter, "insert=")) {
std::vector<std::string> insert_into;
SplitStringUsing(
std::vector<std::string> insert_into = Split(
StripPrefixString(
parameter, insert_endlines ? "insert_endlines=" : "insert="),
",", &insert_into);
",", true);
for (size_t i = 0; i < insert_into.size(); i++) {
{

Some files were not shown because too many files have changed in this diff Show More