From 1d4e9593745c1cbff916cb5ab9c0c87d07369fcb Mon Sep 17 00:00:00 2001 From: Hao Nguyen Date: Fri, 3 May 2019 14:33:43 -0700 Subject: [PATCH] Down integrate to GitHub --- cmake/libprotobuf-lite.cmake | 3 + cmake/libprotobuf.cmake | 2 - .../ExtensionRegistryFactoryTest.java | 3 +- js/gulpfile.js | 104 ++++++---- .../protobuf/internal/text_format_test.py | 12 ++ python/google/protobuf/text_format.py | 43 +++-- src/Makefile.am | 3 +- src/google/protobuf/compiler/cpp/cpp_enum.cc | 179 +++++++++++++++--- src/google/protobuf/compiler/cpp/cpp_file.cc | 16 +- .../protobuf/compiler/cpp/cpp_map_field.cc | 21 +- src/google/protobuf/descriptor.cc | 5 +- src/google/protobuf/generated_enum_util.cc | 95 ++++++++++ src/google/protobuf/generated_enum_util.h | 32 ++++ .../protobuf/generated_message_reflection.cc | 9 +- src/google/protobuf/lite_unittest.cc | 94 +++++++++ src/google/protobuf/map.h | 3 +- src/google/protobuf/map_entry_lite.h | 127 +++++-------- src/google/protobuf/map_field.h | 3 - src/google/protobuf/message.cc | 45 ----- src/google/protobuf/message.h | 29 --- src/google/protobuf/message_lite.cc | 48 ++++- src/google/protobuf/message_lite.h | 33 +++- src/google/protobuf/parse_context.h | 2 +- src/google/protobuf/struct.pb.cc | 16 +- src/google/protobuf/unittest_lite.proto | 15 +- 25 files changed, 655 insertions(+), 287 deletions(-) create mode 100755 src/google/protobuf/generated_enum_util.cc diff --git a/cmake/libprotobuf-lite.cmake b/cmake/libprotobuf-lite.cmake index 652eb91cf..c06cf9658 100644 --- a/cmake/libprotobuf-lite.cmake +++ b/cmake/libprotobuf-lite.cmake @@ -2,6 +2,7 @@ set(libprotobuf_lite_files ${protobuf_source_dir}/src/google/protobuf/any_lite.cc ${protobuf_source_dir}/src/google/protobuf/arena.cc ${protobuf_source_dir}/src/google/protobuf/extension_set.cc + ${protobuf_source_dir}/src/google/protobuf/generated_enum_util.cc ${protobuf_source_dir}/src/google/protobuf/generated_message_table_driven_lite.cc ${protobuf_source_dir}/src/google/protobuf/generated_message_util.cc ${protobuf_source_dir}/src/google/protobuf/implicit_weak_message.cc @@ -9,6 +10,7 @@ set(libprotobuf_lite_files ${protobuf_source_dir}/src/google/protobuf/io/io_win32.cc ${protobuf_source_dir}/src/google/protobuf/io/strtod.cc ${protobuf_source_dir}/src/google/protobuf/io/zero_copy_stream.cc + ${protobuf_source_dir}/src/google/protobuf/io/zero_copy_stream_impl.cc ${protobuf_source_dir}/src/google/protobuf/io/zero_copy_stream_impl_lite.cc ${protobuf_source_dir}/src/google/protobuf/message_lite.cc ${protobuf_source_dir}/src/google/protobuf/parse_context.cc @@ -36,6 +38,7 @@ set(libprotobuf_lite_includes ${protobuf_source_dir}/src/google/protobuf/io/coded_stream.h ${protobuf_source_dir}/src/google/protobuf/io/strtod.h ${protobuf_source_dir}/src/google/protobuf/io/zero_copy_stream.h + ${protobuf_source_dir}/src/google/protobuf/io/zero_copy_stream_impl.h ${protobuf_source_dir}/src/google/protobuf/io/zero_copy_stream_impl_lite.h ${protobuf_source_dir}/src/google/protobuf/message_lite.h ${protobuf_source_dir}/src/google/protobuf/repeated_field.h diff --git a/cmake/libprotobuf.cmake b/cmake/libprotobuf.cmake index 9a434a7db..fd70da7e4 100644 --- a/cmake/libprotobuf.cmake +++ b/cmake/libprotobuf.cmake @@ -17,7 +17,6 @@ set(libprotobuf_files ${protobuf_source_dir}/src/google/protobuf/io/gzip_stream.cc ${protobuf_source_dir}/src/google/protobuf/io/printer.cc ${protobuf_source_dir}/src/google/protobuf/io/tokenizer.cc - ${protobuf_source_dir}/src/google/protobuf/io/zero_copy_stream_impl.cc ${protobuf_source_dir}/src/google/protobuf/map_field.cc ${protobuf_source_dir}/src/google/protobuf/message.cc ${protobuf_source_dir}/src/google/protobuf/reflection_ops.cc @@ -72,7 +71,6 @@ set(libprotobuf_includes ${protobuf_source_dir}/src/google/protobuf/io/gzip_stream.h ${protobuf_source_dir}/src/google/protobuf/io/printer.h ${protobuf_source_dir}/src/google/protobuf/io/tokenizer.h - ${protobuf_source_dir}/src/google/protobuf/io/zero_copy_stream_impl.h ${protobuf_source_dir}/src/google/protobuf/map_field.h ${protobuf_source_dir}/src/google/protobuf/message.h ${protobuf_source_dir}/src/google/protobuf/reflection_ops.h diff --git a/java/core/src/test/java/com/google/protobuf/ExtensionRegistryFactoryTest.java b/java/core/src/test/java/com/google/protobuf/ExtensionRegistryFactoryTest.java index 3eb0cedc4..c6aed074f 100644 --- a/java/core/src/test/java/com/google/protobuf/ExtensionRegistryFactoryTest.java +++ b/java/core/src/test/java/com/google/protobuf/ExtensionRegistryFactoryTest.java @@ -123,8 +123,7 @@ public class ExtensionRegistryFactoryTest extends TestCase { assertTrue( "Test is using a non-lite extension", - GeneratedMessage.GeneratedExtension.class.isAssignableFrom( - NonNestedExtension.nonNestedExtension.getClass())); + Extension.class.isAssignableFrom(NonNestedExtension.nonNestedExtension.getClass())); assertNotNull( "Extension is registered in masqueraded full registry", fullRegistry2.findImmutableExtensionByName("protobuf_unittest.nonNestedExtension")); diff --git a/js/gulpfile.js b/js/gulpfile.js index 449ae502c..1f0946caf 100644 --- a/js/gulpfile.js +++ b/js/gulpfile.js @@ -144,7 +144,7 @@ function getClosureBuilderCommand(exportsFile, outputFile) { exportsFile + ' > ' + outputFile; } -gulp.task('dist', gulp.series(['genproto_wellknowntypes'], function (cb) { +gulp.task('dist', gulp.series(['genproto_wellknowntypes'], function(cb) { // TODO(haberman): minify this more aggressively. // Will require proper externs/exports. exec(getClosureBuilderCommand('commonjs/export.js', 'google-protobuf.js'), @@ -179,49 +179,75 @@ gulp.task('commonjs_testdeps', function (cb) { }); }); -gulp.task('make_commonjs_out', gulp.series(['dist', 'genproto_well_known_types_commonjs', 'genproto_group1_commonjs', 'genproto_group2_commonjs', 'genproto_commonjs_wellknowntypes', 'commonjs_asserts', 'commonjs_testdeps', 'genproto_group3_commonjs_strict'], function (cb) { - // TODO(haberman): minify this more aggressively. - // Will require proper externs/exports. - var cmd = "mkdir -p commonjs_out/binary && mkdir -p commonjs_out/test_node_modules && "; - function addTestFile(file) { - cmd += 'node commonjs/rewrite_tests_for_commonjs.js < ' + file + - ' > commonjs_out/' + file + '&& '; - } +gulp.task( + 'make_commonjs_out', + gulp.series( + [ + 'dist', 'genproto_well_known_types_commonjs', + 'genproto_group1_commonjs', 'genproto_group2_commonjs', + 'genproto_commonjs_wellknowntypes', 'commonjs_asserts', + 'commonjs_testdeps', 'genproto_group3_commonjs_strict' + ], + function(cb) { + // TODO(haberman): minify this more aggressively. + // Will require proper externs/exports. + var cmd = + 'mkdir -p commonjs_out/binary && mkdir -p commonjs_out/test_node_modules && '; + function addTestFile(file) { + cmd += 'node commonjs/rewrite_tests_for_commonjs.js < ' + file + + ' > commonjs_out/' + file + '&& '; + } - glob.sync('*_test.js').forEach(addTestFile); - glob.sync('binary/*_test.js').forEach(addTestFile); + glob.sync('*_test.js').forEach(addTestFile); + glob.sync('binary/*_test.js').forEach(addTestFile); - exec(cmd + - 'cp commonjs/jasmine.json commonjs_out/jasmine.json && ' + - 'cp google-protobuf.js commonjs_out/test_node_modules && ' + - 'cp commonjs/strict_test.js commonjs_out/strict_test.js &&' + - 'cp commonjs/import_test.js commonjs_out/import_test.js', - function (err, stdout, stderr) { - console.log(stdout); - console.log(stderr); - cb(err); - }); -})); + exec( + cmd + 'cp commonjs/jasmine.json commonjs_out/jasmine.json && ' + + 'cp google-protobuf.js commonjs_out/test_node_modules && ' + + 'cp commonjs/strict_test.js commonjs_out/strict_test.js &&' + + 'cp commonjs/import_test.js commonjs_out/import_test.js', + function(err, stdout, stderr) { + console.log(stdout); + console.log(stderr); + cb(err); + }); + })); -gulp.task('deps', gulp.series(['genproto_well_known_types_closure', 'genproto_group1_closure', 'genproto_group2_closure'], function (cb) { - exec('./node_modules/google-closure-library/closure/bin/build/depswriter.py binary/arith.js binary/constants.js binary/decoder.js binary/encoder.js binary/reader.js binary/utils.js binary/writer.js debug.js map.js message.js node_loader.js test_bootstrap.js > deps.js', - function (err, stdout, stderr) { - console.log(stdout); - console.log(stderr); - cb(err); - }); -})); +gulp.task( + 'deps', + gulp.series( + [ + 'genproto_well_known_types_closure', 'genproto_group1_closure', + 'genproto_group2_closure' + ], + function(cb) { + exec( + './node_modules/google-closure-library/closure/bin/build/depswriter.py binary/arith.js binary/constants.js binary/decoder.js binary/encoder.js binary/reader.js binary/utils.js binary/writer.js debug.js map.js message.js node_loader.js test_bootstrap.js > deps.js', + function(err, stdout, stderr) { + console.log(stdout); + console.log(stderr); + cb(err); + }); + })); -gulp.task('test_closure', gulp.series(['genproto_well_known_types_closure', 'genproto_group1_closure', 'genproto_group2_closure', 'deps'], function (cb) { - exec('JASMINE_CONFIG_PATH=jasmine.json ./node_modules/.bin/jasmine', - function (err, stdout, stderr) { - console.log(stdout); - console.log(stderr); - cb(err); - }); -})); +gulp.task( + 'test_closure', + gulp.series( + [ + 'genproto_well_known_types_closure', 'genproto_group1_closure', + 'genproto_group2_closure', 'deps' + ], + function(cb) { + exec( + 'JASMINE_CONFIG_PATH=jasmine.json ./node_modules/.bin/jasmine', + function(err, stdout, stderr) { + console.log(stdout); + console.log(stderr); + cb(err); + }); + })); -gulp.task('test_commonjs', gulp.series(['make_commonjs_out'], function (cb) { +gulp.task('test_commonjs', gulp.series(['make_commonjs_out'], function(cb) { exec('cd commonjs_out && JASMINE_CONFIG_PATH=jasmine.json NODE_PATH=test_node_modules ../node_modules/.bin/jasmine', function (err, stdout, stderr) { console.log(stdout); diff --git a/python/google/protobuf/internal/text_format_test.py b/python/google/protobuf/internal/text_format_test.py index 8aff9506b..c705dcd76 100755 --- a/python/google/protobuf/internal/text_format_test.py +++ b/python/google/protobuf/internal/text_format_test.py @@ -316,6 +316,18 @@ class TextFormatMessageToStringTests(TextFormatBase): 'payload {{\n {0}\n {1}\n {2}\n {3}\n}}\n'.format( *formatted_fields)) + # Test default float_format has 8 valid digits. + message.payload.optional_float = 1.2345678912 + message.payload.optional_double = 1.2345678912 + formatted_fields = ['optional_float: 1.2345679', + 'optional_double: 1.2345678912', + 'repeated_float: -5642', 'repeated_double: 7.89e-5'] + text_message = text_format.MessageToString(message) + self.CompareToGoldenText( + self.RemoveRedundantZeros(text_message), + 'payload {{\n {0}\n {1}\n {2}\n {3}\n}}\n'.format( + *formatted_fields)) + def testMessageToString(self, message_module): message = message_module.ForeignMessage() message.c = 123 diff --git a/python/google/protobuf/text_format.py b/python/google/protobuf/text_format.py index 028a6acc7..9f860741e 100755 --- a/python/google/protobuf/text_format.py +++ b/python/google/protobuf/text_format.py @@ -133,10 +133,10 @@ def MessageToString(message, # type: (...) -> str """Convert protobuf message to text format. - Floating point values can be formatted compactly with 15 digits of + Double values can be formatted compactly with 15 digits of precision (which is the most that IEEE 754 "double" can guarantee) - using float_format='.15g'. To ensure that converting to text and back to a - proto will result in an identical value, float_format='.17g' should be used. + using double_format='.15g'. To ensure that converting to text and back to a + proto will result in an identical value, double_format='.17g' should be used. Args: message: The protocol buffers message. @@ -153,11 +153,12 @@ def MessageToString(message, determined by the extension number. By default, use the field number order. float_format: If set, use this to specify float field formatting - (per the "Format Specification Mini-Language"); otherwise, str() is used. - Also affect double field if double_format is not set. + (per the "Format Specification Mini-Language"); otherwise, 8 valid digits + is used (default '.8g'). Also affect double field if double_format is + not set but float_format is set. double_format: If set, use this to specify double field formatting - (per the "Format Specification Mini-Language"); otherwise, float_format - is used. + (per the "Format Specification Mini-Language"); if it is not set but + float_format is set, use float_format. Otherwise, use str() use_field_number: If True, print field numbers instead of names. descriptor_pool: A DescriptorPool used to resolve Any types. indent: The initial indent level, in terms of spaces, for pretty print. @@ -322,10 +323,10 @@ class _Printer(object): print_unknown_fields=False): """Initialize the Printer. - Floating point values can be formatted compactly with 15 digits of - precision (which is the most that IEEE 754 "double" can guarantee) - using float_format='.15g'. To ensure that converting to text and back to a - proto will result in an identical value, float_format='.17g' should be used. + Double values can be formatted compactly with 15 digits of precision + (which is the most that IEEE 754 "double" can guarantee) using + double_format='.15g'. To ensure that converting to text and back to a proto + will result in an identical value, double_format='.17g' should be used. Args: out: To record the text format result. @@ -340,11 +341,13 @@ class _Printer(object): use_index_order: If True, print fields of a proto message using the order defined in source code instead of the field number. By default, use the field number order. - float_format: If set, use this to specify floating point number formatting - (per the "Format Specification Mini-Language"); otherwise, str() is - used. Also affect double field if double_format is not set. - double_format: If set, use this to specify double field formatting; - otherwise, float_format is used. + float_format: If set, use this to specify float field formatting + (per the "Format Specification Mini-Language"); otherwise, 8 valid + digits is used (default '.8g'). Also affect double field if + double_format is not set but float_format is set. + double_format: If set, use this to specify double field formatting + (per the "Format Specification Mini-Language"); if it is not set but + float_format is set, use float_format. Otherwise, str() is used. use_field_number: If True, print field numbers instead of names. descriptor_pool: A DescriptorPool used to resolve Any types. message_formatter: A function(message, indent, as_one_line): unicode|None @@ -589,9 +592,11 @@ class _Printer(object): out.write('true') else: out.write('false') - elif (field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_FLOAT and - self.float_format is not None): - out.write('{1:{0}}'.format(self.float_format, value)) + elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_FLOAT: + if self.float_format is not None: + out.write('{1:{0}}'.format(self.float_format, value)) + else: + out.write(str(float(format(value, '.8g')))) elif (field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_DOUBLE and self.double_format is not None): out.write('{1:{0}}'.format(self.double_format, value)) diff --git a/src/Makefile.am b/src/Makefile.am index 41bf7b413..0627861a9 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -202,6 +202,7 @@ libprotobuf_lite_la_SOURCES = \ google/protobuf/any_lite.cc \ google/protobuf/arena.cc \ google/protobuf/extension_set.cc \ + google/protobuf/generated_enum_util.cc \ google/protobuf/generated_message_util.cc \ google/protobuf/generated_message_table_driven_lite.h \ google/protobuf/generated_message_table_driven_lite.cc \ @@ -214,6 +215,7 @@ libprotobuf_lite_la_SOURCES = \ google/protobuf/io/coded_stream_inl.h \ google/protobuf/io/strtod.cc \ google/protobuf/io/zero_copy_stream.cc \ + google/protobuf/io/zero_copy_stream_impl.cc \ google/protobuf/io/zero_copy_stream_impl_lite.cc libprotobuf_la_LIBADD = $(PTHREAD_LIBS) $(LIBATOMIC_LIBS) @@ -258,7 +260,6 @@ libprotobuf_la_SOURCES = \ google/protobuf/io/gzip_stream.cc \ google/protobuf/io/printer.cc \ google/protobuf/io/tokenizer.cc \ - google/protobuf/io/zero_copy_stream_impl.cc \ google/protobuf/compiler/importer.cc \ google/protobuf/compiler/parser.cc \ google/protobuf/util/delimited_message_util.cc \ diff --git a/src/google/protobuf/compiler/cpp/cpp_enum.cc b/src/google/protobuf/compiler/cpp/cpp_enum.cc index 5e2ac2536..366fcb66e 100644 --- a/src/google/protobuf/compiler/cpp/cpp_enum.cc +++ b/src/google/protobuf/compiler/cpp/cpp_enum.cc @@ -57,6 +57,17 @@ bool ShouldGenerateArraySize(const EnumDescriptor* descriptor) { } return max_value != kint32max; } + +// Returns the number of unique numeric enum values. This is less than +// descriptor->value_count() when there are aliased values. +int CountUniqueValues(const EnumDescriptor* descriptor) { + std::set values; + for (int i = 0; i < descriptor->value_count(); ++i) { + values.insert(descriptor->value(i)->number()); + } + return values.size(); +} + } // namespace EnumGenerator::EnumGenerator(const EnumDescriptor* descriptor, @@ -143,25 +154,42 @@ void EnumGenerator::GenerateDefinition(io::Printer* printer) { format( "$dllexport_decl $const ::$proto_ns$::EnumDescriptor* " "$classname$_descriptor();\n"); - // The _Name and _Parse methods - // Support a stricter, type-checked enum-to-string method that - // statically checks whether the parameter is the exact enum type or is - // an integral type. + } + + // The _Name and _Parse functions. The lite implementation is table-based, so + // we make sure to keep the tables hidden in the .cc file. + if (!HasDescriptorMethods(descriptor_->file(), options_)) { + format("const std::string& $classname$_Name($classname$ value);\n"); + } + // The _Name() function accepts the enum type itself but also any integral + // type. + format( + "template\n" + "inline const std::string& $classname$_Name(T enum_t_value) {\n" + " static_assert(::std::is_same::value ||\n" + " ::std::is_integral::value,\n" + " \"Incorrect type passed to function $classname$_Name.\");\n"); + if (HasDescriptorMethods(descriptor_->file(), options_)) { format( - "template\n" - "inline const std::string& $classname$_Name(T enum_t_value) {\n" - " static_assert(::std::is_same::value ||\n" - " ::std::is_integral::value,\n" - " \"Incorrect type passed to function $classname$_Name.\");\n" " return ::$proto_ns$::internal::NameOfEnum(\n" - " $classname$_descriptor(), enum_t_value);\n" - "}\n"); + " $classname$_descriptor(), enum_t_value);\n"); + } else { + format( + " return $classname$_Name(static_cast<$classname$>(enum_t_value));\n"); + } + format("}\n"); + + if (HasDescriptorMethods(descriptor_->file(), options_)) { format( "inline bool $classname$_Parse(\n" " const std::string& name, $classname$* value) {\n" " return ::$proto_ns$::internal::ParseNamedEnum<$classname$>(\n" " $classname$_descriptor(), name, value);\n" "}\n"); + } else { + format( + "bool $classname$_Parse(\n" + " const std::string& name, $classname$* value);\n"); } } @@ -216,24 +244,21 @@ void EnumGenerator::GenerateSymbolImports(io::Printer* printer) const { "$nested_name$_descriptor() {\n" " return $classname$_descriptor();\n" "}\n"); - // Support a stricter, type-checked enum-to-string method that - // statically checks whether the parameter is the exact enum type or is - // an integral type. - format( - "template\n" - "static inline const std::string& $nested_name$_Name(T enum_t_value) " - "{\n" - " static_assert(::std::is_same::value ||\n" - " ::std::is_integral::value,\n" - " \"Incorrect type passed to function $nested_name$_Name.\");\n" - " return $classname$_Name(enum_t_value);\n" - "}\n"); - format( - "static inline bool $nested_name$_Parse(const std::string& name,\n" - " $resolved_name$* value) {\n" - " return $classname$_Parse(name, value);\n" - "}\n"); } + + format( + "template\n" + "static inline const std::string& $nested_name$_Name(T enum_t_value) {\n" + " static_assert(::std::is_same::value ||\n" + " ::std::is_integral::value,\n" + " \"Incorrect type passed to function $nested_name$_Name.\");\n" + " return $classname$_Name(enum_t_value);\n" + "}\n"); + format( + "static inline bool $nested_name$_Parse(const std::string& name,\n" + " $resolved_name$* value) {\n" + " return $classname$_Parse(name, value);\n" + "}\n"); } void EnumGenerator::GenerateMethods(int idx, io::Printer* printer) { @@ -274,6 +299,104 @@ void EnumGenerator::GenerateMethods(int idx, io::Printer* printer) { "}\n" "\n"); + if (!HasDescriptorMethods(descriptor_->file(), options_)) { + // In lite mode (where descriptors are unavailable), we generate separate + // tables for mapping between enum names and numbers. The _entries table + // contains the bulk of the data and is sorted by name, while + // _entries_by_number is sorted by number and just contains pointers into + // _entries. The two tables allow mapping from name to number and number to + // name, both in time logarithmic in the number of enum entries. This could + // probably be made faster, but for now the tables are intended to be simple + // and compact. + // + // Enums with allow_alias = true support multiple entries with the same + // numerical value. In cases where there are multiple names for the same + // number, we treat the first name appearing in the .proto file as the + // canonical one. + std::map name_to_number; + std::map number_to_canonical_name; + for (int i = 0; i < descriptor_->value_count(); i++) { + const EnumValueDescriptor* value = descriptor_->value(i); + name_to_number.emplace(value->name(), value->number()); + // The same number may appear with multiple names, so we use emplace() to + // let the first name win. + number_to_canonical_name.emplace(value->number(), value->name()); + } + + format( + "static ::$proto_ns$::internal::ExplicitlyConstructed " + "$classname$_strings[$1$] = {};\n\n", + CountUniqueValues(descriptor_)); + + // We concatenate all the names for a given enum into one big string + // literal. If instead we store an array of string literals, the linker + // seems to put all enum strings for a given .proto file in the same + // section, which hinders its ability to strip out unused strings. + format("static const char $classname$_names[] ="); + for (const auto& p : name_to_number) { + format("\n \"$1$\"", p.first); + } + format(";\n\n"); + + format( + "static const ::$proto_ns$::internal::EnumEntry $classname$_entries[] " + "= {\n"); + int i = 0; + std::map number_to_index; + int data_index = 0; + for (const auto& p : name_to_number) { + format(" { {$classname$_names + $1$, $2$}, $3$ },\n", data_index, + p.first.size(), p.second); + if (number_to_canonical_name[p.second] == p.first) { + number_to_index.emplace(p.second, i); + } + ++i; + data_index += p.first.size(); + } + + format( + "};\n" + "\n" + "static const int $classname$_entries_by_number[] = {\n"); + for (const auto& p : number_to_index) { + format(" $1$, // $2$ -> $3$\n", p.second, p.first, + number_to_canonical_name[p.first]); + } + format( + "};\n" + "\n"); + + format( + "const std::string& $classname$_Name(\n" + " $classname$ value) {\n" + " static const bool dummy =\n" + " ::$proto_ns$::internal::InitializeEnumStrings(\n" + " $classname$_entries,\n" + " $classname$_entries_by_number,\n" + " $1$, $classname$_strings);\n" + " (void) dummy;\n" + " int idx = ::$proto_ns$::internal::LookUpEnumName(\n" + " $classname$_entries,\n" + " $classname$_entries_by_number,\n" + " $1$, value);\n" + " return idx == -1 ? ::$proto_ns$::internal::GetEmptyString() :\n" + " $classname$_strings[idx].get();\n" + "}\n", + CountUniqueValues(descriptor_)); + format( + "bool $classname$_Parse(\n" + " const std::string& name, $classname$* value) {\n" + " int int_value;\n" + " bool success = ::$proto_ns$::internal::LookUpEnumValue(\n" + " $classname$_entries, $1$, name, &int_value);\n" + " if (success) {\n" + " *value = static_cast<$classname$>(int_value);\n" + " }\n" + " return success;\n" + "}\n", + descriptor_->value_count()); + } + if (descriptor_->containing_type() != NULL) { std::string parent = ClassName(descriptor_->containing_type(), false); // Before C++17, we must define the static constants which were diff --git a/src/google/protobuf/compiler/cpp/cpp_file.cc b/src/google/protobuf/compiler/cpp/cpp_file.cc index 08b9f2887..8bc1f40bc 100644 --- a/src/google/protobuf/compiler/cpp/cpp_file.cc +++ b/src/google/protobuf/compiler/cpp/cpp_file.cc @@ -892,11 +892,17 @@ void FileGenerator::GenerateReflectionInitializationCode(io::Printer* printer) { protodef_name, file_data.size(), sccs_.size(), num_deps, message_generators_.size()); - format( - "// Force running AddDescriptors() at dynamic initialization time.\n" - "static bool $1$ = (" - " ::$proto_ns$::internal::AddDescriptors(&$desc_table$), true);\n", - UniqueName("dynamic_init_dummy", file_, options_)); + // For descriptor.proto we want to avoid doing any dynamic initialization, + // because in some situations that would otherwise pull in a lot of + // unnecessary code that can't be stripped by --gc-sections. Descriptor + // initialization will still be performed lazily when it's needed. + if (file_->name() != "net/proto2/proto/descriptor.proto") { + format( + "// Force running AddDescriptors() at dynamic initialization time.\n" + "static bool $1$ = (" + " ::$proto_ns$::internal::AddDescriptors(&$desc_table$), true);\n", + UniqueName("dynamic_init_dummy", file_, options_)); + } } void FileGenerator::GenerateInitForSCC(const SCC* scc, io::Printer* printer) { diff --git a/src/google/protobuf/compiler/cpp/cpp_map_field.cc b/src/google/protobuf/compiler/cpp/cpp_map_field.cc index 0738b91e6..38f0584c3 100644 --- a/src/google/protobuf/compiler/cpp/cpp_map_field.cc +++ b/src/google/protobuf/compiler/cpp/cpp_map_field.cc @@ -65,15 +65,12 @@ void SetMessageVariables(const FieldDescriptor* descriptor, switch (val->cpp_type()) { case FieldDescriptor::CPPTYPE_MESSAGE: (*variables)["val_cpp"] = FieldMessageTypeName(val, options); - (*variables)["wrapper"] = "MapEntryWrapper"; break; case FieldDescriptor::CPPTYPE_ENUM: (*variables)["val_cpp"] = ClassName(val->enum_type(), true); - (*variables)["wrapper"] = "MapEnumEntryWrapper"; break; default: (*variables)["val_cpp"] = PrimitiveTypeName(options, val->cpp_type()); - (*variables)["wrapper"] = "MapEntryWrapper"; } (*variables)["key_wire_type"] = "TYPE_" + ToUpper(DeclaredTypeMethodName(key->type())); @@ -250,17 +247,16 @@ static void GenerateSerializationLoop(const Formatter& format, bool string_key, } format.Indent(); - format( - "$map_classname$::$wrapper$ entry(nullptr, $1$->first, $1$->second);\n", - ptr); if (to_array) { format( - "target = ::$proto_ns$::internal::WireFormatLite::InternalWrite" - "$declared_type$NoVirtualToArray($number$, entry, target);\n"); + "target = $map_classname$::Funcs::SerializeToArray($number$, " + "$1$->first, $1$->second, target);\n", + ptr); } else { format( - "::$proto_ns$::internal::WireFormatLite::Write$stream_writer$($number$," - " entry, output);\n"); + "$map_classname$::Funcs::SerializeToCodedStream($number$, " + "$1$->first, $1$->second, output);\n", + ptr); } if (string_key || string_value) { @@ -370,9 +366,8 @@ void MapFieldGenerator::GenerateByteSize(io::Printer* printer) const { "for (::$proto_ns$::Map< $key_cpp$, $val_cpp$ >::const_iterator\n" " it = this->$name$().begin();\n" " it != this->$name$().end(); ++it) {\n" - " $map_classname$::$wrapper$ entry(nullptr, it->first, it->second);\n" - " total_size += ::$proto_ns$::internal::WireFormatLite::\n" - " $declared_type$SizeNoVirtual(entry);\n" + " total_size += $map_classname$::Funcs::ByteSizeLong(it->first, " + "it->second);\n" "}\n"); } diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index 735c519ff..059498b90 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -1335,7 +1335,10 @@ DescriptorPool* DescriptorPool::internal_generated_pool() { } const DescriptorPool* DescriptorPool::generated_pool() { - return internal_generated_pool(); + const DescriptorPool* pool = internal_generated_pool(); + // Ensure that descriptor.proto has been registered in the generated pool. + DescriptorProto::descriptor(); + return pool; } diff --git a/src/google/protobuf/generated_enum_util.cc b/src/google/protobuf/generated_enum_util.cc new file mode 100755 index 000000000..d0c25a96d --- /dev/null +++ b/src/google/protobuf/generated_enum_util.cc @@ -0,0 +1,95 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include + +#include + +#include + +namespace google { +namespace protobuf { +namespace internal { +namespace { + +bool EnumCompareByName(const EnumEntry& a, const EnumEntry& b) { + return StringPiece(a.name) < StringPiece(b.name); +} + +// Gets the numeric value of the EnumEntry at the given index, but returns a +// special value for the index -1. This gives a way to use std::lower_bound on a +// sorted array of indices while searching for value that we associate with -1. +int GetValue(const EnumEntry* enums, int i, int target) { + if (i == -1) { + return target; + } else { + return enums[i].value; + } +} + +} // namespace + +bool LookUpEnumValue(const EnumEntry* enums, size_t size, + StringPiece name, int* value) { + EnumEntry target{name, 0}; + auto it = std::lower_bound(enums, enums + size, target, EnumCompareByName); + if (it != enums + size && it->name == name) { + *value = it->value; + return true; + } + return false; +} + +int LookUpEnumName(const EnumEntry* enums, const int* sorted_indices, + size_t size, int value) { + auto comparator = [enums, value](int a, int b) { + return GetValue(enums, a, value) < GetValue(enums, b, value); + }; + auto it = + std::lower_bound(sorted_indices, sorted_indices + size, -1, comparator); + if (it != sorted_indices + size && enums[*it].value == value) { + return it - sorted_indices; + } + return -1; +} + +bool InitializeEnumStrings( + const EnumEntry* enums, const int* sorted_indices, size_t size, + internal::ExplicitlyConstructed* enum_strings) { + for (int i = 0; i < size; ++i) { + enum_strings[i].Construct(enums[sorted_indices[i]].name); + internal::OnShutdownDestroyString(enum_strings[i].get_mutable()); + } + return true; +} + +} // namespace internal +} // namespace protobuf +} // namespace google diff --git a/src/google/protobuf/generated_enum_util.h b/src/google/protobuf/generated_enum_util.h index 55db124c8..f1002e2d1 100644 --- a/src/google/protobuf/generated_enum_util.h +++ b/src/google/protobuf/generated_enum_util.h @@ -33,6 +33,11 @@ #include +#include +#include + +#include + #ifdef SWIG #error "You cannot SWIG proto headers" #endif @@ -45,7 +50,34 @@ namespace protobuf { template struct is_proto_enum : ::std::false_type {}; +namespace internal { + +// The table entry format for storing enum name-to-value mapping used with lite +// protos. This struct and the following related functions should only be used +// by protobuf generated code. +struct EnumEntry { + StringPiece name; + int value; +}; + +// Looks up a numeric enum value given the string name. +PROTOBUF_EXPORT bool LookUpEnumValue(const EnumEntry* enums, size_t size, + StringPiece name, int* value); + +// Looks up an enum name given the numeric value. +PROTOBUF_EXPORT int LookUpEnumName(const EnumEntry* enums, + const int* sorted_indices, size_t size, + int value); + +// Initializes the list of enum names in std::string form. +PROTOBUF_EXPORT bool InitializeEnumStrings( + const EnumEntry* enums, const int* sorted_indices, size_t size, + internal::ExplicitlyConstructed* enum_strings); + +} // namespace internal } // namespace protobuf } // namespace google +#include + #endif // GOOGLE_PROTOBUF_GENERATED_ENUM_UTIL_H__ diff --git a/src/google/protobuf/generated_message_reflection.cc b/src/google/protobuf/generated_message_reflection.cc index 52e8746db..b7cdc87b1 100644 --- a/src/google/protobuf/generated_message_reflection.cc +++ b/src/google/protobuf/generated_message_reflection.cc @@ -203,8 +203,8 @@ GeneratedMessageReflection::GeneratedMessageReflection( const DescriptorPool* pool, MessageFactory* factory) : descriptor_(descriptor), schema_(schema), - descriptor_pool_((pool == NULL) ? DescriptorPool::generated_pool() - : pool), + descriptor_pool_( + (pool == nullptr) ? DescriptorPool::internal_generated_pool() : pool), message_factory_(factory), last_non_weak_field_index_(-1) { last_non_weak_field_index_ = descriptor_->field_count() - 1; @@ -2250,7 +2250,7 @@ class AssignDescriptorsHelper { descriptor, MigrationToReflectionSchema(default_instance_data_, offsets_, *schemas_), - DescriptorPool::generated_pool(), factory_); + DescriptorPool::internal_generated_pool(), factory_); for (int i = 0; i < descriptor->enum_type_count(); i++) { AssignEnumDescriptor(descriptor->enum_type(i)); } @@ -2317,7 +2317,8 @@ void AssignDescriptorsImpl(const DescriptorTable* table) { } // Fill the arrays with pointers to descriptors and reflection classes. const FileDescriptor* file = - DescriptorPool::generated_pool()->FindFileByName(table->filename); + DescriptorPool::internal_generated_pool()->FindFileByName( + table->filename); GOOGLE_CHECK(file != NULL); MessageFactory* factory = MessageFactory::generated_factory(); diff --git a/src/google/protobuf/lite_unittest.cc b/src/google/protobuf/lite_unittest.cc index c16bdf7e6..ccaea13c3 100644 --- a/src/google/protobuf/lite_unittest.cc +++ b/src/google/protobuf/lite_unittest.cc @@ -1097,5 +1097,99 @@ TEST(Lite, DebugString) { EXPECT_NE(message1.DebugString(), message2.DebugString()); } +TEST(Lite, EnumValueToName) { + EXPECT_EQ("FOREIGN_LITE_FOO", protobuf_unittest::ForeignEnumLite_Name( + protobuf_unittest::FOREIGN_LITE_FOO)); + EXPECT_EQ("FOREIGN_LITE_BAR", protobuf_unittest::ForeignEnumLite_Name( + protobuf_unittest::FOREIGN_LITE_BAR)); + EXPECT_EQ("FOREIGN_LITE_BAZ", protobuf_unittest::ForeignEnumLite_Name( + protobuf_unittest::FOREIGN_LITE_BAZ)); + EXPECT_EQ("", protobuf_unittest::ForeignEnumLite_Name(0)); + EXPECT_EQ("", protobuf_unittest::ForeignEnumLite_Name(999)); +} + +TEST(Lite, NestedEnumValueToName) { + EXPECT_EQ("FOO", protobuf_unittest::TestAllTypesLite::NestedEnum_Name( + protobuf_unittest::TestAllTypesLite::FOO)); + EXPECT_EQ("BAR", protobuf_unittest::TestAllTypesLite::NestedEnum_Name( + protobuf_unittest::TestAllTypesLite::BAR)); + EXPECT_EQ("BAZ", protobuf_unittest::TestAllTypesLite::NestedEnum_Name( + protobuf_unittest::TestAllTypesLite::BAZ)); + EXPECT_EQ("", protobuf_unittest::TestAllTypesLite::NestedEnum_Name(0)); + EXPECT_EQ("", protobuf_unittest::TestAllTypesLite::NestedEnum_Name(999)); +} + +TEST(Lite, EnumNameToValue) { + protobuf_unittest::ForeignEnumLite value; + + ASSERT_TRUE( + protobuf_unittest::ForeignEnumLite_Parse("FOREIGN_LITE_FOO", &value)); + EXPECT_EQ(protobuf_unittest::FOREIGN_LITE_FOO, value); + + ASSERT_TRUE( + protobuf_unittest::ForeignEnumLite_Parse("FOREIGN_LITE_BAR", &value)); + EXPECT_EQ(protobuf_unittest::FOREIGN_LITE_BAR, value); + + ASSERT_TRUE( + protobuf_unittest::ForeignEnumLite_Parse("FOREIGN_LITE_BAZ", &value)); + EXPECT_EQ(protobuf_unittest::FOREIGN_LITE_BAZ, value); + + // Non-existent values + EXPECT_FALSE(protobuf_unittest::ForeignEnumLite_Parse("E", &value)); + EXPECT_FALSE( + protobuf_unittest::ForeignEnumLite_Parse("FOREIGN_LITE_C", &value)); + EXPECT_FALSE(protobuf_unittest::ForeignEnumLite_Parse("G", &value)); +} + +TEST(Lite, NestedEnumNameToValue) { + protobuf_unittest::TestAllTypesLite::NestedEnum value; + + ASSERT_TRUE( + protobuf_unittest::TestAllTypesLite::NestedEnum_Parse("FOO", &value)); + EXPECT_EQ(protobuf_unittest::TestAllTypesLite::FOO, value); + + ASSERT_TRUE( + protobuf_unittest::TestAllTypesLite::NestedEnum_Parse("BAR", &value)); + EXPECT_EQ(protobuf_unittest::TestAllTypesLite::BAR, value); + + ASSERT_TRUE( + protobuf_unittest::TestAllTypesLite::NestedEnum_Parse("BAZ", &value)); + EXPECT_EQ(protobuf_unittest::TestAllTypesLite::BAZ, value); + + // Non-existent values + EXPECT_FALSE( + protobuf_unittest::TestAllTypesLite::NestedEnum_Parse("A", &value)); + EXPECT_FALSE( + protobuf_unittest::TestAllTypesLite::NestedEnum_Parse("C", &value)); + EXPECT_FALSE( + protobuf_unittest::TestAllTypesLite::NestedEnum_Parse("G", &value)); +} + +TEST(Lite, AliasedEnum) { + // Enums with allow_alias = true can have multiple entries with the same + // value. + EXPECT_EQ("FOO1", protobuf_unittest::DupEnum::TestEnumWithDupValueLite_Name( + protobuf_unittest::DupEnum::FOO1)); + EXPECT_EQ("FOO1", protobuf_unittest::DupEnum::TestEnumWithDupValueLite_Name( + protobuf_unittest::DupEnum::FOO2)); + EXPECT_EQ("BAR1", protobuf_unittest::DupEnum::TestEnumWithDupValueLite_Name( + protobuf_unittest::DupEnum::BAR1)); + EXPECT_EQ("BAR1", protobuf_unittest::DupEnum::TestEnumWithDupValueLite_Name( + protobuf_unittest::DupEnum::BAR2)); + EXPECT_EQ("BAZ", protobuf_unittest::DupEnum::TestEnumWithDupValueLite_Name( + protobuf_unittest::DupEnum::BAZ)); + EXPECT_EQ("", protobuf_unittest::DupEnum::TestEnumWithDupValueLite_Name(999)); + + protobuf_unittest::DupEnum::TestEnumWithDupValueLite value; + ASSERT_TRUE( + protobuf_unittest::DupEnum::TestEnumWithDupValueLite_Parse("FOO1", &value)); + EXPECT_EQ(protobuf_unittest::DupEnum::FOO1, value); + + value = static_cast(0); + ASSERT_TRUE( + protobuf_unittest::DupEnum::TestEnumWithDupValueLite_Parse("FOO2", &value)); + EXPECT_EQ(protobuf_unittest::DupEnum::FOO2, value); +} + } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/map.h b/src/google/protobuf/map.h index 6619fc81c..bc9770159 100644 --- a/src/google/protobuf/map.h +++ b/src/google/protobuf/map.h @@ -937,7 +937,8 @@ class Map { // Return a randomish value. size_type Seed() const { size_type s = static_cast(reinterpret_cast(this)); -#if defined(__x86_64__) && defined(__GNUC__) && !defined(GOOGLE_PROTOBUF_NO_RDTSC) +#if defined(__x86_64__) && defined(__GNUC__) && \ + !defined(GOOGLE_PROTOBUF_NO_RDTSC) uint32 hi, lo; asm("rdtsc" : "=a"(lo), "=d"(hi)); s += ((static_cast(hi) << 32) | lo); diff --git a/src/google/protobuf/map_entry_lite.h b/src/google/protobuf/map_entry_lite.h index c7d7558bd..af7e309ca 100644 --- a/src/google/protobuf/map_entry_lite.h +++ b/src/google/protobuf/map_entry_lite.h @@ -101,6 +101,51 @@ struct MoveHelper { // strings and similar } }; +// Functions for operating on a map entry. Does not contain any representation +// (this class is not intended to be instantiated). +template +struct MapEntryFuncs { + typedef MapTypeHandler KeyTypeHandler; + typedef MapTypeHandler ValueTypeHandler; + static const int kKeyFieldNumber = 1; + static const int kValueFieldNumber = 2; + + static void SerializeToCodedStream(int field_number, const Key& key, + const Value& value, + io::CodedOutputStream* output) { + WireFormatLite::WriteTag(field_number, + WireFormatLite::WIRETYPE_LENGTH_DELIMITED, output); + output->WriteVarint32(GetCachedSize(key, value)); + KeyTypeHandler::Write(kKeyFieldNumber, key, output); + ValueTypeHandler::Write(kValueFieldNumber, value, output); + } + + static ::google::protobuf::uint8* SerializeToArray(int field_number, const Key& key, + const Value& value, ::google::protobuf::uint8* output) { + output = WireFormatLite::WriteTagToArray( + field_number, WireFormatLite::WIRETYPE_LENGTH_DELIMITED, output); + output = io::CodedOutputStream::WriteVarint32ToArray( + static_cast(GetCachedSize(key, value)), output); + output = KeyTypeHandler::WriteToArray(kKeyFieldNumber, key, output); + output = ValueTypeHandler::WriteToArray(kValueFieldNumber, value, output); + return output; + } + + static size_t ByteSizeLong(const Key& key, const Value& value) { + // Tags for key and value will both be one byte (field numbers 1 and 2). + size_t inner_length = + 2 + KeyTypeHandler::ByteSize(key) + ValueTypeHandler::ByteSize(value); + return inner_length + io::CodedOutputStream::VarintSize32(inner_length); + } + + static int GetCachedSize(const Key& key, const Value& value) { + // Tags for key and value will both be one byte (field numbers 1 and 2). + return 2 + KeyTypeHandler::GetCachedSize(key) + + ValueTypeHandler::GetCachedSize(value); + } +}; + // MapEntryImpl is used to implement parsing and serialization of map entries. // It uses Curious Recursive Template Pattern (CRTP) to provide the type of // the eventual code to the template code. @@ -108,6 +153,9 @@ template class MapEntryImpl : public Base { + public: + typedef MapEntryFuncs Funcs; + protected: // Provide utilities to parse/serialize key/value. Provide utilities to // manipulate internal stored type. @@ -350,20 +398,6 @@ class MapEntryImpl : public Base { Arena* GetArena() const override { return GetArenaNoVirtual(); } - // Create a MapEntryImpl for given key and value from Map in - // serialization. This function is only called when value is enum. Enum is - // treated differently because its type in MapEntry is int and its type in - // Map is enum. We cannot create a reference to int from an enum. - static Derived* EnumWrap(const Key& key, const Value value, Arena* arena) { - return Arena::CreateMessage(arena, key, value); - } - - // Like above, but for all the other types. This avoids value copy to create - // MapEntryImpl from Map in serialization. - static Derived* Wrap(const Key& key, const Value& value, Arena* arena) { - return Arena::CreateMessage(arena, key, value); - } - // Parsing using MergePartialFromCodedStream, above, is not as // efficient as it could be. This helper class provides a speedier way. template @@ -535,71 +569,6 @@ class MapEntryImpl : public Base { void clear_has_value() { _has_bits_[0] &= ~0x00000002u; } public: - // Serializing a generated message containing map field involves serializing - // key-value pairs from Map. The wire format of each key-value pair - // after serialization should be the same as that of a MapEntry message - // containing the same key and value inside it. However, Map doesn't - // store key and value as MapEntry message, which disables us to use existing - // code to serialize message. In order to use existing code to serialize - // message, we need to construct a MapEntry from key-value pair. But it - // involves copy of key and value to construct a MapEntry. In order to avoid - // this copy in constructing a MapEntry, we need the following class which - // only takes references of given key and value. - class MapEntryWrapper : public Derived { - typedef Derived BaseClass; - typedef typename BaseClass::KeyMapEntryAccessorType KeyMapEntryAccessorType; - typedef - typename BaseClass::ValueMapEntryAccessorType ValueMapEntryAccessorType; - - public: - MapEntryWrapper(Arena* arena, const Key& key, const Value& value) - : Derived(arena), key_(key), value_(value) { - BaseClass::set_has_key(); - BaseClass::set_has_value(); - } - inline const KeyMapEntryAccessorType& key() const override { return key_; } - inline const ValueMapEntryAccessorType& value() const override { - return value_; - } - - private: - const Key& key_; - const Value& value_; - - friend class ::PROTOBUF_NAMESPACE_ID::Arena; - typedef void InternalArenaConstructable_; - typedef void DestructorSkippable_; - }; - - // Like above, but for enum value only, which stores value instead of - // reference of value field inside. This is needed because the type of value - // field in constructor is an enum, while we need to store it as an int. If we - // initialize a reference to int with a reference to enum, compiler will - // generate a temporary int from enum and initialize the reference to int with - // the temporary. - class MapEnumEntryWrapper : public Derived { - typedef Derived BaseClass; - typedef typename BaseClass::KeyMapEntryAccessorType KeyMapEntryAccessorType; - typedef - typename BaseClass::ValueMapEntryAccessorType ValueMapEntryAccessorType; - - public: - MapEnumEntryWrapper(Arena* arena, const Key& key, const Value& value) - : Derived(arena), key_(key), value_(value) { - BaseClass::set_has_key(); - BaseClass::set_has_value(); - } - inline const KeyMapEntryAccessorType& key() const { return key_; } - inline const ValueMapEntryAccessorType& value() const { return value_; } - - private: - const KeyMapEntryAccessorType& key_; - const ValueMapEntryAccessorType value_; - - friend class ::PROTOBUF_NAMESPACE_ID::Arena; - typedef void DestructorSkippable_; - }; - inline Arena* GetArenaNoVirtual() const { return arena_; } public: // Needed for constructing tables diff --git a/src/google/protobuf/map_field.h b/src/google/protobuf/map_field.h index d804dee76..3b1dbb59f 100644 --- a/src/google/protobuf/map_field.h +++ b/src/google/protobuf/map_field.h @@ -225,9 +225,6 @@ class MapField : public TypeDefinedMapFieldBase { // Define message type for internal repeated field. typedef Derived EntryType; - typedef MapEntryLite - EntryLiteType; // Define abbreviation for parent MapFieldLite typedef MapFieldLiteeof(); -} - -bool Message::ParsePartialFromIstream(std::istream* input) { - io::IstreamInputStream zero_copy_input(input); - return ParsePartialFromZeroCopyStream(&zero_copy_input) && input->eof(); -} - #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER namespace internal { @@ -547,7 +527,6 @@ const char* Message::_InternalParse(const char* ptr, } #endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER - void Message::SerializeWithCachedSizes(io::CodedOutputStream* output) const { const internal::SerializationTable* table = static_cast(InternalGetTable()); @@ -574,30 +553,6 @@ size_t Message::SpaceUsedLong() const { return GetReflection()->SpaceUsedLong(*this); } -bool Message::SerializeToFileDescriptor(int file_descriptor) const { - io::FileOutputStream output(file_descriptor); - return SerializeToZeroCopyStream(&output) && output.Flush(); -} - -bool Message::SerializePartialToFileDescriptor(int file_descriptor) const { - io::FileOutputStream output(file_descriptor); - return SerializePartialToZeroCopyStream(&output) && output.Flush(); -} - -bool Message::SerializeToOstream(std::ostream* output) const { - { - io::OstreamOutputStream zero_copy_output(output); - if (!SerializeToZeroCopyStream(&zero_copy_output)) return false; - } - return output->good(); -} - -bool Message::SerializePartialToOstream(std::ostream* output) const { - io::OstreamOutputStream zero_copy_output(output); - return SerializePartialToZeroCopyStream(&zero_copy_output); -} - - // ============================================================================= // Reflection and associated Template Specializations diff --git a/src/google/protobuf/message.h b/src/google/protobuf/message.h index 925c24944..8c9051aad 100644 --- a/src/google/protobuf/message.h +++ b/src/google/protobuf/message.h @@ -285,35 +285,6 @@ class PROTOBUF_EXPORT Message : public MessageLite { // Convenience function useful in GDB. Prints DebugString() to stdout. void PrintDebugString() const; - // Heavy I/O ------------------------------------------------------- - // Additional parsing and serialization methods not implemented by - // MessageLite because they are not supported by the lite library. - - // Parse a protocol buffer from a file descriptor. If successful, the entire - // input will be consumed. - bool ParseFromFileDescriptor(int file_descriptor); - // Like ParseFromFileDescriptor(), but accepts messages that are missing - // required fields. - bool ParsePartialFromFileDescriptor(int file_descriptor); - // Parse a protocol buffer from a C++ istream. If successful, the entire - // input will be consumed. - bool ParseFromIstream(std::istream* input); - // Like ParseFromIstream(), but accepts messages that are missing - // required fields. - bool ParsePartialFromIstream(std::istream* input); - - // Serialize the message and write it to the given file descriptor. All - // required fields must be set. - bool SerializeToFileDescriptor(int file_descriptor) const; - // Like SerializeToFileDescriptor(), but allows missing required fields. - bool SerializePartialToFileDescriptor(int file_descriptor) const; - // Serialize the message and write it to the given C++ ostream. All - // required fields must be set. - bool SerializeToOstream(std::ostream* output) const; - // Like SerializeToOstream(), but allows missing required fields. - bool SerializePartialToOstream(std::ostream* output) const; - - // Reflection-based methods ---------------------------------------- // These methods are pure-virtual in MessageLite, but Message provides // reflection-based default implementations. diff --git a/src/google/protobuf/message_lite.cc b/src/google/protobuf/message_lite.cc index e8f13440a..0848f34de 100644 --- a/src/google/protobuf/message_lite.cc +++ b/src/google/protobuf/message_lite.cc @@ -33,6 +33,8 @@ // Based on original Protocol Buffers design by // Sanjay Ghemawat, Jeff Dean, and others. +#include + #include #include #include @@ -43,11 +45,11 @@ #include #include #include +#include #include #include #include #include -#include #include #include @@ -276,6 +278,26 @@ bool MessageLite::ParsePartialFromZeroCopyStream( return ParseFrom(input); } +bool MessageLite::ParseFromFileDescriptor(int file_descriptor) { + io::FileInputStream input(file_descriptor); + return ParseFromZeroCopyStream(&input) && input.GetErrno() == 0; +} + +bool MessageLite::ParsePartialFromFileDescriptor(int file_descriptor) { + io::FileInputStream input(file_descriptor); + return ParsePartialFromZeroCopyStream(&input) && input.GetErrno() == 0; +} + +bool MessageLite::ParseFromIstream(std::istream* input) { + io::IstreamInputStream zero_copy_input(input); + return ParseFromZeroCopyStream(&zero_copy_input) && input->eof(); +} + +bool MessageLite::ParsePartialFromIstream(std::istream* input) { + io::IstreamInputStream zero_copy_input(input); + return ParsePartialFromZeroCopyStream(&zero_copy_input) && input->eof(); +} + bool MessageLite::MergePartialFromBoundedZeroCopyStream( io::ZeroCopyInputStream* input, int size) { return ParseFrom(internal::BoundedZCIS{input, size}); @@ -394,6 +416,29 @@ bool MessageLite::SerializePartialToZeroCopyStream( return SerializePartialToCodedStream(&encoder); } +bool MessageLite::SerializeToFileDescriptor(int file_descriptor) const { + io::FileOutputStream output(file_descriptor); + return SerializeToZeroCopyStream(&output) && output.Flush(); +} + +bool MessageLite::SerializePartialToFileDescriptor(int file_descriptor) const { + io::FileOutputStream output(file_descriptor); + return SerializePartialToZeroCopyStream(&output) && output.Flush(); +} + +bool MessageLite::SerializeToOstream(std::ostream* output) const { + { + io::OstreamOutputStream zero_copy_output(output); + if (!SerializeToZeroCopyStream(&zero_copy_output)) return false; + } + return output->good(); +} + +bool MessageLite::SerializePartialToOstream(std::ostream* output) const { + io::OstreamOutputStream zero_copy_output(output); + return SerializePartialToZeroCopyStream(&zero_copy_output); +} + bool MessageLite::AppendToString(std::string* output) const { GOOGLE_DCHECK(IsInitialized()) << InitializationErrorMessage("serialize", *this); return AppendPartialToString(output); @@ -474,6 +519,7 @@ void MessageLite::SerializeWithCachedSizes( output); } + // The table driven code optimizes the case that the CodedOutputStream buffer // is large enough to serialize into it directly. // If the proto is optimized for speed, this method will be overridden by diff --git a/src/google/protobuf/message_lite.h b/src/google/protobuf/message_lite.h index bc5d4c7ab..7f913fcf3 100644 --- a/src/google/protobuf/message_lite.h +++ b/src/google/protobuf/message_lite.h @@ -114,8 +114,8 @@ inline int ToIntSize(size_t size) { // // Pay special attention to the initialization state of the object. // 1. The object is "uninitialized" to begin with. -// 2. Call DefaultConstruct() only if the object is uninitialized. -// After the call, the object becomes "initialized". +// 2. Call Construct() or DefaultConstruct() only if the object is +// uninitialized. After the call, the object becomes "initialized". // 3. Call get() and get_mutable() only if the object is initialized. // 4. Call Destruct() only if the object is initialized. // After the call, the object becomes uninitialized. @@ -124,6 +124,11 @@ class ExplicitlyConstructed { public: void DefaultConstruct() { new (&union_) T(); } + template + void Construct(Args&&... args) { + new (&union_) T(std::forward(args)...); + } + void Destruct() { get_mutable()->~T(); } constexpr const T& get() const { return reinterpret_cast(union_); } @@ -262,6 +267,18 @@ class PROTOBUF_EXPORT MessageLite { // Like ParseFromZeroCopyStream(), but accepts messages that are missing // required fields. bool ParsePartialFromZeroCopyStream(io::ZeroCopyInputStream* input); + // Parse a protocol buffer from a file descriptor. If successful, the entire + // input will be consumed. + bool ParseFromFileDescriptor(int file_descriptor); + // Like ParseFromFileDescriptor(), but accepts messages that are missing + // required fields. + bool ParsePartialFromFileDescriptor(int file_descriptor); + // Parse a protocol buffer from a C++ istream. If successful, the entire + // input will be consumed. + bool ParseFromIstream(std::istream* input); + // Like ParseFromIstream(), but accepts messages that are missing + // required fields. + bool ParsePartialFromIstream(std::istream* input); // Read a protocol buffer from the given zero-copy input stream, expecting // the message to be exactly "size" bytes long. If successful, exactly // this many bytes will have been consumed from the input. @@ -355,12 +372,24 @@ class PROTOBUF_EXPORT MessageLite { // Like SerializeAsString(), but allows missing required fields. std::string SerializePartialAsString() const; + // Serialize the message and write it to the given file descriptor. All + // required fields must be set. + bool SerializeToFileDescriptor(int file_descriptor) const; + // Like SerializeToFileDescriptor(), but allows missing required fields. + bool SerializePartialToFileDescriptor(int file_descriptor) const; + // Serialize the message and write it to the given C++ ostream. All + // required fields must be set. + bool SerializeToOstream(std::ostream* output) const; + // Like SerializeToOstream(), but allows missing required fields. + bool SerializePartialToOstream(std::ostream* output) const; + // Like SerializeToString(), but appends to the data to the string's // existing contents. All required fields must be set. bool AppendToString(std::string* output) const; // Like AppendToString(), but allows missing required fields. bool AppendPartialToString(std::string* output) const; + // Computes the serialized size of the message. This recursively calls // ByteSizeLong() on all embedded messages. // diff --git a/src/google/protobuf/parse_context.h b/src/google/protobuf/parse_context.h index 5c8f986c1..5172af776 100644 --- a/src/google/protobuf/parse_context.h +++ b/src/google/protobuf/parse_context.h @@ -31,9 +31,9 @@ #ifndef GOOGLE_PROTOBUF_PARSE_CONTEXT_H__ #define GOOGLE_PROTOBUF_PARSE_CONTEXT_H__ +#include #include #include -#include #include #include diff --git a/src/google/protobuf/struct.pb.cc b/src/google/protobuf/struct.pb.cc index b7028f087..2cad5a9f5 100644 --- a/src/google/protobuf/struct.pb.cc +++ b/src/google/protobuf/struct.pb.cc @@ -390,16 +390,14 @@ void Struct::SerializeWithCachedSizes( } ::std::sort(&items[0], &items[static_cast(n)], Less()); for (size_type i = 0; i < n; i++) { - Struct_FieldsEntry_DoNotUse::MapEntryWrapper entry(nullptr, items[static_cast(i)]->first, items[static_cast(i)]->second); - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray(1, entry, output); + Struct_FieldsEntry_DoNotUse::Funcs::SerializeToCodedStream(1, items[static_cast(i)]->first, items[static_cast(i)]->second, output); Utf8Check::Check(&(*items[static_cast(i)])); } } else { for (::PROTOBUF_NAMESPACE_ID::Map< std::string, PROTOBUF_NAMESPACE_ID::Value >::const_iterator it = this->fields().begin(); it != this->fields().end(); ++it) { - Struct_FieldsEntry_DoNotUse::MapEntryWrapper entry(nullptr, it->first, it->second); - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray(1, entry, output); + Struct_FieldsEntry_DoNotUse::Funcs::SerializeToCodedStream(1, it->first, it->second, output); Utf8Check::Check(&(*it)); } } @@ -446,16 +444,14 @@ void Struct::SerializeWithCachedSizes( } ::std::sort(&items[0], &items[static_cast(n)], Less()); for (size_type i = 0; i < n; i++) { - Struct_FieldsEntry_DoNotUse::MapEntryWrapper entry(nullptr, items[static_cast(i)]->first, items[static_cast(i)]->second); - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::InternalWriteMessageNoVirtualToArray(1, entry, target); + target = Struct_FieldsEntry_DoNotUse::Funcs::SerializeToArray(1, items[static_cast(i)]->first, items[static_cast(i)]->second, target); Utf8Check::Check(&(*items[static_cast(i)])); } } else { for (::PROTOBUF_NAMESPACE_ID::Map< std::string, PROTOBUF_NAMESPACE_ID::Value >::const_iterator it = this->fields().begin(); it != this->fields().end(); ++it) { - Struct_FieldsEntry_DoNotUse::MapEntryWrapper entry(nullptr, it->first, it->second); - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::InternalWriteMessageNoVirtualToArray(1, entry, target); + target = Struct_FieldsEntry_DoNotUse::Funcs::SerializeToArray(1, it->first, it->second, target); Utf8Check::Check(&(*it)); } } @@ -488,9 +484,7 @@ size_t Struct::ByteSizeLong() const { for (::PROTOBUF_NAMESPACE_ID::Map< std::string, PROTOBUF_NAMESPACE_ID::Value >::const_iterator it = this->fields().begin(); it != this->fields().end(); ++it) { - Struct_FieldsEntry_DoNotUse::MapEntryWrapper entry(nullptr, it->first, it->second); - total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: - MessageSizeNoVirtual(entry); + total_size += Struct_FieldsEntry_DoNotUse::Funcs::ByteSizeLong(it->first, it->second); } int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); diff --git a/src/google/protobuf/unittest_lite.proto b/src/google/protobuf/unittest_lite.proto index f3ff2b5d1..d6f1149cf 100644 --- a/src/google/protobuf/unittest_lite.proto +++ b/src/google/protobuf/unittest_lite.proto @@ -174,8 +174,8 @@ message ForeignMessageLite { enum ForeignEnumLite { FOREIGN_LITE_FOO = 4; - FOREIGN_LITE_BAR = 5; FOREIGN_LITE_BAZ = 6; + FOREIGN_LITE_BAR = 5; } message TestPackedTypesLite { @@ -473,3 +473,16 @@ message PackedFixed32 { message NonPackedFixed32 { repeated fixed32 repeated_fixed32 = 2048; } + +// Test an enum that has multiple values with the same number. +message DupEnum { + enum TestEnumWithDupValueLite { + option allow_alias = true; + + FOO1 = 1; + BAR1 = 2; + BAZ = 3; + FOO2 = 1; + BAR2 = 2; + } +}